From 6d1477e6932dec81860eb3207c1001786276ae58 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 02:26:27 +0000 Subject: [PATCH 01/11] more TDB --- source/libs/CMakeLists.txt | 2 +- source/libs/tdb/src/db/tdbBtree.c | 12 ++--- source/libs/tdb/src/db/tdbDb.c | 4 +- source/libs/tdb/src/db/tdbEnv.c | 2 +- source/libs/tdb/src/db/tdbOs.c | 14 +++++ source/libs/tdb/src/db/tdbPCache.c | 6 +-- source/libs/tdb/src/db/tdbPager.c | 12 ++--- source/libs/tdb/src/inc/tdbInt.h | 2 + source/libs/tdb/src/inc/tdbOs.h | 85 ++++++++++++++++++++++++++++++ source/libs/tdb/src/inc/tdbPage.h | 38 ++++++------- source/libs/tdb/src/inc/tdbUtil.h | 34 ++++++------ source/libs/tdb/src/page/tdbPage.c | 4 +- 12 files changed, 158 insertions(+), 57 deletions(-) create mode 100644 source/libs/tdb/src/db/tdbOs.c create mode 100644 source/libs/tdb/src/inc/tdbOs.h diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index b448a43dcb..a1b9337fa8 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -# add_subdirectory(tdb) +add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5ead5ac8a4..0800ebbc49 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -87,7 +87,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S *ppBt = NULL; - pBt = (SBTree *)calloc(1, sizeof(*pBt)); + pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); if (pBt == NULL) { return -1; } @@ -121,7 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S // TODO: pBt->root ret = tdbBtreeOpenImpl(pBt); if (ret < 0) { - free(pBt); + tdbOsFree(pBt); return -1; } @@ -550,7 +550,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { pCell = tdbPageGetCell(pParent, sIdx + i); szDivCell[i] = tdbBtreeCellSize(pParent, pCell); - pDivCell[i] = malloc(szDivCell[i]); + pDivCell[i] = tdbOsMalloc(szDivCell[i]); memcpy(pDivCell[i], pCell, szDivCell[i]); } @@ -740,13 +740,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeDecodeCell(pPage, pCell, &cd); // TODO: pCell here may be inserted as an overflow cell, handle it - SCell *pNewCell = malloc(cd.kLen + 9); + SCell *pNewCell = tdbOsMalloc(cd.kLen + 9); int szNewCell; SPgno pgno; pgno = TDB_PAGE_PGNO(pNews[iNew]); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); - free(pNewCell); + tdbOsFree(pNewCell); } // move to next new page @@ -798,7 +798,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { for (int i = 0; i < 3; i++) { if (pDivCell[i]) { - free(pDivCell[i]); + tdbOsFree(pDivCell[i]); } } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 4e74dc4cbb..1117550ed1 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -34,7 +34,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDB *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)tdbOsCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -126,7 +126,7 @@ int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { int tdbDbcClose(STDBC *pDbc) { if (pDbc) { - free(pDbc); + tdbOsFree(pDbc); } return 0; diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 9a4dcdbcd5..ad3b5c41f2 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -27,7 +27,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) dsize = strlen(rootDir); zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/db/tdbOs.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 3c7d037faa..1e93d87ab8 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -63,7 +63,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { void *pPtr; SPage *pPgHdr; - pCache = (SPCache *)calloc(1, sizeof(*pCache)); + pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache)); if (pCache == NULL) { return -1; } @@ -72,7 +72,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { pCache->cacheSize = cacheSize; if (tdbPCacheOpenImpl(pCache) < 0) { - free(pCache); + tdbOsFree(pCache); return -1; } @@ -268,7 +268,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // Open the hash table pCache->nPage = 0; pCache->nHash = pCache->cacheSize; - pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); + pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { // TODO return -1; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index a45a4dad52..db2568c5aa 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -20,8 +20,8 @@ struct SPager { char *jFileName; int pageSize; uint8_t fid[TDB_FILE_ID_LEN]; - int fd; - int jfd; + tdb_fd_t fd; + tdb_fd_t jfd; SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; @@ -60,7 +60,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { zsize = sizeof(*pPager) /* SPager */ + fsize + 1 /* dbFileName */ + fsize + 8 + 1; /* jFileName */ - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { return -1; } @@ -80,7 +80,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { // pPager->pCache pPager->pCache = pCache; - pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755); + pPager->fd = tdbOsOpen(pPager->dbFileName, O_RDWR | O_CREAT, 0755); if (pPager->fd < 0) { return -1; } @@ -168,7 +168,7 @@ int tdbPagerBegin(SPager *pPager) { } // Open the journal - pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755); + pPager->jfd = tdbOsOpen(pPager->jFileName, O_RDWR | O_CREAT, 0755); if (pPager->jfd < 0) { return -1; } @@ -208,7 +208,7 @@ int tdbPagerCommit(SPager *pPager) { fsync(pPager->fd); - close(pPager->jfd); + tdbOsClose(pPager->jfd); remove(pPager->jFileName); pPager->jfd = -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 06c09aba3f..c926d9358c 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -148,6 +148,8 @@ typedef struct SPager SPager; typedef struct SPCache SPCache; typedef struct SPage SPage; +#include "tdbOs.h" + #include "tdbUtil.h" #include "tdbPCache.h" diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h new file mode 100644 index 0000000000..b05ce47ac5 --- /dev/null +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_OS_H_ +#define _TDB_OS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// TODO: kmake +#define TDB_FOR_TDENGINE + +// For memor +#ifdef TDB_FOR_TDENGINE +#define tdbOsMalloc taosMemoryMalloc +#define tdbOsCalloc taosMemoryCalloc +#define tdbOsRealloc taosMemoryRealloc +#define tdbOsFree taosMemoryFree +#else +#define tdbOsMalloc malloc +#define tdbOsCalloc calloc +#define tdbOsRealloc realloc +#define tdbOsFree free +#endif + +// For file +#ifdef TDB_FOR_TDENGINE +typedef TdFilePtr tdb_fd_t; + +#define tdbOsOpen taosOpenFile +#define tdbOsClose taosCloseFile +#define tdbOsRead taosReadFile +#define tdbOsPRead taosPReadFile +#define tdbOsWrite taosWriteFile +#else +#define tdbOsOpen open +#define tdbOsClose close +#define tdbOsRead read +#define tdbOsPRead pread +#define tdbOsWrite write +#endif + +// For threads and lock +#ifdef TDB_FOR_TDENGINE + +// spin lock +typedef TdThreadSpinlock tdb_spinlock_t; + +#define tdbSpinlockInit taosThreadSpinInit +#define tdbSpinlockDestroy taosThreadSpinDestroy +#define tdbSpinlockLock taosThreadSpinLock +#define tdbSpinlockUnlock taosThreadSpinUnlock +#define tdbSpinlockTrylock + +#else + +// spin lock +typedef pthread_spinlock_t tdb_spinlock_t; + +#define tdbSpinlockInit pthread_spin_init +#define tdbSpinlockDestroy pthread_spin_destroy +#define tdbSpinlockLock pthread_spin_lock +#define tdbSpinlockUnlock pthread_spin_unlock +#define tdbSpinlockTrylock pthread_spin_trylock + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_OS_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index a6f9fbf615..49aa9f4398 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -53,10 +53,10 @@ typedef struct __attribute__((__packed__)) { } SPageFtr; struct SPage { - pthread_spinlock_t lock; - int pageSize; - u8 *pData; - SPageMethods *pPageMethods; + tdb_spinlock_t lock; + int pageSize; + u8 *pData; + SPageMethods *pPageMethods; // Fields below used by pager and am u8 *pPageHdr; u8 *pCellIdx; @@ -80,21 +80,21 @@ struct SPage { #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 -#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0) -#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock)) -#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock)) -#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) \ - ({ \ - int ret; \ - if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \ - ret = P_LOCK_SUCC; \ - } else if (errno == EBUSY) { \ - ret = P_LOCK_BUSY; \ - } else { \ - ret = P_LOCK_FAIL; \ - } \ - ret; \ +#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) \ + ({ \ + int ret; \ + if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \ + ret = P_LOCK_SUCC; \ + } else if (errno == EBUSY) { \ + ret = P_LOCK_BUSY; \ + } else { \ + ret = P_LOCK_FAIL; \ + } \ + ret; \ }) // APIs diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index f7b5a31012..0633d4e48b 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -40,37 +40,37 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); int tdbWrite(int fd, void *pData, int count); -#define TDB_REALLOC(PTR, SIZE) \ - ({ \ - void *nPtr; \ - if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ - nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ - if (nPtr) { \ - ((int *)nPtr)[0] = (SIZE); \ - nPtr = (char *)nPtr + sizeof(int); \ - } \ - } else { \ - nPtr = (PTR); \ - } \ - nPtr; \ +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ }) #define TDB_FREE(PTR) \ do { \ if (PTR) { \ - free((char *)(PTR) - sizeof(int)); \ + tdbOsFree((char *)(PTR) - sizeof(int)); \ } \ } while (0) -static inline void *tdbOsMalloc(void *arg, size_t size) { +static inline void *tdbDefaultMalloc(void *arg, size_t size) { void *ptr; - ptr = malloc(size); + ptr = tdbOsMalloc(size); return ptr; } -static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } +static inline void tdbDefaultFree(void *arg, void *ptr) { tdbOsFree(ptr); } static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index f1eee48b0e..7f92f740da 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -48,7 +48,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); if (xMalloc == NULL) { - xMalloc = tdbOsMalloc; + xMalloc = tdbDefaultMalloc; } ptr = (u8 *)((*xMalloc)(arg, size)); @@ -76,7 +76,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) u8 *ptr; if (!xFree) { - xFree = tdbOsFree; + xFree = tdbDefaultFree; } ptr = pPage->pData; From 8f6ba1fc5b05919c0414a02a98ed2bfa958d2901 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 02:40:55 +0000 Subject: [PATCH 02/11] more TDB --- source/libs/tdb/src/db/tdbPCache.c | 28 ++++++++++++++-------------- source/libs/tdb/src/db/tdbPager.c | 2 +- source/libs/tdb/src/inc/tdbOs.h | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 1e93d87ab8..981dd63593 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -15,16 +15,16 @@ #include "tdbInt.h" struct SPCache { - int pageSize; - int cacheSize; - pthread_mutex_t mutex; - int nFree; - SPage *pFree; - int nPage; - int nHash; - SPage **pgHash; - int nRecyclable; - SPage lru; + int pageSize; + int cacheSize; + tdb_mutex_t mutex; + int nFree; + SPage *pFree; + int nPage; + int nHash; + SPage **pgHash; + int nRecyclable; + SPage lru; }; #define PCACHE_PAGE_HASH(pPgid) \ @@ -116,13 +116,13 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { } } -static void tdbPCacheInitLock(SPCache *pCache) { pthread_mutex_init(&(pCache->mutex), NULL); } +static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); } -static void tdbPCacheClearLock(SPCache *pCache) { pthread_mutex_destroy(&(pCache->mutex)); } +static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); } -static void tdbPCacheLock(SPCache *pCache) { pthread_mutex_lock(&(pCache->mutex)); } +static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); } -static void tdbPCacheUnlock(SPCache *pCache) { pthread_mutex_unlock(&(pCache->mutex)); } +static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); } static bool tdbPCacheLocked(SPCache *pCache) { assert(0); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index db2568c5aa..5f79d60a78 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -206,7 +206,7 @@ int tdbPagerCommit(SPager *pPager) { // TODO: release the page } - fsync(pPager->fd); + tdbOsFSync(pPager->fd); tdbOsClose(pPager->jfd); remove(pPager->jFileName); diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index b05ce47ac5..0d7a3299f1 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -45,12 +45,14 @@ typedef TdFilePtr tdb_fd_t; #define tdbOsRead taosReadFile #define tdbOsPRead taosPReadFile #define tdbOsWrite taosWriteFile +#define tdbOsFSync taosFsyncFile #else #define tdbOsOpen open #define tdbOsClose close #define tdbOsRead read #define tdbOsPRead pread #define tdbOsWrite write +#define tdbOsFSync fsync #endif // For threads and lock @@ -65,6 +67,14 @@ typedef TdThreadSpinlock tdb_spinlock_t; #define tdbSpinlockUnlock taosThreadSpinUnlock #define tdbSpinlockTrylock +// mutex lock +typedef TdThreadMutex tdb_mutex_t; + +#define tdbMutexInit taosThreadMutexInit +#define tdbMutexDestroy taosThreadMutexDestroy +#define tdbMutexLock taosThreadMutexLock +#define tdbMutexUnlock taosThreadMutexUnlock + #else // spin lock @@ -76,6 +86,14 @@ typedef pthread_spinlock_t tdb_spinlock_t; #define tdbSpinlockUnlock pthread_spin_unlock #define tdbSpinlockTrylock pthread_spin_trylock +// mutex lock +typedef pthread_mutex_t tdb_mutex_t; + +#define tdbMutexInit pthread_mutex_init +#define tdbMutexDestroy pthread_mutex_destroy +#define tdbMutexLock pthread_mutex_lock +#define tdbMutexUnlock pthread_mutex_unlock + #endif #ifdef __cplusplus From a8882c5c1469681bba434741bdeaf2dca9039ee2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 02:50:39 +0000 Subject: [PATCH 03/11] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbDb.c | 2 +- source/libs/tdb/src/db/tdbPager.c | 2 +- source/libs/tdb/src/inc/tdbOs.h | 8 ++++---- source/libs/tdb/src/page/tdbPage.c | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0800ebbc49..faced8e839 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -166,7 +166,7 @@ int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, // TODO: refact code here pBt = pBtc->pBt; if (!pBt->pTmp) { - pBt->pTmp = (u8 *)malloc(pBt->pageSize); + pBt->pTmp = (u8 *)tdbOsMalloc(pBt->pageSize); if (pBt->pTmp == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 1117550ed1..499116f091 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -101,7 +101,7 @@ int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { STDBC *pDbc = NULL; *ppDbc = NULL; - pDbc = malloc(sizeof(*pDbc)); + pDbc = (STDBC *)tdbOsMalloc(sizeof(*pDbc)); if (pDbc == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5f79d60a78..f55f427b36 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -222,7 +222,7 @@ static int tdbPagerReadPage(SPager *pPager, SPage *pPage) { ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0); offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize); - ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset); + ret = tdbOsPRead(pPager->fd, pPage->pData, pPager->pageSize, offset); if (ret < 0) { // TODO: handle error return -1; diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 0d7a3299f1..ac538341f1 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -49,9 +49,9 @@ typedef TdFilePtr tdb_fd_t; #else #define tdbOsOpen open #define tdbOsClose close -#define tdbOsRead read -#define tdbOsPRead pread -#define tdbOsWrite write +#define tdbOsRead read // TODO +#define tdbOsPRead pread // TODO +#define tdbOsWrite write // TODO #define tdbOsFSync fsync #endif @@ -65,7 +65,7 @@ typedef TdThreadSpinlock tdb_spinlock_t; #define tdbSpinlockDestroy taosThreadSpinDestroy #define tdbSpinlockLock taosThreadSpinLock #define tdbSpinlockUnlock taosThreadSpinUnlock -#define tdbSpinlockTrylock +#define tdbSpinlockTrylock pthread_spin_trylock // TODO // mutex lock typedef TdThreadMutex tdb_mutex_t; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 7f92f740da..3301202a33 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -144,7 +144,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl } // TODO: here has memory leak - pNewCell = (SCell *)malloc(szCell); + pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pCell, szCell); pPage->apOvfl[iOvfl] = pNewCell; From 3146ac2d3b33cbc8b908de0e11d62cd531fe78c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:00:02 +0000 Subject: [PATCH 04/11] more TDB --- source/libs/tdb/src/db/tdbPager.c | 8 ++++---- source/libs/tdb/src/inc/tdbOs.h | 32 +++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index f55f427b36..b811fcb135 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -80,7 +80,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { // pPager->pCache pPager->pCache = pCache; - pPager->fd = tdbOsOpen(pPager->dbFileName, O_RDWR | O_CREAT, 0755); + pPager->fd = tdbOsOpen(pPager->dbFileName, O_RDWR | O_CREAT); if (pPager->fd < 0) { return -1; } @@ -90,7 +90,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { return -1; } - pPager->jfd = -1; + // pPager->jfd = -1; pPager->pageSize = tdbPCacheGetPageSize(pCache); *ppPager = pPager; @@ -168,7 +168,7 @@ int tdbPagerBegin(SPager *pPager) { } // Open the journal - pPager->jfd = tdbOsOpen(pPager->jFileName, O_RDWR | O_CREAT, 0755); + pPager->jfd = tdbOsOpen(pPager->jFileName, O_RDWR | O_CREAT); if (pPager->jfd < 0) { return -1; } @@ -210,7 +210,7 @@ int tdbPagerCommit(SPager *pPager) { tdbOsClose(pPager->jfd); remove(pPager->jFileName); - pPager->jfd = -1; + // pPager->jfd = -1; return 0; } diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index ac538341f1..12d81e3fcc 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -20,10 +20,10 @@ extern "C" { #endif -// TODO: kmake +// TODO: use cmake to control the option #define TDB_FOR_TDENGINE -// For memor +// For memory #ifdef TDB_FOR_TDENGINE #define tdbOsMalloc taosMemoryMalloc #define tdbOsCalloc taosMemoryCalloc @@ -36,8 +36,10 @@ extern "C" { #define tdbOsFree free #endif -// For file +// For file and directory #ifdef TDB_FOR_TDENGINE + +// file typedef TdFilePtr tdb_fd_t; #define tdbOsOpen taosOpenFile @@ -46,28 +48,42 @@ typedef TdFilePtr tdb_fd_t; #define tdbOsPRead taosPReadFile #define tdbOsWrite taosWriteFile #define tdbOsFSync taosFsyncFile + +// directory +#define tdbOsMkdir taosMkDir +#define tdbOsRmdir taosRemoveDir + #else + +// file +typedef int tdb_fd_t; + #define tdbOsOpen open #define tdbOsClose close #define tdbOsRead read // TODO #define tdbOsPRead pread // TODO #define tdbOsWrite write // TODO #define tdbOsFSync fsync + +// directory +#define tdbOsMkdir mkdir +#define tdbOsRmdir rmdir + #endif // For threads and lock #ifdef TDB_FOR_TDENGINE -// spin lock +/* spin lock */ typedef TdThreadSpinlock tdb_spinlock_t; #define tdbSpinlockInit taosThreadSpinInit #define tdbSpinlockDestroy taosThreadSpinDestroy #define tdbSpinlockLock taosThreadSpinLock #define tdbSpinlockUnlock taosThreadSpinUnlock -#define tdbSpinlockTrylock pthread_spin_trylock // TODO +#define tdbSpinlockTrylock pthread_spin_trylock -// mutex lock +/* mutex lock */ typedef TdThreadMutex tdb_mutex_t; #define tdbMutexInit taosThreadMutexInit @@ -77,7 +93,7 @@ typedef TdThreadMutex tdb_mutex_t; #else -// spin lock +/* spin lock */ typedef pthread_spinlock_t tdb_spinlock_t; #define tdbSpinlockInit pthread_spin_init @@ -86,7 +102,7 @@ typedef pthread_spinlock_t tdb_spinlock_t; #define tdbSpinlockUnlock pthread_spin_unlock #define tdbSpinlockTrylock pthread_spin_trylock -// mutex lock +/* mutex lock */ typedef pthread_mutex_t tdb_mutex_t; #define tdbMutexInit pthread_mutex_init From ca8e40687e8e4f65512fcb9bf53ee9257bffd1d1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:13:59 +0000 Subject: [PATCH 05/11] more TDB --- source/libs/tdb/src/inc/tdbOs.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 12d81e3fcc..c737ed867d 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -25,21 +25,25 @@ extern "C" { // For memory #ifdef TDB_FOR_TDENGINE + #define tdbOsMalloc taosMemoryMalloc #define tdbOsCalloc taosMemoryCalloc #define tdbOsRealloc taosMemoryRealloc #define tdbOsFree taosMemoryFree + #else + #define tdbOsMalloc malloc #define tdbOsCalloc calloc #define tdbOsRealloc realloc #define tdbOsFree free + #endif // For file and directory #ifdef TDB_FOR_TDENGINE -// file +/* file */ typedef TdFilePtr tdb_fd_t; #define tdbOsOpen taosOpenFile @@ -49,13 +53,13 @@ typedef TdFilePtr tdb_fd_t; #define tdbOsWrite taosWriteFile #define tdbOsFSync taosFsyncFile -// directory +/* directory */ #define tdbOsMkdir taosMkDir #define tdbOsRmdir taosRemoveDir #else -// file +/* file */ typedef int tdb_fd_t; #define tdbOsOpen open @@ -65,7 +69,7 @@ typedef int tdb_fd_t; #define tdbOsWrite write // TODO #define tdbOsFSync fsync -// directory +/* directory */ #define tdbOsMkdir mkdir #define tdbOsRmdir rmdir From b704fa2354bdddb78219a5d0f31a85835e911f32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:33:25 +0000 Subject: [PATCH 06/11] more TDB --- source/libs/tdb/inc/tdb.h | 37 ------------------------------- source/libs/tdb/src/db/tdbPager.c | 8 +++---- source/libs/tdb/src/inc/tdbInt.h | 3 ++- source/libs/tdb/src/inc/tdbOs.h | 20 +++++++++-------- 4 files changed, 17 insertions(+), 51 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 467e40325e..71ac3d97ed 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,43 +22,6 @@ extern "C" { #endif -// typedef struct STDb TDB; -// typedef struct STDbEnv TENV; -// typedef struct STDbCurosr TDBC; - -// typedef int32_t pgsz_t; -// typedef int32_t cachesz_t; - -// typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); - -// // TEVN -// int tdbEnvCreate(TENV **ppEnv, const char *rootDir); -// int tdbEnvOpen(TENV *ppEnv); -// int tdbEnvClose(TENV *pEnv); - -// int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); -// pgsz_t tdbEnvGetPageSize(TENV *pEnv); -// cachesz_t tdbEnvGetCacheSize(TENV *pEnv); - -// int tdbEnvBeginTxn(TENV *pEnv); -// int tdbEnvCommit(TENV *pEnv); - -// // TDB -// int tdbCreate(TDB **ppDb); -// int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); -// int tdbClose(TDB *pDb); -// int tdbDrop(TDB *pDb); - -// int tdbSetKeyLen(TDB *pDb, int klen); -// int tdbSetValLen(TDB *pDb, int vlen); -// int tdbSetDup(TDB *pDb, int dup); -// int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); -// int tdbGetKeyLen(TDB *pDb); -// int tdbGetValLen(TDB *pDb); -// int tdbGetDup(TDB *pDb); - -// int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); - #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index b811fcb135..0abc64d0b0 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -377,12 +377,12 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { pgno = TDB_PAGE_PGNO(pPage); - ret = tdbWrite(pPager->jfd, &pgno, sizeof(pgno)); + ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { return -1; } - ret = tdbWrite(pPager->jfd, pPage->pData, pPage->pageSize); + ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { return -1; } @@ -395,12 +395,12 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { int ret; offset = pPage->pageSize * TDB_PAGE_PGNO(pPage); - if (lseek(pPager->fd, offset, SEEK_SET) < 0) { + if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); return -1; } - ret = tdbWrite(pPager->fd, pPage->pData, pPage->pageSize); + ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { ASSERT(0); return -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index c926d9358c..84c92f896c 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -16,10 +16,11 @@ #ifndef _TD_TDB_INTERNAL_H_ #define _TD_TDB_INTERNAL_H_ +#include "os.h" #include "tlist.h" #include "tlockfree.h" -// #include "tdb.h" +#include "tdb.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index c737ed867d..fd51136957 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -23,7 +23,7 @@ extern "C" { // TODO: use cmake to control the option #define TDB_FOR_TDENGINE -// For memory +// For memory ----------------- #ifdef TDB_FOR_TDENGINE #define tdbOsMalloc taosMemoryMalloc @@ -40,18 +40,19 @@ extern "C" { #endif -// For file and directory +// For file and directory ----------------- #ifdef TDB_FOR_TDENGINE /* file */ typedef TdFilePtr tdb_fd_t; -#define tdbOsOpen taosOpenFile -#define tdbOsClose taosCloseFile -#define tdbOsRead taosReadFile -#define tdbOsPRead taosPReadFile -#define tdbOsWrite taosWriteFile -#define tdbOsFSync taosFsyncFile +#define tdbOsOpen taosOpenFile +#define tdbOsClose(FD) taosCloseFile(&(FD)) +#define tdbOsRead taosReadFile +#define tdbOsPRead taosPReadFile +#define tdbOsWrite taosWriteFile +#define tdbOsFSync taosFsyncFile +#define tdbOsLSeek taosLSeekFile /* directory */ #define tdbOsMkdir taosMkDir @@ -68,6 +69,7 @@ typedef int tdb_fd_t; #define tdbOsPRead pread // TODO #define tdbOsWrite write // TODO #define tdbOsFSync fsync +#define tdbOsLSeek lseek /* directory */ #define tdbOsMkdir mkdir @@ -75,7 +77,7 @@ typedef int tdb_fd_t; #endif -// For threads and lock +// For threads and lock ----------------- #ifdef TDB_FOR_TDENGINE /* spin lock */ From 5c9c9695e5700b5e9fbadf1921163d9f59dc19e5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:41:27 +0000 Subject: [PATCH 07/11] more TDB --- source/libs/tdb/CMakeLists.txt | 1 + source/libs/tdb/src/db/tdbOs.c | 25 ++++++++++++++++++++++++- source/libs/tdb/src/inc/tdbOs.h | 8 +++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 8612c9dc8f..0b1378dc5a 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -9,6 +9,7 @@ target_sources(tdb "src/db/tdbDb.c" "src/db/tdbEnv.c" "src/db/tdbTxn.c" + "src/db/tdbOs.c" "src/page/tdbPage.c" "src/page/tdbPageL.c" ) diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index 6dea4a4e57..d8df761069 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -11,4 +11,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tdbInt.h" + +// tdbOsRead +i64 tdbOsRead(tdb_fd_t fd, void *pBuf, i64 nBytes) { + // TODO + ASSERT(0); + return 0; +} + +// tdbOsPRead +i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset) { + // TODO + ASSERT(0); + return 0; +} + +// tdbOsWrite +i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index fd51136957..851dd69c29 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -65,9 +65,11 @@ typedef int tdb_fd_t; #define tdbOsOpen open #define tdbOsClose close -#define tdbOsRead read // TODO -#define tdbOsPRead pread // TODO -#define tdbOsWrite write // TODO + +i64 tdbOsRead(tdb_fd_t fd, void *pBuf, i64 nBytes); +i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset); +i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes); + #define tdbOsFSync fsync #define tdbOsLSeek lseek From 5f90bae8bbce6eaef907a084d8e60fce979fdd47 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:51:02 +0000 Subject: [PATCH 08/11] more TDB --- source/libs/tdb/src/db/tdbOs.c | 40 ++++++++++++++++++++++--- source/libs/tdb/src/db/tdbPager.c | 2 +- source/libs/tdb/src/db/tdbUtil.c | 50 +------------------------------ source/libs/tdb/src/inc/tdbOs.h | 10 ++++--- source/libs/tdb/src/inc/tdbUtil.h | 16 +++------- 5 files changed, 48 insertions(+), 70 deletions(-) diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index d8df761069..210d582b92 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -31,7 +31,39 @@ i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset) { // tdbOsWrite i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file + // TODO + ASSERT(0); + return 0; +} + +#if 0 +int tdbPRead(int fd, void *pData, int count, i64 offset) { + void *pBuf; + int nbytes; + i64 ioffset; + int iread; + + pBuf = pData; + nbytes = count; + ioffset = offset; + while (nbytes > 0) { + iread = pread(fd, pBuf, nbytes, ioffset); + if (iread < 0) { + /* TODO */ + } else if (iread == 0) { + return (count - iread); + } + + nbytes = nbytes - iread; + pBuf = (void *)((u8 *)pBuf + iread); + ioffset += iread; + } + + return count; +} + +int tdbWrite(int fd, void *pData, int count) { + // TODO + return write(fd, pData, count); +} +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 0abc64d0b0..4fac00d5ad 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -209,7 +209,7 @@ int tdbPagerCommit(SPager *pPager) { tdbOsFSync(pPager->fd); tdbOsClose(pPager->jfd); - remove(pPager->jFileName); + tdbOsRemove(pPager->jFileName); // pPager->jfd = -1; return 0; diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index e7de0a859a..4abc890f94 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -33,28 +33,10 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { return 0; } -// int tdbCheckFileAccess(const char *pathname, int mode) { -// int flags = 0; - -// if (mode & TDB_F_OK) { -// flags |= F_OK; -// } - -// if (mode & TDB_R_OK) { -// flags |= R_OK; -// } - -// if (mode & TDB_W_OK) { -// flags |= W_OK; -// } - -// return access(pathname, flags); -// } - int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) { struct stat st; int ret; - int64_t file_size = 0; + int64_t file_size = 0; ret = taosStatFile(fname, &file_size, NULL); if (ret != 0) { return -1; @@ -64,34 +46,4 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) { *pSize = file_size / pgSize; return 0; -} - -int tdbPRead(int fd, void *pData, int count, i64 offset) { - void *pBuf; - int nbytes; - i64 ioffset; - int iread; - - pBuf = pData; - nbytes = count; - ioffset = offset; - while (nbytes > 0) { - iread = pread(fd, pBuf, nbytes, ioffset); - if (iread < 0) { - /* TODO */ - } else if (iread == 0) { - return (count - iread); - } - - nbytes = nbytes - iread; - pBuf = (void *)((u8 *)pBuf + iread); - ioffset += iread; - } - - return count; -} - -int tdbWrite(int fd, void *pData, int count) { - // TODO - return write(fd, pData, count); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 851dd69c29..98a4a190e0 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -53,6 +53,7 @@ typedef TdFilePtr tdb_fd_t; #define tdbOsWrite taosWriteFile #define tdbOsFSync taosFsyncFile #define tdbOsLSeek taosLSeekFile +#define tdbOsRemove remove /* directory */ #define tdbOsMkdir taosMkDir @@ -70,12 +71,13 @@ i64 tdbOsRead(tdb_fd_t fd, void *pBuf, i64 nBytes); i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset); i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes); -#define tdbOsFSync fsync -#define tdbOsLSeek lseek +#define tdbOsFSync fsync +#define tdbOsLSeek lseek +#define tdbOsRemove remove /* directory */ -#define tdbOsMkdir mkdir -#define tdbOsRmdir rmdir +#define tdbOsMkdir mkdir +#define tdbOsRmdir rmdir #endif diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 0633d4e48b..6e6faf9b74 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -30,16 +30,8 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -// #define TDB_F_OK 0x1 -// #define TDB_R_OK 0x2 -// #define TDB_W_OK 0x4 -// int tdbCheckFileAccess(const char *pathname, int mode); - int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); -int tdbPRead(int fd, void *pData, int count, i64 offset); -int tdbWrite(int fd, void *pData, int count); - #define TDB_REALLOC(PTR, SIZE) \ ({ \ void *nPtr; \ @@ -55,11 +47,11 @@ int tdbWrite(int fd, void *pData, int count); nPtr; \ }) -#define TDB_FREE(PTR) \ - do { \ - if (PTR) { \ +#define TDB_FREE(PTR) \ + do { \ + if (PTR) { \ tdbOsFree((char *)(PTR) - sizeof(int)); \ - } \ + } \ } while (0) static inline void *tdbDefaultMalloc(void *arg, size_t size) { From 54ca6c4e8cd06c390a9df303cfd3446983f6addd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 03:55:37 +0000 Subject: [PATCH 09/11] make TDB can compile --- source/libs/tdb/src/db/tdbUtil.c | 15 --------------- source/libs/tdb/src/inc/tdbUtil.h | 2 -- 2 files changed, 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 4abc890f94..fc299b3fc1 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -30,20 +30,5 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { ((uint64_t *)fileid)[2] = taosRand(); } - return 0; -} - -int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) { - struct stat st; - int ret; - int64_t file_size = 0; - ret = taosStatFile(fname, &file_size, NULL); - if (ret != 0) { - return -1; - } - - ASSERT(file_size % pgSize == 0); - - *pSize = file_size / pgSize; return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 6e6faf9b74..c06d9d18c9 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -30,8 +30,6 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); - #define TDB_REALLOC(PTR, SIZE) \ ({ \ void *nPtr; \ From 4c43901c44f371fa4ce8aea6d3883f83cc27f9d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 05:40:54 +0000 Subject: [PATCH 10/11] more TDB --- source/libs/tdb/src/db/tdbOs.c | 101 ++++++++++++++++++++------------ source/libs/tdb/src/inc/tdbOs.h | 8 +-- 2 files changed, 69 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index 210d582b92..e52f788da9 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -15,55 +15,84 @@ #include "tdbInt.h" +#ifndef TDB_FOR_TDENGINE + // tdbOsRead -i64 tdbOsRead(tdb_fd_t fd, void *pBuf, i64 nBytes) { - // TODO - ASSERT(0); - return 0; +i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) { + i64 nRead = 0; + i64 iRead = 0; + u8 *pBuf = (u8 *)pData; + + while (nBytes > 0) { + iRead = read(fd, pBuf, nBytes); + if (iRead < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (iRead == 0) { + break; + } + + nRead += iRead; + pBuf += iRead; + nBytes -= iRead; + } + + return nRead; } // tdbOsPRead -i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset) { - // TODO - ASSERT(0); - return 0; +i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) { + i64 nRead = 0; + i64 iRead = 0; + i64 iOffset = offset; + u8 *pBuf = (u8 *)pData; + + while (nBytes > 0) { + iRead = pread(fd, pBuf, nBytes, iOffset); + if (iRead < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (iRead == 0) { + break; + } + + nRead += iRead; + pBuf += iRead; + iOffset += iRead; + nBytes -= iRead; + } + + return nRead; } // tdbOsWrite -i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes) { - // TODO - ASSERT(0); - return 0; -} +i64 taosWriteFile(tdb_fd_t fd, const void *pData, i64 nBytes) { + i64 nWrite = 0; + i64 iWrite = 0; + u8 *pBuf = (u8 *)pData; -#if 0 -int tdbPRead(int fd, void *pData, int count, i64 offset) { - void *pBuf; - int nbytes; - i64 ioffset; - int iread; + while (nBytes > 0) { + iWrite = write(fd, pBuf, nBytes); + if (iWrite < 0) { + if (errno == EINTR) { + continue; + } - pBuf = pData; - nbytes = count; - ioffset = offset; - while (nbytes > 0) { - iread = pread(fd, pBuf, nbytes, ioffset); - if (iread < 0) { - /* TODO */ - } else if (iread == 0) { - return (count - iread); + return -1; } - nbytes = nbytes - iread; - pBuf = (void *)((u8 *)pBuf + iread); - ioffset += iread; + nWrite += iWrite; + pBuf += iWrite; + nBytes -= iWrite; } - return count; + return nWrite; } -int tdbWrite(int fd, void *pData, int count) { - // TODO - return write(fd, pData, count); -} #endif \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 98a4a190e0..b8ae85ea65 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -21,7 +21,7 @@ extern "C" { #endif // TODO: use cmake to control the option -#define TDB_FOR_TDENGINE +// #define TDB_FOR_TDENGINE // For memory ----------------- #ifdef TDB_FOR_TDENGINE @@ -67,9 +67,9 @@ typedef int tdb_fd_t; #define tdbOsOpen open #define tdbOsClose close -i64 tdbOsRead(tdb_fd_t fd, void *pBuf, i64 nBytes); -i64 tdbOsPRead(tdb_fd_t fd, void *pBuf, i64 nBytes, i64 offset); -i64 taosWriteFile(tdb_fd_t fd, const void *pBuf, i64 nBytes); +i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes); +i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset); +i64 taosWriteFile(tdb_fd_t fd, const void *pData, i64 nBytes); #define tdbOsFSync fsync #define tdbOsLSeek lseek From b23d2c7b6ec3c3e9a332390f52c426d82b38c682 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 05:44:03 +0000 Subject: [PATCH 11/11] make TDB can compile --- source/libs/tdb/src/db/tdbOs.c | 2 +- source/libs/tdb/src/inc/tdbOs.h | 4 ++-- source/libs/tdb/test/tdbTest.cpp | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index e52f788da9..5ca3578985 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -72,7 +72,7 @@ i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) { } // tdbOsWrite -i64 taosWriteFile(tdb_fd_t fd, const void *pData, i64 nBytes) { +i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) { i64 nWrite = 0; i64 iWrite = 0; u8 *pBuf = (u8 *)pData; diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index b8ae85ea65..751c105913 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -21,7 +21,7 @@ extern "C" { #endif // TODO: use cmake to control the option -// #define TDB_FOR_TDENGINE +#define TDB_FOR_TDENGINE // For memory ----------------- #ifdef TDB_FOR_TDENGINE @@ -69,7 +69,7 @@ typedef int tdb_fd_t; i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes); i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset); -i64 taosWriteFile(tdb_fd_t fd, const void *pData, i64 nBytes); +i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes); #define tdbOsFSync fsync #define tdbOsLSeek lseek diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 6889e33902..e8c6477de0 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -11,7 +11,7 @@ typedef struct SPoolMem { } SPoolMem; static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); + SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); pPool->prev = pPool->next = pPool; pPool->size = 0; @@ -31,12 +31,12 @@ static void closePool(SPoolMem *pPool) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - free(pMem); + tdbOsFree(pMem); } while (1); assert(pPool->size == 0); - free(pPool); + tdbOsFree(pPool); } #define clearPool closePool @@ -46,7 +46,7 @@ static void *poolMalloc(void *arg, int size) { SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pMem; - pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); + pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); if (pMem == NULL) { assert(0); } @@ -73,7 +73,7 @@ static void poolFree(void *arg, void *ptr) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - free(pMem); + tdbOsFree(pMem); } static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {