more TDB

source/libs/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_subdirectory(transport)
 add_subdirectory(sync)
-# add_subdirectory(tdb)
+add_subdirectory(tdb)
 add_subdirectory(index)
 add_subdirectory(wal)
 add_subdirectory(parser)

source/libs/tdb/src/db/tdbBtree.c

@@ -87,7 +87,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
 
   *ppBt = NULL;
 
-  pBt = (SBTree *)calloc(1, sizeof(*pBt));
+  pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt));
   if (pBt == NULL) {
     return -1;
   }
@@ -121,7 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
   // TODO: pBt->root
   ret = tdbBtreeOpenImpl(pBt);
   if (ret < 0) {
-    free(pBt);
+    tdbOsFree(pBt);
     return -1;
   }
 
@@ -550,7 +550,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) {
           pCell = tdbPageGetCell(pParent, sIdx + i);
           szDivCell[i] = tdbBtreeCellSize(pParent, pCell);
-          pDivCell[i] = malloc(szDivCell[i]);
+          pDivCell[i] = tdbOsMalloc(szDivCell[i]);
           memcpy(pDivCell[i], pCell, szDivCell[i]);
         }
 
@@ -740,13 +740,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
               tdbBtreeDecodeCell(pPage, pCell, &cd);
 
               // TODO: pCell here may be inserted as an overflow cell, handle it
-              SCell *pNewCell = malloc(cd.kLen + 9);
+              SCell *pNewCell = tdbOsMalloc(cd.kLen + 9);
               int    szNewCell;
               SPgno  pgno;
               pgno = TDB_PAGE_PGNO(pNews[iNew]);
               tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell);
               tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0);
-              free(pNewCell);
+              tdbOsFree(pNewCell);
             }
 
             // move to next new page
@@ -798,7 +798,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
 
   for (int i = 0; i < 3; i++) {
     if (pDivCell[i]) {
-      free(pDivCell[i]);
+      tdbOsFree(pDivCell[i]);
     }
   }
 

source/libs/tdb/src/db/tdbDb.c

@@ -34,7 +34,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
 
   *ppDb = NULL;
 
-  pDb = (STDB *)calloc(1, sizeof(*pDb));
+  pDb = (STDB *)tdbOsCalloc(1, sizeof(*pDb));
   if (pDb == NULL) {
     return -1;
   }
@@ -126,7 +126,7 @@ int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
 
 int tdbDbcClose(STDBC *pDbc) {
   if (pDbc) {
-    free(pDbc);
+    tdbOsFree(pDbc);
   }
 
   return 0;

source/libs/tdb/src/db/tdbEnv.c

@@ -27,7 +27,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv)
   dsize = strlen(rootDir);
   zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
 
-  pPtr = (uint8_t *)calloc(1, zsize);
+  pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
   if (pPtr == NULL) {
     return -1;
   }

source/libs/tdb/src/db/tdbOs.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/libs/tdb/src/db/tdbPCache.c

@@ -63,7 +63,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
   void    *pPtr;
   SPage   *pPgHdr;
 
-  pCache = (SPCache *)calloc(1, sizeof(*pCache));
+  pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache));
   if (pCache == NULL) {
     return -1;
   }
@@ -72,7 +72,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
   pCache->cacheSize = cacheSize;
 
   if (tdbPCacheOpenImpl(pCache) < 0) {
-    free(pCache);
+    tdbOsFree(pCache);
     return -1;
   }
 
@@ -268,7 +268,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
   // Open the hash table
   pCache->nPage = 0;
   pCache->nHash = pCache->cacheSize;
-  pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *));
+  pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
   if (pCache->pgHash == NULL) {
     // TODO
     return -1;

source/libs/tdb/src/db/tdbPager.c

@@ -20,8 +20,8 @@ struct SPager {
   char    *jFileName;
   int      pageSize;
   uint8_t  fid[TDB_FILE_ID_LEN];
-  int      fd;
+  tdb_fd_t fd;
-  int      jfd;
+  tdb_fd_t jfd;
   SPCache *pCache;
   SPgno    dbFileSize;
   SPgno    dbOrigSize;
@@ -60,7 +60,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
   zsize = sizeof(*pPager)  /* SPager */
           + fsize + 1      /* dbFileName */
           + fsize + 8 + 1; /* jFileName */
-  pPtr = (uint8_t *)calloc(1, zsize);
+  pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
   if (pPtr == NULL) {
     return -1;
   }
@@ -80,7 +80,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
   // pPager->pCache
   pPager->pCache = pCache;
 
-  pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755);
+  pPager->fd = tdbOsOpen(pPager->dbFileName, O_RDWR | O_CREAT, 0755);
   if (pPager->fd < 0) {
     return -1;
   }
@@ -168,7 +168,7 @@ int tdbPagerBegin(SPager *pPager) {
   }
 
   // Open the journal
-  pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755);
+  pPager->jfd = tdbOsOpen(pPager->jFileName, O_RDWR | O_CREAT, 0755);
   if (pPager->jfd < 0) {
     return -1;
   }
@@ -208,7 +208,7 @@ int tdbPagerCommit(SPager *pPager) {
 
   fsync(pPager->fd);
 
-  close(pPager->jfd);
+  tdbOsClose(pPager->jfd);
   remove(pPager->jFileName);
   pPager->jfd = -1;
 

source/libs/tdb/src/inc/tdbInt.h

@@ -148,6 +148,8 @@ typedef struct SPager  SPager;
 typedef struct SPCache SPCache;
 typedef struct SPage   SPage;
 
+#include "tdbOs.h"
+
 #include "tdbUtil.h"
 
 #include "tdbPCache.h"

source/libs/tdb/src/inc/tdbOs.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TDB_OS_H_
+#define _TDB_OS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// TODO: kmake
+#define TDB_FOR_TDENGINE
+
+// For memor
+#ifdef TDB_FOR_TDENGINE
+#define tdbOsMalloc  taosMemoryMalloc
+#define tdbOsCalloc  taosMemoryCalloc
+#define tdbOsRealloc taosMemoryRealloc
+#define tdbOsFree    taosMemoryFree
+#else
+#define tdbOsMalloc  malloc
+#define tdbOsCalloc  calloc
+#define tdbOsRealloc realloc
+#define tdbOsFree    free
+#endif
+
+// For file
+#ifdef TDB_FOR_TDENGINE
+typedef TdFilePtr tdb_fd_t;
+
+#define tdbOsOpen  taosOpenFile
+#define tdbOsClose taosCloseFile
+#define tdbOsRead  taosReadFile
+#define tdbOsPRead taosPReadFile
+#define tdbOsWrite taosWriteFile
+#else
+#define tdbOsOpen  open
+#define tdbOsClose close
+#define tdbOsRead  read
+#define tdbOsPRead pread
+#define tdbOsWrite write
+#endif
+
+// For threads and lock
+#ifdef TDB_FOR_TDENGINE
+
+// spin lock
+typedef TdThreadSpinlock tdb_spinlock_t;
+
+#define tdbSpinlockInit    taosThreadSpinInit
+#define tdbSpinlockDestroy taosThreadSpinDestroy
+#define tdbSpinlockLock    taosThreadSpinLock
+#define tdbSpinlockUnlock  taosThreadSpinUnlock
+#define tdbSpinlockTrylock
+
+#else
+
+// spin lock
+typedef pthread_spinlock_t tdb_spinlock_t;
+
+#define tdbSpinlockInit    pthread_spin_init
+#define tdbSpinlockDestroy pthread_spin_destroy
+#define tdbSpinlockLock    pthread_spin_lock
+#define tdbSpinlockUnlock  pthread_spin_unlock
+#define tdbSpinlockTrylock pthread_spin_trylock
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TDB_OS_H_*/

source/libs/tdb/src/inc/tdbPage.h

@@ -53,10 +53,10 @@ typedef struct __attribute__((__packed__)) {
 } SPageFtr;
 
 struct SPage {
-  pthread_spinlock_t lock;
+  tdb_spinlock_t lock;
-  int                pageSize;
+  int            pageSize;
-  u8                *pData;
+  u8            *pData;
-  SPageMethods      *pPageMethods;
+  SPageMethods  *pPageMethods;
   // Fields below used by pager and am
   u8       *pPageHdr;
   u8       *pCellIdx;
@@ -80,21 +80,21 @@ struct SPage {
 #define P_LOCK_BUSY 1
 #define P_LOCK_FAIL -1
 
-#define TDB_INIT_PAGE_LOCK(pPage)    pthread_spin_init(&((pPage)->lock), 0)
+#define TDB_INIT_PAGE_LOCK(pPage)    tdbSpinlockInit(&((pPage)->lock), 0)
-#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock))
+#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock))
-#define TDB_LOCK_PAGE(pPage)         pthread_spin_lock(&((pPage)->lock))
+#define TDB_LOCK_PAGE(pPage)         tdbSpinlockLock(&((pPage)->lock))
-#define TDB_UNLOCK_PAGE(pPage)       pthread_spin_unlock(&((pPage)->lock))
+#define TDB_UNLOCK_PAGE(pPage)       tdbSpinlockUnlock(&((pPage)->lock))
-#define TDB_TRY_LOCK_PAGE(pPage)                       \
+#define TDB_TRY_LOCK_PAGE(pPage)                     \
-  ({                                                   \
+  ({                                                 \
-    int ret;                                           \
+    int ret;                                         \
-    if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \
+    if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \
-      ret = P_LOCK_SUCC;                               \
+      ret = P_LOCK_SUCC;                             \
-    } else if (errno == EBUSY) {                       \
+    } else if (errno == EBUSY) {                     \
-      ret = P_LOCK_BUSY;                               \
+      ret = P_LOCK_BUSY;                             \
-    } else {                                           \
+    } else {                                         \
-      ret = P_LOCK_FAIL;                               \
+      ret = P_LOCK_FAIL;                             \
-    }                                                  \
+    }                                                \
-    ret;                                               \
+    ret;                                             \
   })
 
 // APIs

source/libs/tdb/src/inc/tdbUtil.h

@@ -40,37 +40,37 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize);
 int tdbPRead(int fd, void *pData, int count, i64 offset);
 int tdbWrite(int fd, void *pData, int count);
 
-#define TDB_REALLOC(PTR, SIZE)                                                          \
+#define TDB_REALLOC(PTR, SIZE)                                                               \
-  ({                                                                                    \
+  ({                                                                                         \
-    void *nPtr;                                                                         \
+    void *nPtr;                                                                              \
-    if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) {                                 \
+    if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) {                                      \
-      nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \
+      nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \
-      if (nPtr) {                                                                       \
+      if (nPtr) {                                                                            \
-        ((int *)nPtr)[0] = (SIZE);                                                      \
+        ((int *)nPtr)[0] = (SIZE);                                                           \
-        nPtr = (char *)nPtr + sizeof(int);                                              \
+        nPtr = (char *)nPtr + sizeof(int);                                                   \
-      }                                                                                 \
+      }                                                                                      \
-    } else {                                                                            \
+    } else {                                                                                 \
-      nPtr = (PTR);                                                                     \
+      nPtr = (PTR);                                                                          \
-    }                                                                                   \
+    }                                                                                        \
-    nPtr;                                                                               \
+    nPtr;                                                                                    \
   })
 
 #define TDB_FREE(PTR)                    \
   do {                                   \
     if (PTR) {                           \
-      free((char *)(PTR) - sizeof(int)); \
+      tdbOsFree((char *)(PTR) - sizeof(int)); \
     }                                    \
   } while (0)
 
-static inline void *tdbOsMalloc(void *arg, size_t size) {
+static inline void *tdbDefaultMalloc(void *arg, size_t size) {
   void *ptr;
 
-  ptr = malloc(size);
+  ptr = tdbOsMalloc(size);
 
   return ptr;
 }
 
-static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); }
+static inline void tdbDefaultFree(void *arg, void *ptr) { tdbOsFree(ptr); }
 
 static inline int tdbPutVarInt(u8 *p, int v) {
   int n = 0;

source/libs/tdb/src/page/tdbPage.c

@@ -48,7 +48,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
   *ppPage = NULL;
   size = pageSize + sizeof(*pPage);
   if (xMalloc == NULL) {
-    xMalloc = tdbOsMalloc;
+    xMalloc = tdbDefaultMalloc;
   }
 
   ptr = (u8 *)((*xMalloc)(arg, size));
@@ -76,7 +76,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
   u8 *ptr;
 
   if (!xFree) {
-    xFree = tdbOsFree;
+    xFree = tdbDefaultFree;
   }
 
   ptr = pPage->pData;