Merge pull request #9806 from taosdata/feature/vnode

Feature/vnode

.devcontainer/devcontainer.json

@@ -26,7 +26,8 @@
 		"eamodio.gitlens",
 		"matepek.vscode-catch2-test-adapter",
 		"spmeesseman.vscode-taskexplorer",
-		"cschlosser.doxdocgen"
+		"cschlosser.doxdocgen",
+		"urosvujosevic.explorer-manager"
 	],
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],

source/dnode/vnode/inc/vnode.h

@@ -35,28 +35,26 @@ typedef struct SDnode SDnode;
 typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
 
 typedef struct SVnodeCfg {
-  int32_t vgId;
-  SDnode *pDnode;
-  struct {
-    uint64_t wsize;
-    uint64_t ssize;
-    uint64_t lsize;
-    bool isHeapAllocator;
-  };
+  int32_t  vgId;
+  SDnode * pDnode;
+  uint64_t wsize;
+  uint64_t ssize;
+  uint64_t lsize;
+  bool     isHeapAllocator;
   uint32_t ttl;
   uint32_t keep;
-  bool isWeak;
+  bool     isWeak;
   STsdbCfg tsdbCfg;
   SMetaCfg metaCfg;
-  STqCfg tqCfg;
-  SWalCfg walCfg;
+  STqCfg   tqCfg;
+  SWalCfg  walCfg;
 } SVnodeCfg;
 
 typedef struct {
   int32_t           sver;
-  char             *timezone;
-  char             *locale;
-  char             *charset;
+  char *            timezone;
+  char *            locale;
+  char *            charset;
   uint16_t          nthreads;  // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
   PutReqToVQueryQFp putReqToVQueryQFp;
 } SVnodeOpt;

source/dnode/vnode/src/inc/vnodeDef.h

@@ -82,12 +82,12 @@ int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq);
 // For Log
 extern int32_t vDebugFlag;
 
-#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
-#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
-#define vWarn(...)  do { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
-#define vInfo(...)  do { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
-#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define vWarn(...)  do { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }}      while(0)
+#define vInfo(...)  do { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("VND ", 255, __VA_ARGS__); }}           while(0)
+#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -17,9 +17,11 @@
 
 int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp) {
   // Check if mem is there. If not, create one.
-  pTsdb->mem = tsdbNewMemTable(pTsdb);
   if (pTsdb->mem == NULL) {
-    return -1;
+    pTsdb->mem = tsdbNewMemTable(pTsdb);
+    if (pTsdb->mem == NULL) {
+      return -1;
+    }
   }
   return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
 }

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -116,7 +116,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
   // Check if it needs to commit
   if (vnodeShouldCommit(pVnode)) {
-    tsem_wait(&(pVnode->canCommit));
+    // tsem_wait(&(pVnode->canCommit));
     if (vnodeAsyncCommit(pVnode) < 0) {
       // TODO: handle error
     }

source/libs/tdb/CMakeLists.txt

@@ -1,10 +1,11 @@
-aux_source_directory(src TDB_SRC)
+
+set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr")
+foreach(TDB_SUBDIR ${TDB_SUBDIRS})
+    aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC)
+endforeach()
+
 add_library(tdb STATIC ${TDB_SRC})
-# target_include_directories(
-#     tkv
-#     PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
-#     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-# )
+
 target_include_directories(
     tdb
     PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
@@ -17,5 +18,5 @@ target_link_libraries(
 )
 
 if(${BUILD_TEST})
-    # add_subdirectory(test)
+    add_subdirectory(test)
 endif(${BUILD_TEST})

source/libs/tdb/inc/tdb.h

@@ -22,10 +22,14 @@
 extern "C" {
 #endif
 
+#define TDB_EXTERN
+#define TDB_PUBLIC
+#define TDB_STATIC static
+
 typedef enum {
-  TDB_BTREE = 0,
-  TDB_HASH,
-  TDB_HEAP,
+  TDB_BTREE_T = 0,
+  TDB_HASH_T,
+  TDB_HEAP_T,
 } tdb_db_t;
 
 // Forward declaration
@@ -39,9 +43,9 @@ typedef struct {
 } TDB_KEY, TDB_VALUE;
 
 // TDB Operations
-int tdbCreateDB(TDB** dbpp);
-int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags);
-int tdbCloseDB(TDB* dbp, uint32_t flags);
+TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type);
+TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags);
+TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags);
 
 #ifdef __cplusplus
 }

source/libs/tdb/src/db/tdbDB.c

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbDB.h"
+#include "tdb.h"
+
+TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) {
+  TDB* dbp;
+  int  ret;
+
+  dbp = calloc(1, sizeof(*dbp));
+  if (dbp == NULL) {
+    return -1;
+  }
+
+  dbp->pageSize = TDB_DEFAULT_PGSIZE;
+  dbp->type = type;
+
+  switch (type) {
+    case TDB_BTREE_T:
+      //   ret = tdbInitBtreeDB(dbp);
+      //   if (ret < 0) goto _err;
+      break;
+    case TDB_HASH_T:
+      //   ret = tdbInitHashDB(dbp);
+      //   if (ret < 0) goto _err;
+      break;
+    case TDB_HEAP_T:
+      //   ret = tdbInitHeapDB(dbp);
+      //   if (ret < 0) goto _err;
+      break;
+    default:
+      break;
+  }
+
+  *dbpp = dbp;
+  return 0;
+
+_err:
+  if (dbp) {
+    free(dbp);
+  }
+  *dbpp = NULL;
+  return 0;
+}
+
+TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags) {
+  // TODO
+  return 0;
+}
+
+TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) {
+  // TODO
+  return 0;
+}

source/libs/tdb/src/inc/tdbBtree.h

@@ -16,7 +16,7 @@
 #ifndef _TD_TDB_BTREE_H_
 #define _TD_TDB_BTREE_H_
 
-#include "tkvDef.h"
+#include "tdbDef.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -26,6 +26,8 @@ typedef struct {
   pgid_t root;  // root page number
 } TDB_BTREE;
 
+TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/src/inc/tdbDB.h

@@ -16,20 +16,22 @@
 #ifndef _TD_TDB_DB_H_
 #define _TD_TDB_DB_H_
 
+#include "tdb.h"
 #include "tdbBtree.h"
 #include "tdbHash.h"
+#include "tdbHeap.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
 struct TDB {
   pgsize_t pageSize;
   tdb_db_t type;
   union {
-    TDB_BTREE btree;
-    TDB_HASH  hash;
+    TDB_BTREE *btree;
+    TDB_HASH * hash;
+    TDB_HEAP * heap;
   } dbam;  // Different access methods
 };
 

source/libs/tdb/src/inc/tdbDef.h

@@ -24,16 +24,17 @@ extern "C" {
 
 // pgid_t
 typedef int32_t pgid_t;
-#define TKV_IVLD_PGID ((pgid_t)-1)
+#define TDB_IVLD_PGID ((pgid_t)-1)
 
 // framd_id_t
 typedef int32_t frame_id_t;
 
 // pgsize_t
 typedef int32_t pgsize_t;
-#define TKV_MIN_PGSIZE 512
-#define TKV_MAX_PGSIZE 16384
-#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
+#define TDB_MIN_PGSIZE 512
+#define TDB_MAX_PGSIZE 16384
+#define TDB_DEFAULT_PGSIZE 4096
+#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
 
 #ifdef __cplusplus
 }

source/libs/tdb/src/inc/tdbHash.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TKV_HAHS_H_
-#define _TD_TKV_HAHS_H_
+#ifndef _TD_TDB_HASH_H_
+#define _TD_TDB_HASH_H_
 
 #include "tdbDef.h"
 
@@ -26,8 +26,10 @@ typedef struct {
   // TODO
 } TDB_HASH;
 
+TDB_PUBLIC int tdbInitHashDB(TDB *dbp);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TKV_HAHS_H_*/
+#endif /*_TD_TDB_HASH_H_*/

source/libs/tdb/src/inc/tdbHeap.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDB_HEAP_H_
+#define _TD_TDB_HEAP_H_
+
+#include "tdbDef.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  // TODO
+} TDB_HEAP;
+
+TDB_PUBLIC int tdbInitHeapDB(TDB *dbp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDB_HEAP_H_*/

source/libs/tdb/test/tdbTest.cpp

@@ -6,9 +6,9 @@ TEST(tdb_api_test, tdb_create_open_close_db_test) {
   int  ret;
   TDB *dbp;
 
-  tdbCreateDB(&dbp);
+  tdbCreateDB(&dbp, TDB_BTREE_T);
 
-  tdbOpenDB(dbp, TDB_BTREE, 0);
+  tdbOpenDB(dbp, 0);
 
   tdbCloseDB(dbp, 0);
 }