Merge remote-tracking branch 'origin/feature/vnode' into feature/dnode3

.gitignore

@@ -98,6 +98,6 @@ tramp
 .\#*
 TAGS
 
-deps/*
+contrib/*
-!deps/CMakeLists.txt
+!contrib/CMakeLists.txt
-!deps/test
+!contrib/test

CMakeLists.txt

@@ -6,77 +6,14 @@ project(
     DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)"
 )
 
-# ============================================================================
-# DEPENDENCIES
-# ============================================================================
 set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
+set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
 include(${CMAKE_SUPPORT_DIR}/cmake.options)
 
-function(cat IN_FILE OUT_FILE)
-    file(READ ${IN_FILE} CONTENTS)
-    file(APPEND ${OUT_FILE} "${CONTENTS}")
-endfunction(cat IN_FILE OUT_FILE)
-
-set(DEPS_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
-configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-
-## googletest
-if(${BUILD_TEST})
-    cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-endif(${BUILD_TEST})
-
-## lz4
-cat("${CMAKE_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-
-## zlib
-cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-
-## cJson
-cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-
-## leveldb
-if(${BUILD_WITH_LEVELDB})
-    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-endif(${BUILD_WITH_LEVELDB})
-
-## rocksdb
-if(${BUILD_WITH_ROCKSDB})
-    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-    add_definitions(-DUSE_ROCKSDB)
-endif(${BUILD_WITH_ROCKSDB})
-
-## bdb
-if(${BUILD_WITH_BDB})
-    cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-endif(${BUILD_WITH_DBD})
-
-## sqlite
-if(${BUILD_WITH_SQLITE})
-    cat("${CMAKE_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-endif(${BUILD_WITH_SQLITE})
-
-## lucene
-if(${BUILD_WITH_LUCENE})
-    cat("${CMAKE_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-    add_definitions(-DUSE_LUCENE)
-endif(${BUILD_WITH_LUCENE})
-
-## NuRaft
-if(${BUILD_WITH_NURAFT})
-    cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${DEPS_TMP_FILE})
-endif(${BUILD_WITH_NURAFT})
-
-## download dependencies
-configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
-execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
-        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/deps/deps-download")
-execute_process(COMMAND "${CMAKE_COMMAND}" --build .
-        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/deps/deps-download")
-
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma")
 
-# deps
+# contrib
-add_subdirectory(deps)
+add_subdirectory(contrib)
 
 # api
 add_library(api INTERFACE)

cmake/bdb_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(bdb
         GIT_REPOSITORY https://github.com/berkeleydb/libdb.git
         GIT_TAG v5.3.28
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/bdb"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/bdb"
-        BINARY_DIR "${CMAKE_SOURCE_DIR}/deps/bdb"
+        BINARY_DIR "${CMAKE_CONTRIB_DIR}/bdb"
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND "./dist/configure"
         BUILD_COMMAND "$(MAKE)"

cmake/cjson_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(cjson
         GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git
         GIT_TAG v1.7.15
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/cJson"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cJson"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/cmake.options

@@ -52,5 +52,5 @@ option(
 option(
     BUILD_DOCS
     "If use doxygen build documents" 
-    ON
+    OFF
 )

cmake/gtest_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(googletest
         GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git
         GIT_TAG release-1.11.0
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/googletest"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/googletest"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/leveldb_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(leveldb
         GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
         GIT_TAG master
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/leveldb"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/lucene_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # lucene
 ExternalProject_Add(lucene
         GIT_REPOSITORY https://github.com/yihaoDeng/LucenePlusPlus.git
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lucene"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/lucene"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/lz4_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(lz4
         GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git
         GIT_TAG v1.9.3
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lz4"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/lz4"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/nuraft_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(NuRaft
         GIT_REPOSITORY https://github.com/eBay/NuRaft.git
         GIT_TAG v1.3.0
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/nuraft"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/nuraft"
-        BINARY_DIR "${CMAKE_SOURCE_DIR}/deps/nuraft"
+        BINARY_DIR "${CMAKE_CONTRIB_DIR}/nuraft"
         CONFIGURE_COMMAND "./prepare.sh"
         BUILD_COMMAND ""
         INSTALL_COMMAND ""

cmake/rocksdb_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(rocksdb
         GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
         GIT_TAG v6.23.3
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/rocksdb"
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""
         INSTALL_COMMAND ""

cmake/sqlite_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(sqlite
         GIT_REPOSITORY https://github.com/sqlite/sqlite.git
         GIT_TAG version-3.36.0
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/sqlite"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/sqlite"
-        BINARY_DIR "${CMAKE_SOURCE_DIR}/deps/sqlite"
+        BINARY_DIR "${CMAKE_CONTRIB_DIR}/sqlite"
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND "./configure"
         BUILD_COMMAND "$(MAKE)"

cmake/zlib_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(zlib
         GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
         GIT_TAG v1.2.11
-        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/zlib"
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/zlib"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

contrib/CMakeLists.txt

@@ -1,5 +1,69 @@
 # ================================================================================================
-# DEPENDENCIES
+# Download
+# ================================================================================================
+function(cat IN_FILE OUT_FILE)
+    file(READ ${IN_FILE} CONTENTS)
+    file(APPEND ${OUT_FILE} "${CONTENTS}")
+endfunction(cat IN_FILE OUT_FILE)
+
+set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
+configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+
+# googletest
+if(${BUILD_TEST})
+    cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_TEST})
+
+# lz4
+cat("${CMAKE_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+
+# zlib
+cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+
+# cJson
+cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+
+# leveldb
+if(${BUILD_WITH_LEVELDB})
+    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_WITH_LEVELDB})
+
+# rocksdb
+if(${BUILD_WITH_ROCKSDB})
+    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    add_definitions(-DUSE_ROCKSDB)
+endif(${BUILD_WITH_ROCKSDB})
+
+# bdb
+if(${BUILD_WITH_BDB})
+    cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_WITH_DBD})
+
+# sqlite
+if(${BUILD_WITH_SQLITE})
+    cat("${CMAKE_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_WITH_SQLITE})
+
+# lucene
+if(${BUILD_WITH_LUCENE})
+    cat("${CMAKE_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    add_definitions(-DUSE_LUCENE)
+endif(${BUILD_WITH_LUCENE})
+
+# NuRaft
+if(${BUILD_WITH_NURAFT})
+    cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_WITH_NURAFT})
+
+# download dependencies
+configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
+execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
+        WORKING_DIRECTORY "${CMAKE_CONTRIB_DIR}/deps-download")
+execute_process(COMMAND "${CMAKE_COMMAND}" --build .
+        WORKING_DIRECTORY "${CMAKE_CONTRIB_DIR}/deps-download")
+
+# ================================================================================================
+# Build
 # ================================================================================================
 # googletest
 if(${BUILD_TEST})
@@ -82,7 +146,7 @@ endif(${BUILD_WITH_NURAFT})
 
 # BDB
 if(${BUILD_WITH_BDB})
-    add_library(bdb STATIC IMPORTED)
+    add_library(bdb STATIC IMPORTED GLOBAL)
     set_target_properties(bdb PROPERTIES
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb"
@@ -93,8 +157,9 @@ if(${BUILD_WITH_BDB})
 endif(${BUILD_WITH_BDB})
 
 # SQLite
+# see https://stackoverflow.com/questions/8774593/cmake-link-to-external-library#comment58570736_10550334
 if(${BUILD_WITH_SQLITE})
-    add_library(sqlite STATIC IMPORTED)
+    add_library(sqlite STATIC IMPORTED GLOBAL)
     set_target_properties(sqlite PROPERTIES
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/.libs/libsqlite3.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/sqlite"
@@ -109,7 +174,7 @@ endif(${BUILD_WITH_SQLITE})
 
 
 # ================================================================================================
-# DEPENDENCY TEST
+# Build test
 # ================================================================================================
 if(${BUILD_DEPENDENCY_TESTS})
     add_subdirectory(test)

source/dnode/vnode/meta/CMakeLists.txt

@@ -7,6 +7,7 @@ target_include_directories(
 )
 target_link_libraries(
     meta
+    PUBLIC sqlite
     PUBLIC common
     PUBLIC tkv
 )

source/dnode/vnode/meta/inc/metaDB.h

@@ -17,6 +17,7 @@
 #define _TD_META_DB_H_
 
 #include "rocksdb/c.h"
+#include "sqlite3.h"
 
 #include "meta.h"
 
@@ -29,7 +30,7 @@ typedef struct {
   rocksdb_t *nameDb;    // name -> uid
   rocksdb_t *tagDb;     // uid -> tag
   rocksdb_t *schemaDb;  // uid+version -> schema
-  rocksdb_t *mapDb;     // suid -> uid_list
+  sqlite3 *  mapDb;     // suid -> uid_list
 } meta_db_t;
 
 int  metaOpenDB(SMeta *pMeta);

source/dnode/vnode/meta/src/metaDB.c

@@ -17,7 +17,7 @@
 
 static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema);
 static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion);
-static int  metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid);
+// static int  metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid);
 
 #define SCHEMA_KEY_LEN (sizeof(tb_uid_t) + sizeof(int))
 
@@ -65,8 +65,14 @@ int metaOpenDB(SMeta *pMeta) {
   META_OPEN_DB_IMPL(pMeta->pDB->schemaDb, options, dir, err);
 
   // mapDb
-  sprintf(dir, "%s/map_db", pMeta->path);
+  sprintf(dir, "%s/meta.db", pMeta->path);
-  META_OPEN_DB_IMPL(pMeta->pDB->mapDb, options, dir, err);
+  if (sqlite3_open(dir, &(pMeta->pDB->mapDb)) != SQLITE_OK) {
+    // TODO
+  }
+
+  // // set read uncommitted
+  sqlite3_exec(pMeta->pDB->mapDb, "PRAGMA read_uncommitted=true;", 0, 0, 0);
+  sqlite3_exec(pMeta->pDB->mapDb, "BEGIN;", 0, 0, 0);
 
   rocksdb_options_destroy(options);
   return 0;
@@ -82,7 +88,12 @@ int metaOpenDB(SMeta *pMeta) {
 
 void metaCloseDB(SMeta *pMeta) {
   if (pMeta->pDB) {
-    META_CLOSE_DB_IMPL(pMeta->pDB->mapDb);
+    if (pMeta->pDB->mapDb) {
+      sqlite3_exec(pMeta->pDB->mapDb, "COMMIT;", 0, 0, 0);
+      sqlite3_close(pMeta->pDB->mapDb);
+      pMeta->pDB->mapDb = NULL;
+    }
+
     META_CLOSE_DB_IMPL(pMeta->pDB->schemaDb);
     META_CLOSE_DB_IMPL(pMeta->pDB->tagDb);
     META_CLOSE_DB_IMPL(pMeta->pDB->nameDb);
@@ -97,6 +108,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) {
   char *   err = NULL;
   size_t   size;
   char     pBuf[1024];  // TODO
+  char     sql[128];
 
   rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
 
@@ -124,8 +136,12 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) {
       // save schemaDB
       metaSaveSchemaDB(pMeta, uid, pTbOptions->stbCfg.pSchema);
 
-      // save mapDB (really need?)
+      // // save mapDB (really need?)
-      rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err);
+      // rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err);
+      sprintf(sql, "create table st_%" PRIu64 " (uid BIGINT);", uid);
+      if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) {
+        // fprintf(stderr, "Failed to create table, since %s\n", err);
+      }
       break;
     case META_CHILD_TABLE:
       // save tagDB
@@ -133,7 +149,10 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) {
                   kvRowLen(pTbOptions->ctbCfg.pTag), &err);
 
       // save mapDB
-      metaSaveMapDB(pMeta, pTbOptions->ctbCfg.suid, uid);
+      sprintf(sql, "insert into st_%" PRIu64 " values (%" PRIu64 ");", pTbOptions->ctbCfg.suid, uid);
+      if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) {
+        fprintf(stderr, "failed to insert data, since %s\n", err);
+      }
       break;
     default:
       ASSERT(0);
@@ -172,32 +191,32 @@ static void metaGetSchemaDBKey(char *key, tb_uid_t uid, int sversion) {
   *(int *)POINTER_SHIFT(key, sizeof(tb_uid_t)) = sversion;
 }
 
-static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) {
+// static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) {
-  size_t vlen;
+//   size_t vlen;
-  char * val;
+//   char * val;
-  char * err = NULL;
+//   char * err = NULL;
 
-  rocksdb_readoptions_t *ropt = rocksdb_readoptions_create();
+//   rocksdb_readoptions_t *ropt = rocksdb_readoptions_create();
-  val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err);
+//   val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err);
-  rocksdb_readoptions_destroy(ropt);
+//   rocksdb_readoptions_destroy(ropt);
 
-  void *nval = malloc(vlen + sizeof(uid));
+//   void *nval = malloc(vlen + sizeof(uid));
-  if (nval == NULL) {
+//   if (nval == NULL) {
-    return -1;
+//     return -1;
-  }
+//   }
 
-  if (vlen) {
+//   if (vlen) {
-    memcpy(nval, val, vlen);
+//     memcpy(nval, val, vlen);
-  }
+//   }
-  memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid));
+//   memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid));
 
-  rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
+//   rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create();
-  rocksdb_writeoptions_disable_WAL(wopt, 1);
+//   rocksdb_writeoptions_disable_WAL(wopt, 1);
 
-  rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err);
+//   rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err);
 
-  rocksdb_writeoptions_destroy(wopt);
+//   rocksdb_writeoptions_destroy(wopt);
-  free(nval);
+//   free(nval);
 
-  return 0;
+//   return 0;
-}
+// }