Merge remote-tracking branch 'origin/3.0_refact' into feature/dnode3

cmake/cjson_CMakeLists.txt.in

@@ -1,7 +1,7 @@
 
 # cjson
 ExternalProject_Add(cjson
-        GIT_REPOSITORY git@github.com:taosdata-contrib/cJSON.git
+        GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git
         GIT_TAG v1.7.15
         SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/cJson"
         BINARY_DIR ""

cmake/gtest_CMakeLists.txt.in

@@ -1,7 +1,7 @@
 
 # googletest
 ExternalProject_Add(googletest
-        GIT_REPOSITORY git@github.com:taosdata-contrib/googletest.git
+        GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git
         GIT_TAG release-1.11.0
         SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/googletest"
         BINARY_DIR ""

cmake/lz4_CMakeLists.txt.in

@@ -1,7 +1,7 @@
 
 # lz4
 ExternalProject_Add(lz4
-        GIT_REPOSITORY git@github.com:taosdata-contrib/lz4.git
+        GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git
         GIT_TAG v1.9.3
         SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lz4"
         BINARY_DIR ""

cmake/zlib_CMakeLists.txt.in

@@ -1,7 +1,7 @@
 
 # zlib
 ExternalProject_Add(zlib
-        GIT_REPOSITORY git@github.com:taosdata-contrib/zlib.git
+        GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
         GIT_TAG v1.2.11
         SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/zlib"
         BINARY_DIR ""

include/common/taosMsg.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TAOS_MSG_H_
+#define _TD_TAOS_MSG_H_
+
+typedef struct {
+  /* data */
+} SSubmitReq;
+
+typedef struct {
+  /* data */
+} SSubmitRsp;
+
+typedef struct {
+  /* data */
+} SSubmitReqReader;
+
+typedef struct {
+  /* data */
+} SCreateTableReq;
+
+typedef struct {
+  /* data */
+} SCreateTableRsp;
+
+typedef struct {
+  /* data */
+} SDropTableReq;
+
+typedef struct {
+  /* data */
+} SDropTableRsp;
+
+typedef struct {
+  /* data */
+} SAlterTableReq;
+
+typedef struct {
+  /* data */
+} SAlterTableRsp;
+
+#endif /*_TD_TAOS_MSG_H_*/

include/libs/tkv/tkv.h

@@ -20,6 +20,53 @@
 extern "C" {
 #endif
 
+typedef struct tkv_db_s tkv_db_t;
+
+typedef struct {
+  /* data */
+} tkv_key_t;
+
+typedef struct {
+  bool    pinned;
+  int64_t ref;  // TODO: use util library
+                // TODO: add a RW latch here
+  uint64_t offset;
+  void *   pObj;
+} tkv_obj_t;
+
+typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *);
+typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *);
+typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj);
+typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj);
+typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
+typedef void (*tkv_obj_destroy_fn_t)(void *);
+
+typedef struct {
+  uint64_t             memLimit;
+  tkv_get_key_fn_t     getKey;
+  tkv_obj_encode_fn_t  encode;
+  tkv_obj_decode_fn_t  decode;
+  tkv_obj_comp_fn_t    compare;
+  tkv_obj_destroy_fn_t destroy;
+} tkv_db_option_t;
+
+tkv_db_t *       tkvOpenDB(char *dir, tkv_db_option_t *);
+int              tkvCloseDB(tkv_db_t *);
+int              tkvPut(tkv_db_t *, tkv_obj_t *);
+int              tkvPutBatch(tkv_db_t *, tkv_obj_t **, int);  // TODO: use array here
+const tkv_obj_t *tkvGet(tkv_key_t *);
+int              tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **);  // TODO: use array here
+int              tkvDrop(tkv_db_t *, tkv_key_t *);
+int              tkvDropBatch(tkv_db_t *, tkv_key_t **, int);  // TODO: use array here
+int              tkvCommit(tkv_db_t *, void * /*TODO*/);
+
+typedef struct {
+} tkv_db_iter_t;
+
+tkv_db_iter_t *  tkvIterNew(tkv_db_t *);
+void             tkvIterFree(tkv_db_iter_t *);
+const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
+
 #ifdef __cplusplus
 }
 #endif

include/server/vnode/meta/meta.h

@@ -16,10 +16,19 @@
 #ifndef _TD_META_H_
 #define _TD_META_H_
 
+#include "taosMsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SMeta SMeta;
+
+int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq);
+int metaDropTable(SMeta *pMeta, SDropTableReq *pReq);
+int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq);
+int metaCommit(SMeta *pMeta);
+
 #ifdef __cplusplus
 }
 #endif

include/server/vnode/tq/tq.h

@@ -20,6 +20,11 @@
 extern "C" {
 #endif
 
+typedef struct STQ STQ;
+
+int tqPushMsg(void *);
+int tqCommit(STQ *pTQ);
+
 #ifdef __cplusplus
 }
 #endif

include/server/vnode/tsdb/tsdb.h

@@ -17,6 +17,7 @@
 #define _TD_TSDB_H_
 
 #include "os.h"
+#include "taosMsg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,16 +25,32 @@ extern "C" {
 
 typedef struct STsdb STsdb;
 typedef struct {
+  int32_t id; // TODO: use a global definition
+  int32_t days;
+  int32_t keep;
+  int32_t keep1;
+  int32_t keep2;
+  int32_t minRows;
+  int32_t maxRows;
+  int8_t  precision;
+  int8_t  update;
 } STsdbCfg;
 
-int    tsdbInit(int nthreads);
+// Module init and clear
+int tsdbInit();
 int tsdbClear();
+
+// Repository operations
 int    tsdbCreateRepo(int id);
 int    tsdbDropRepo(int id);
 STsdb *tsdbOpenRepo(STsdbCfg *pCfg);
 int    tsdbCloseRepo(STsdb *pTsdb);
 int    tsdbForceCloseRepo(STsdb *pTsdb);
 
+// Data commit
+int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg);
+int tsdbCommit(STsdb *pTsdb);
+
 #ifdef __cplusplus
 }
 #endif

include/util/amalloc.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_AMALLOC_H_
+#define _TD_AMALLOC_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Interfaces to implement
+typedef struct {
+  void *(*malloc)(void *, size_t size);
+  void *(*calloc)(void *, size_t nmemb, size_t size);
+  void (*free)(void *ptr, size_t size); // Do we need to set size in the allocated memory?
+  void *(*realloc)(void *ptr, size_t size);
+} SMemAllocatorIf;
+
+typedef struct {
+  void *          impl;
+  SMemAllocatorIf interface;
+} SMemAllocator;
+
+#define amalloc(allocator, size) \
+  ((allocator) ? (*((allocator)->interface.malloc))((allocator)->impl, (size)) : malloc(size))
+#define acalloc(allocator, nmemb, size) \
+  ((allocator) ? (*((allocator)->interface.calloc))((allocator)->impl, (nmemb), (size)) : calloc((nmemb), (size)))
+#define arealloc(allocator, ptr, size) \
+  ((allocator) ? (*((allocator)->interface.realloc))((allocator)->impl, (ptr), (size)) : realloc((ptr), (size)))
+#define afree(allocator, ptr, size) \
+  ((allocator) ? (*((allocator)->interface.free))((allocator)->impl, (ptr), (size)) : free(ptr))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_AMALLOC_H_*/

include/util/tlockfree.h

@@ -70,7 +70,7 @@ typedef void (*_ref_fn_t)(const void* pObj);
 
 
 // single writer multiple reader lock
-typedef int32_t SRWLatch;
+typedef volatile int32_t SRWLatch;
 
 void taosInitRWLatch(SRWLatch *pLatch);
 void taosWLockLatch(SRWLatch *pLatch);

source/server/vnode/inc/vnodeCommit.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_COMMIT_H_
+#define _TD_VNODE_COMMIT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int vnodeAsyncCommit(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_COMMIT_H_*/

source/server/vnode/inc/vnodeInt.h

@@ -16,14 +16,26 @@
 #ifndef _TD_VNODE_INT_H_
 #define _TD_VNODE_INT_H_
 
+
+#include "os.h"
+#include "amalloc.h"
+#include "meta.h"
+#include "taosmsg.h"
+#include "tq.h"
+#include "trpc.h"
+#include "tsdb.h"
+#include "vnode.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "os.h"
+typedef struct SVnode {
-#include "taosmsg.h"
+  SMeta *        pMeta;
-#include "trpc.h"
+  STsdb *        pTsdb;
-#include "vnode.h"
+  STQ *          pTQ;
+  SMemAllocator *allocator;
+} SVnode;
 
 #ifdef __cplusplus
 }

source/server/vnode/inc/vnodeMemAllocator.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_MEM_ALLOCATOR_H_
+#define _TD_VNODE_MEM_ALLOCATOR_H_
+
+#include "amalloc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/

source/server/vnode/inc/vnodeWrite.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_WRITE_H_
+#define _TD_VNODE_WRITE_H_
+
+int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp);
+
+#endif /*_TD_VNODE_WRITE_H_*/

source/server/vnode/meta/CMakeLists.txt

@@ -5,3 +5,7 @@ target_include_directories(
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
+target_link_libraries(
+    meta
+    PUBLIC common
+)

source/server/vnode/meta/inc/metaInt.h

@@ -20,6 +20,10 @@
 extern "C" {
 #endif
 
+struct {
+  tkv_db_t db;
+} SMeta;
+
 #ifdef __cplusplus
 }
 #endif

source/server/vnode/meta/src/meta.c

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) {
+  // TODO
+  return 0;
+}
+
+int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) {
+  // TODO
+  return 0;
+}
+
+int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) {
+  // TODO
+  return 0;
+}
+
+int metaCommit(SMeta *pMeta) {
+  // TODO
+  return 0;
+}

source/server/vnode/src/vnodeCommit.c

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeInt.h"
+
+static int vnodeStartCommit(SVnode *pVnode);
+static int vnodeEndCommit(SVnode *pVnode);
+
+int vnodeAsyncCommit(SVnode *pVnode) {
+  if (vnodeStartCommit(pVnode) < 0) {
+    // TODO
+  }
+
+  if (tqCommit(pVnode->pTQ) < 0) {
+    // TODO
+  }
+
+  if (metaCommit(pVnode->pMeta) < 0) {
+    // TODO
+  }
+
+  if (tsdbCommit(pVnode->pTsdb) < 0) {
+    // TODO
+  }
+
+  if (vnodeEndCommit(pVnode) < 0) {
+    // TODO
+  }
+
+  return 0;
+}
+
+static int vnodeStartCommit(SVnode *pVnode) {
+  // TODO
+  return 0;
+}
+
+static int vnodeEndCommit(SVnode *pVnode) {
+  // TODO
+  return 0;
+}

source/server/vnode/src/vnodeMemAllocator.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/server/vnode/src/vnodeWrite.c

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeInt.h"
+
+int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) {
+  // TODO: Check inputs
+
+#if 0
+  void *pMem = NULL;
+  if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) {
+    // No more memory to allocate, schedule an async commit
+    // and continue
+    vnodeAsyncCommit(pVnode);
+
+    // Reset allocator and allocat more
+    vnodeResetAllocator(pVnode);
+    pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq));
+    if (pMem == NULL) {
+      // TODO: handle the error
+    }
+  }
+
+  // TODO: if SSubmitReq is compressed or encoded, we need to decode the request
+  memcpy(pMem, pReq, REQ_SIZE(pReq));
+
+  if (tqPushMsg((SSubmitReq *)pReq) < 0) {
+    // TODO: handle error
+  }
+
+  SSubmitReqReader reader;
+  taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem);
+
+  if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) {
+    // TODO: handler error
+  }
+#endif
+
+  return 0;
+}
+
+int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) {
+  // TODO
+  return 0;
+}

source/server/vnode/tsdb/CMakeLists.txt

@@ -5,3 +5,8 @@ target_include_directories(
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tsdb"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
+target_include_directories(
+    tsdb
+    PRIVATE os
+    PRIVATE common
+)