Merge branch '2.0' into refact/slguan

src/CMakeLists.txt

@@ -4,11 +4,11 @@ PROJECT(TDengine)
 ADD_SUBDIRECTORY(os)
 ADD_SUBDIRECTORY(util)
 ADD_SUBDIRECTORY(rpc)
-ADD_SUBDIRECTORY(client)
+# ADD_SUBDIRECTORY(client)
-ADD_SUBDIRECTORY(kit)
+# ADD_SUBDIRECTORY(kit)
-ADD_SUBDIRECTORY(plugins)
+# ADD_SUBDIRECTORY(plugins)
-ADD_SUBDIRECTORY(sdb)
+# ADD_SUBDIRECTORY(sdb)
-ADD_SUBDIRECTORY(mnode)
+# ADD_SUBDIRECTORY(mnode)
-ADD_SUBDIRECTORY(dnode)
+ADD_SUBDIRECTORY(vnode)
-#ADD_SUBDIRECTORY(vnode)
+# ADD_SUBDIRECTORY(dnode)
 #ADD_SUBDIRECTORY(connector/jdbc)

src/dnode/CMakeLists.txt

@@ -10,7 +10,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http)
+  TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb)
 
   #IF (TD_CLUSTER)
   #  TARGET_LINK_LIBRARIES(taosd dcluster)

src/util/inc/tarray.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TAOSARRAY_H
+#define TDENGINE_TAOSARRAY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#define TARRAY_MIN_SIZE 8
+#define TARRAY_GET_ELEM(array, index) ((array)->pData + (index) * (array)->elemSize)
+
+typedef struct SArray {
+  size_t size;
+  size_t capacity;
+  size_t elemSize;
+
+  void* pData;
+} SArray;
+
+/**
+ *
+ * @param size
+ * @param elemSize
+ * @return
+ */
+void* taosArrayInit(size_t size, size_t elemSize);
+
+/**
+ *
+ * @param pArray
+ * @param pData
+ * @return
+ */
+void* taosArrayPush(SArray* pArray, void* pData);
+
+/**
+ *
+ * @param pArray
+ */
+void taosArrayPop(SArray* pArray);
+
+/**
+ *
+ * @param pArray
+ * @param index
+ * @return
+ */
+void* taosArrayGet(SArray* pArray, size_t index);
+
+/**
+ *
+ * @param pArray
+ * @return
+ */
+size_t taosArrayGetSize(SArray* pArray);
+
+/**
+ *
+ * @param pArray
+ * @param index
+ * @param pData
+ */
+void taosArrayInsert(SArray* pArray, int32_t index, void* pData);
+
+/**
+ *
+ * @param pArray
+ */
+void taosArrayDestory(SArray* pArray);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TAOSARRAY_H

src/util/inc/tskiplist.h

@@ -20,59 +20,62 @@
 extern "C" {
 #endif
 
-#define MAX_SKIP_LIST_LEVEL 20
-
-#include <pthread.h>
-#include <stdint.h>
-#include <stdlib.h>
-
 #include "os.h"
 #include "ttypes.h"
+#include "tarray.h"
 
 /*
  * key of each node
  * todo move to as the global structure in all search codes...
  */
+#define MAX_SKIP_LIST_LEVEL 15
+#define SKIP_LIST_RECORD_PERFORMANCE 0
 
-const static size_t SKIP_LIST_STR_KEY_LENGTH_THRESHOLD = 15;
+typedef char *SSkipListKey;
-typedef tVariant    tSkipListKey;
+typedef char *(*__sl_key_fn_t)(const void *);
 
-typedef enum tSkipListPointQueryType {
+/**
-  INCLUDE_POINT_QUERY,
+ * the format of skip list node is as follows:
-  EXCLUDE_POINT_QUERY,
+ * +------------+-----------------------+------------------------+-----+------+
-} tSkipListPointQueryType;
+ * | node level | forward pointer array | backward pointer array | key | data |
+ * +------------+-----------------------+------------------------+-----+------+
+ * the skiplist node is located in a consecutive memory area, key will not be copy to skip list
+ */
+typedef struct SSkipListNode {
+  uint8_t level;
+} SSkipListNode;
 
-typedef struct tSkipListNode {
+#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
-  uint16_t nLevel;
-  char *   pData;
 
-  struct tSkipListNode **pForward;
+#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
-  struct tSkipListNode **pBackward;
+#define SL_GET_BACKWARD_POINTER(n, _l) \
+  ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
 
-  tSkipListKey key;
+#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level))
-} tSkipListNode;
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
+
+#define SL_GET_NODE_LEVEL(n) *(int32_t *)((n))
 
 /*
- * @version 0.2
+ * @version 0.3
  * @date   2017/11/12
- * the simple version of SkipList.
+ * the simple version of skip list.
+ *
  * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate
- * deterministic result. Later, we will remove the lock in SkipList to further
+ * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance.
- * enhance the performance. In this case, one should use the concurrent skip list (by
+ * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of
- * using michael-scott algorithm) instead of this simple version in a multi-thread
+ * this simple version in a multi-thread environment, to achieve higher performance of read/write operations.
- * environment, to achieve higher performance of read/write operations.
  *
  * Note: Duplicated primary key situation.
  * In case of duplicated primary key, two ways can be employed to handle this situation:
  * 1. add as normal insertion without special process.
- * 2. add an overflow pointer at each list node, all nodes with the same key will be added
+ * 2. add an overflow pointer at each list node, all nodes with the same key will be added in the overflow pointer.
- *    in the overflow pointer. In this case, the total steps of each search will be reduced significantly.
+ *    In this case, the total steps of each search will be reduced significantly.
  *    Currently, we implement the skip list in a line with the first means, maybe refactor it soon.
  *
  *    Memory consumption: the memory alignment causes many memory wasted. So, employ a memory
  *    pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs.
  *
- * 3. use the iterator pattern to refactor all routines to make it more clean
  */
 
 // state struct, record following information:
@@ -81,7 +84,7 @@ typedef struct tSkipListNode {
 // avg search rsp time, for latest 1000 queries
 // total memory size
 typedef struct tSkipListState {
-  // in bytes, sizeof(tSkipList)+sizeof(tSkipListNode)*tSkipList->nSize
+  // in bytes, sizeof(SSkipList)+sizeof(SSkipListNode)*SSkipList->nSize
   uint64_t nTotalMemSize;
   uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL];
   uint64_t queryCount;  // total query count
@@ -101,18 +104,28 @@ typedef struct tSkipListState {
   uint64_t nTotalElapsedTimeForInsert;
 } tSkipListState;
 
-typedef struct tSkipList {
+typedef struct SSkipListKeyInfo {
-  tSkipListNode pHead;
+  uint8_t dupKey : 2;  // if allow duplicated key in the skip list
-  uint64_t      nSize;
+  uint8_t type : 6;    // key type
-  uint16_t      nMaxLevel;
+  uint8_t len;         // maximum key length, used in case of string key
-  uint16_t      nLevel;
+} SSkipListKeyInfo;
-  uint16_t      keyType;
-  uint16_t      nMaxKeyLen;
 
-  __compar_fn_t    comparator;
+typedef struct SSkipList {
-  pthread_rwlock_t lock;   // will be removed soon
+  __compar_fn_t    comparFn;
+  __sl_key_fn_t    keyFn;
+  uint32_t         size;
+  uint8_t          maxLevel;
+  uint8_t          level;
+  SSkipListKeyInfo keyInfo;
+
+  pthread_rwlock_t *lock;
+  SSkipListNode *   pHead;
+
+#if SKIP_LIST_RECORD_PERFORMANCE
   tSkipListState state;  // skiplist state
-} tSkipList;
+#endif
+
+} SSkipList;
 
 /*
  * iterate the skiplist
@@ -121,48 +134,65 @@ typedef struct tSkipList {
  * TODO add the ref for skip list when one iterator is created
  */
 typedef struct SSkipListIterator {
-  tSkipList *    pSkipList;
+  SSkipList *    pSkipList;
-  tSkipListNode *cur;
+  SSkipListNode *cur;
   int64_t        num;
 } SSkipListIterator;
 
-/*
+/**
- * query condition structure to denote the range query
+ *
- * todo merge the point query cond with range query condition
+ * @param nMaxLevel   maximum skip list level
+ * @param keyType     type of key
+ * @param dupKey      allow the duplicated key in the skip list
+ * @return
  */
-typedef struct tSKipListQueryCond {
+SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
-  // when the upper bounding == lower bounding, it is a point query
+                           __sl_key_fn_t fn);
-  tSkipListKey lowerBnd;
-  tSkipListKey upperBnd;
-  int32_t      lowerBndRelOptr;  // relation operator to denote if lower bound is
-  int32_t      upperBndRelOptr;  // included or not
-} tSKipListQueryCond;
 
-tSkipList *tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen);
+/**
-
+ *
-void *tSkipListDestroy(tSkipList *pSkipList);
+ * @param pSkipList
-
+ * @return                NULL will always be returned
-// create skip list key
-tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength);
-
-// destroy skip list key
-void tSkipListDestroyKey(tSkipListKey *pKey);
-
-// put data into skiplist
-tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKey, int32_t insertIdenticalKey);
-
-/*
- * get only *one* node of which key is equalled to pKey, even there are more
- * than one nodes are of the same key
  */
-tSkipListNode *tSkipListGetOne(tSkipList *pSkipList, tSkipListKey *pKey);
+void *tSkipListDestroy(SSkipList *pSkipList);
 
-/*
+/**
- * get all data with the same keys
+ *
+ * @param pSkipList
+ * @param level
+ * @param headSize
  */
-int32_t tSkipListGets(tSkipList *pSkipList, tSkipListKey *pKey, tSkipListNode ***pRes);
+void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
 
-int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (*fp)(tSkipListNode *, void *),
+/**
+ * put the skip list node into the skip list.
+ * If failed, NULL will be returned, otherwise, the pNode will be returned.
+ *
+ * @param pSkipList
+ * @param pNode
+ * @return
+ */
+SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
+
+/**
+ * get only *one* node of which key is equalled to pKey, even there are more than one nodes are of the same key
+ *
+ * @param pSkipList
+ * @param pKey
+ * @param keyType
+ * @return
+ */
+SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType);
+
+/**
+ *
+ * @param pSkipList
+ * @param pRes
+ * @param fp
+ * @param param
+ * @return
+ */
+int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *),
                              void *param);
 
 /*
@@ -173,30 +203,16 @@ int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (
  * true: one node has been removed
  * false: no node has been removed
  */
-bool tSkipListRemove(tSkipList *pSkipList, tSkipListKey *pKey);
+bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey);
 
 /*
  * remove the specified node in parameters
  */
-void tSkipListRemoveNode(tSkipList *pSkipList, tSkipListNode *pNode);
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
 
-// for debug purpose only
+int32_t        tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter);
-void tSkipListPrint(tSkipList *pSkipList, int16_t nlevel);
-
-/*
- * range query & single point query function
- */
-int32_t tSkipListQuery(tSkipList *pSkipList, tSKipListQueryCond *pQueryCond, tSkipListNode ***pResult);
-
-/*
- * include/exclude point query
- */
-int32_t tSkipListPointQuery(tSkipList *pSkipList, tSkipListKey *pKey, int32_t numOfKey, tSkipListPointQueryType type,
-                            tSkipListNode ***pResult);
-
-int32_t tSkipListIteratorReset(tSkipList *pSkipList, SSkipListIterator *iter);
 bool           tSkipListIteratorNext(SSkipListIterator *iter);
-tSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter);
+SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter);
 
 #ifdef __cplusplus
 }

src/util/src/tarray.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tarray.h"
+
+void* taosArrayInit(size_t size, size_t elemSize) {
+  assert(elemSize > 0);
+
+  if (size < TARRAY_MIN_SIZE) {
+    size = TARRAY_MIN_SIZE;
+  }
+
+  SArray* pArray = calloc(1, sizeof(SArray));
+  if (pArray == NULL) {
+    return NULL;
+  }
+
+  pArray->pData = calloc(size, elemSize * size);
+  if (pArray->pData == NULL) {
+    free(pArray);
+    return NULL;
+  }
+
+  pArray->capacity = size;
+  pArray->elemSize = elemSize;
+  return pArray;
+}
+
+static void taosArrayResize(SArray* pArray) {
+  assert(pArray->size >= pArray->capacity);
+
+  size_t size = pArray->capacity;
+  size = (size << 1u);
+
+  void* tmp = realloc(pArray->pData, size * pArray->elemSize);
+  if (tmp == NULL) {
+    // todo
+  }
+
+  pArray->pData = tmp;
+  pArray->capacity = size;
+}
+
+void* taosArrayPush(SArray* pArray, void* pData) {
+  if (pArray == NULL || pData == NULL) {
+    return NULL;
+  }
+
+  if (pArray->size >= pArray->capacity) {
+    taosArrayResize(pArray);
+  }
+
+  void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
+  memcpy(dst, pData, pArray->elemSize);
+
+  pArray->size += 1;
+  return dst;
+}
+
+void taosArrayPop(SArray* pArray) {
+  if (pArray == NULL || pArray->size == 0) {
+    return;
+  }
+
+  pArray->size -= 1;
+}
+
+void* taosArrayGet(SArray* pArray, size_t index) {
+  assert(index < pArray->size);
+  return TARRAY_GET_ELEM(pArray, index);
+}
+
+size_t taosArrayGetSize(SArray* pArray) { return pArray->size; }
+
+void taosArrayInsert(SArray* pArray, int32_t index, void* pData) {
+  if (pArray == NULL || pData == NULL) {
+    return;
+  }
+
+  if (index >= pArray->size) {
+    taosArrayPush(pArray, pData);
+    return;
+  }
+
+  if (pArray->size >= pArray->capacity) {
+    taosArrayResize(pArray);
+  }
+
+  void* dst = TARRAY_GET_ELEM(pArray, index);
+
+  int32_t remain = pArray->size - index;
+  memmove(dst + pArray->elemSize, dst, pArray->elemSize * remain);
+  memcpy(dst, pData, pArray->elemSize);
+
+  pArray->size += 1;
+}
+
+void taosArrayDestory(SArray* pArray) {
+  if (pArray == NULL) {
+    return;
+  }
+
+  free(pArray->pData);
+  free(pArray);
+}

src/vnode/CMakeLists.txt

@@ -1,7 +1,5 @@
 cmake_minimum_required(VERSION 2.8)
 
-project(tsdb)
-
 add_subdirectory(common)
 
 add_subdirectory(tsdb)

src/vnode/common/inc/dataformat.h

@@ -1,3 +1,17 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 #if !defined(_TD_DATA_FORMAT_H_)
 #define _TD_DATA_FORMAT_H_
 
@@ -5,6 +19,9 @@
 
 #include "schema.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 // ----------------- Data row structure
 
 /* A data row, the format of it is like below:
@@ -50,6 +67,8 @@ typedef char * SDataCols;
 #define TD_DATAROW_LEN(pDataRow) (*(int32_t *)(pDataRow))
 #define TD_DATAROW_DATA(pDataRow) ((pDataRow) + sizeof(int32_t))
 
+SDataRow tdSDataRowDup(SDataRow rdata);
+
 // ---- operation on SDataRows
 #define TD_DATAROWS_LEN(pDataRows) (*(int32_t *)(pDataRows))
 #define TD_DATAROWS_ROWS(pDataRows) (*(int32_t *)(pDataRows + sizeof(int32_t)))
@@ -69,4 +88,8 @@ typedef char * SDataCols;
  */
 int32_t tdGetMaxDataRowSize(SSchema *pSchema);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif // _TD_DATA_FORMAT_H_

src/vnode/common/inc/key.h

@@ -0,0 +1,10 @@
+#if !defined(_TD_KEY_H_)
+#define _TD_KEY_H_
+
+typedef struct {
+
+} key;
+
+
+
+#endif // _TD_KEY_H_

src/vnode/common/inc/list.h

@@ -0,0 +1,20 @@
+#if !defined(_TD_LIST_H_)
+#define _TD_LIST_H_
+
+#include <stdint.h>
+
+typedef enum { TD_LIST_ORDERED, TD_LIST_UNORDERED } TLIST_TYPE;
+
+typedef int32_t (* comparefn(void *key1, void *key2));
+
+struct _list_type {
+    TLIST_TYPE type;
+};
+
+typedef struct _list_node {
+} SListNode;
+
+typedef struct _list {
+} SList;
+
+#endif  // _TD_LIST_H_

src/vnode/common/inc/schema.h

@@ -67,6 +67,7 @@ typedef char *SISchema;
 SISchema tdConvertSchemaToInline(SSchema *pSchema);
 int32_t tdGetColumnIdxByName(SSchema *pSchema, char *colName);
 int32_t tdGetColumnIdxById(SSchema *pSchema, int32_t colId);
+SSchema *tdDupSchema(SSchema *pSchema);
 
 // ---- TODO: operations to modify schema
 

src/vnode/common/inc/type.h

@@ -16,18 +16,7 @@ typedef enum {
   TD_DATATYPE_BINARY
 } td_datatype_t;
 
-const int32_t rowDataLen[] = {
+extern const int32_t rowDataLen[];
-    sizeof(int8_t),   // TD_DATATYPE_BOOL,
-    sizeof(int8_t),   // TD_DATATYPE_TINYINT,
-    sizeof(int16_t),  // TD_DATATYPE_SMALLINT,
-    sizeof(int32_t),  // TD_DATATYPE_INT,
-    sizeof(int64_t),  // TD_DATATYPE_BIGINT,
-    sizeof(float),    // TD_DATATYPE_FLOAT,
-    sizeof(double),   // TD_DATATYPE_DOUBLE,
-    sizeof(int32_t),  // TD_DATATYPE_VARCHAR,
-    sizeof(int32_t),  // TD_DATATYPE_NCHAR,
-    sizeof(int32_t)   // TD_DATATYPE_BINARY
-};
 
 // TODO: finish below
 #define TD_DATATYPE_BOOL_NULL

src/vnode/common/src/dataformat.c

@@ -5,15 +5,13 @@
 int32_t tdGetMaxDataRowSize(SSchema *pSchema) {
   int32_t nbytes = 0;
 
-    for (int32_t i = 0; i < TD_SCHEMA_NCOLS(pSchema); i++)
+  for (int32_t i = 0; i < TD_SCHEMA_NCOLS(pSchema); i++) {
-    {
     SColumn *     pCol = TD_SCHEMA_COLUMN_AT(pSchema, i);
     td_datatype_t type = TD_COLUMN_TYPE(pCol);
 
     nbytes += rowDataLen[type];
 
-        switch (type)
+    switch (type) {
-        {
       case TD_DATATYPE_VARCHAR:
         nbytes += TD_COLUMN_BYTES(pCol);
         break;
@@ -23,6 +21,8 @@ int32_t tdGetMaxDataRowSize(SSchema *pSchema) {
       case TD_DATATYPE_BINARY:
         nbytes += TD_COLUMN_BYTES(pCol);
         break;
+      default:
+        break;
     }
   }
 
@@ -30,3 +30,5 @@ int32_t tdGetMaxDataRowSize(SSchema *pSchema) {
 
   return nbytes;
 }
+
+SDataRow tdSDataRowDup(SDataRow rdata) { return NULL; }

src/vnode/common/src/schema.c

@@ -1,6 +1,18 @@
 #include <stdlib.h>
 
 #include "schema.h"
+const int32_t rowDataLen[] = {
+    sizeof(int8_t),   // TD_DATATYPE_BOOL,
+    sizeof(int8_t),   // TD_DATATYPE_TINYINT,
+    sizeof(int16_t),  // TD_DATATYPE_SMALLINT,
+    sizeof(int32_t),  // TD_DATATYPE_INT,
+    sizeof(int64_t),  // TD_DATATYPE_BIGINT,
+    sizeof(float),    // TD_DATATYPE_FLOAT,
+    sizeof(double),   // TD_DATATYPE_DOUBLE,
+    sizeof(int32_t),  // TD_DATATYPE_VARCHAR,
+    sizeof(int32_t),  // TD_DATATYPE_NCHAR,
+    sizeof(int32_t)   // TD_DATATYPE_BINARY
+};
 
 static size_t tdGetEstimatedISchemaLen(SSchema *pSchema) {
   size_t colNameLen = 0;
@@ -83,3 +95,7 @@ int32_t tdGetColumnIdxById(SSchema *pSchema, int32_t colId) {
   }
   return -1;
 }
+
+SSchema *tdDupSchema(SSchema *pSchema) {
+  return NULL;
+}

src/vnode/tests/tsdb/tsdbTests.cpp

@@ -4,9 +4,20 @@
 #include "tsdb.h"
 
 TEST(TsdbTest, createTsdbRepo) {
-    STSDBCfg *pCfg = (STSDBCfg *)malloc(sizeof(STSDBCfg));
+  STsdbCfg config;
 
-    free(pCfg);
+  config.precision = TSDB_PRECISION_MILLI;
+  config.tsdbId = 0;
+  config.maxTables = 100;
+  config.daysPerFile = 10;
+  config.keep = 3650;
+  config.minRowsPerFileBlock = 100;
+  config.maxRowsPerFileBlock = 4096;
+  config.maxCacheSize = 4 * 1024 * 1024;
 
-    ASSERT_EQ(1, 2/2);
+  tsdb_repo_t *pRepo = tsdbCreateRepo("/root/mnt/test/vnode0", &config, NULL);
+
+  ASSERT_NE(pRepo, nullptr);
+
+  tsdbCloseRepo(pRepo);
 }

src/vnode/tsdb/CMakeLists.txt

@@ -1,9 +1,10 @@
 aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
 
-message(STATUS "tsdb source files: ${SOURCE_LIST}")
-
 add_library(tsdb STATIC ${SOURCE_LIST})
+target_link_libraries(tsdb common tutil)
 
-target_link_libraries(tsdb common)
+target_include_directories(tsdb 
-
+  PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" 
-target_include_directories(tsdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc")
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/util/inc"
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/os/linux/inc"
+  )

src/vnode/tsdb/inc/tsdb.h

@@ -1,6 +1,17 @@
-/**************************************
+/*
- * FOR OUTSIDE USAGE
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- **************************************/
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 #if !defined(_TD_TSDB_H_)
 #define _TD_TSDB_H_
 
@@ -8,17 +19,24 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-// #include "cache.h"
+#include "dataformat.h"
 #include "schema.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TSDB_VERSION_MAJOR 1
 #define TSDB_VERSION_MINOR 0
 
 typedef void tsdb_repo_t;  // use void to hide implementation details from outside
-typedef int32_t table_id_t;   // table ID type in this repository
-typedef int16_t tsdb_id_t;    // TSDB repository ID
 
-// Submit message
+typedef struct {
+  int64_t uid;  // the unique table ID
+  int32_t tid;  // the table ID in the repository.
+} STableId;
+
+// Submit message for this TSDB
 typedef struct {
   int32_t numOfTables;
   int32_t compressed;
@@ -27,77 +45,48 @@ typedef struct {
 
 // Submit message for one table
 typedef struct {
-  table_id_t tableId;       // table ID to insert
+  STableId tableId;
   int32_t  sversion;   // data schema version
   int32_t  numOfRows;  // number of rows data
-  int64_t    uid;           // table UID to insert
   char     data[];
 } SSubmitBlock;
 
-// Retention policy.
+enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
-typedef struct {
-  // TODO: Need a more fancy description
-  int32_t keep1;
-  int32_t keep2;
-  int32_t keep3;
-} SRetentionPolicy;
-
-// Data sharding policy.
-typedef struct {
-  // TODO: Need a more fancy description
-  int32_t daysPerFile;
-} SDataShardPolicy;
-
-// Rows in file block policy
-typedef struct {
-  // TODO: Need a more fancy description
-  int32_t minRowsPerFileBlock;
-  int32_t maxRowsPerFileBlock;
-} SBlockRowsPolicy;
-
-// Applications trying to manipulate a table should provide both uid and tableId.
-// tableId is used for table quick access and uid for verification.
-typedef struct {
-  int64_t    uid;      // the unique table ID
-  table_id_t tableId;  // the table ID in the repository.
-} STableId;
 
 // the TSDB repository configuration
 typedef struct {
-  char *           rootDir;  // TSDB repository root directory, TODO: need to adjust here
+  int8_t  precision;
-  tsdb_id_t        tsdbId;
+  int32_t tsdbId;
   int32_t maxTables;            // maximum number of tables this repository can have
-  SDataShardPolicy dataShardPolicy;
+  int32_t daysPerFile;          // day per file sharding policy
-  SBlockRowsPolicy blockRowsPolicy;
+  int32_t minRowsPerFileBlock;  // minimum rows per file block
-  SRetentionPolicy retentionPlicy;  // retention configuration
+  int32_t maxRowsPerFileBlock;  // maximum rows per file block
-  void *           cachePool;       // the cache pool the repository to use
+  int32_t keep;                 // day of data to keep
-} STSDBCfg;
+  int64_t maxCacheSize;         // maximum cache size this TSDB can use
+} STsdbCfg;
 
 // the TSDB repository info
-typedef struct STSDBRepoInfo {
+typedef struct STsdbRepoInfo {
-  STSDBCfg tsdbCfg;
+  STsdbCfg tsdbCfg;
   int64_t  version;            // version of the repository
   int64_t  tsdbTotalDataSize;  // the original inserted data size
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
-} STSDBRepoInfo;
+} STsdbRepoInfo;
 
 // the meter configuration
 typedef struct {
-  char *     tableName;
+  STableId tableId;
-  int64_t    uid;      // uid given by upper layer
-  table_id_t tableId;  // table ID allocated from upper layer
 
-  char *stableName;  // if not NULL, the table is created from a super table, need to make sure the super
-                     // table exists in this TSDB.
   int64_t stableUid;
+  int64_t createdTime;
 
   int32_t  numOfCols;  // number of columns. For table form super table, not includes the tag schema
   SSchema *schema;     // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL
                        // If numOfCols < schema->numOfCols, it is a table created from super table
                        // assert(numOfCols <= schema->numOfCols);
 
-  char *tagValues;  // NULL if it is normal table
+  SDataRow tagValues;  // NULL if it is normal table
                        // otherwise, it contains the tag values.
 } STableCfg;
 
@@ -111,11 +100,12 @@ typedef struct {
 
 /**
  * Create a new TSDB repository
+ * @param rootDir the TSDB repository root directory
  * @param pCfg the TSDB repository configuration, upper layer to free the pointer
  *
  * @return a TSDB repository handle on success, NULL for failure and the error number is set
  */
-tsdb_repo_t *tsdbCreateRepo(STSDBCfg *pCfg);
+tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
 
 /**
  * Close and free all resources taken by the repository
@@ -149,7 +139,7 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo);
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbConfigRepo(tsdb_repo_t repo, STSDBCfg *pCfg);
+int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
 
 /**
  * Get the TSDB repository information, including some statistics
@@ -159,7 +149,7 @@ int32_t tsdbConfigRepo(tsdb_repo_t repo, STSDBCfg *pCfg);
  * @return a info struct handle on success, NULL for failure and the error number is set. The upper
  *         layers should free the info handle themselves or memory leak will occur
  */
-STSDBRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
+STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
 
 // -- For table manipulation
 
@@ -181,7 +171,7 @@ int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tid, int32_t *error);
+int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tid);
 
 /**
  * Get the information of a table in the repository
@@ -191,7 +181,7 @@ int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tid, int32_t *error);
  *
  * @return a table information handle for success, NULL for failure and the error number is set
  */
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid, int32_t *error);
+STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
 
 // -- FOR INSERT DATA
 /**
@@ -203,7 +193,7 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid, int32_t *error);
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(tsdb_repo_t *pRepo, STableId tid, char *pData, int32_t *error);
+int32_t tsdbInsertData(tsdb_repo_t *pRepo, STableId tid, char *pData);
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -251,7 +241,6 @@ typedef struct STableIDList {
 } STableIDList;
 
 typedef struct {
-
 } SFields;
 
 /**
@@ -275,7 +264,8 @@ tsdb_query_handle_t *tsdbQueryFromTableID(tsdb_repo_t *pRepo, STSDBQueryCond *pC
  * @param pTagFilterStr tag filter info
  * @return
  */
-tsdb_query_handle_t *tsdbQueryFromTagConds(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
+tsdb_query_handle_t *tsdbQueryFromTagConds(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, int16_t stableId,
+                                           const char *pTagFilterStr);
 
 /**
  *  Reset to the start(end) position of current query, from which the iterator starts.
@@ -349,4 +339,8 @@ STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);
  */
 STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // _TD_TSDB_H_

src/vnode/tsdb/inc/tsdbCache.h

@@ -1,3 +1,17 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 #if !defined(_TD_TSDBCACHE_H_)
 #define _TD_TSDBCACHE_H_
 
@@ -5,6 +19,10 @@
 
 // #include "cache.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16*1024*1024 /* 16M */
 
 typedef struct {
@@ -38,4 +56,8 @@ typedef struct STSDBCache {
 SCacheHandle *tsdbCreateCache(int32_t numOfBlocks);
 int32_t tsdbFreeCache(SCacheHandle *pHandle);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // _TD_TSDBCACHE_H_

src/vnode/tsdb/inc/tsdbFile.h

@@ -1,9 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 #if !defined(_TD_TSDB_FILE_H_)
 #define _TD_TSDB_FILE_H_
 
 #include <stdint.h>
 // #include "tstring.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef int32_t file_id_t;
 
 typedef enum {
@@ -13,12 +31,7 @@ typedef enum {
   TSDB_FILE_TYPE_META   // .meta file type
 } TSDB_FILE_TYPE;
 
-const char *tsdbFileSuffix[] = {
+extern const char *tsdbFileSuffix[];
-  ".head",  // TSDB_FILE_TYPE_HEAD
-  ".data",  // TSDB_FILE_TYPE_DATA
-  ".last",  // TSDB_FILE_TYPE_LAST
-  ".meta"   // TSDB_FILE_TYPE_META
-};
 
 typedef struct {
   int64_t fileSize;
@@ -36,6 +49,12 @@ typedef struct {
 //   int16_t numOfBlocks;
 // } SDataBlock;
 
+#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_META)
+
 char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // _TD_TSDB_FILE_H_

src/vnode/tsdb/inc/tsdbMeta.h

@@ -1,9 +1,28 @@
-/************************************
+/*
- * For internal usage
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- ************************************/
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#if !defined(_TSDB_META_H_)
+#define _TSDB_META_H_
 
 #include <pthread.h>
 
+#include "dataformat.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 // #include "taosdef.h"
 
 // Initially, there are 4 tables
@@ -16,15 +35,15 @@ typedef enum {
 } TSDB_TABLE_TYPE;
 
 typedef struct STable {
-  tsdb_id_t       tableId;
+  STableId        tableId;
-  int64_t         uid;
-  char *          tableName;
   TSDB_TABLE_TYPE type;
 
   int64_t createdTime;
 
-  // super table UID
+  // super table UID -1 for normal table
-  tsdb_id_t superTableId;
+  int32_t stableUid;
+
+  int32_t numOfCols;
 
   // Schema for this table
   // For TSDB_SUPER_TABLE, it is the schema including tags
@@ -35,7 +54,7 @@ typedef struct STable {
   // Tag value for this table
   // For TSDB_SUPER_TABLE and TSDB_NTABLE, it is NULL
   // For TSDB_STABLE, it is the tag value string
-  char *pTagVal;
+  SDataRow pTagVal;
 
   // Object content;
   // For TSDB_SUPER_TABLE, it is the index of tables created from it
@@ -46,23 +65,21 @@ typedef struct STable {
   } content;
 
   // A handle to deal with event
-  void *eventHandle;
+  void *eventHandler;
 
   // A handle to deal with stream
-  void *streamHandle;
+  void *streamHandler;
+
+  struct STable *next;
 
 } STable;
 
 typedef struct {
-  int32_t numOfTables;       // Number of tables not including TSDB_SUPER_TABLE (#TSDB_NTABLE + #TSDB_STABLE)
+  int32_t  maxTables;
-  int32_t numOfSuperTables;  // Number of super tables (#TSDB_SUPER_TABLE)
+  STable **tables;    // array of normal tables
-  // An array of tables (TSDB_NTABLE and TSDB_STABLE) in this TSDB repository
+  STable * stables;   // linked list of super tables
-  STable **pTables;
+  void *   tableMap;  // hash map of uid ==> STable *
-
+} STsdbMeta;
-  // A map of tableName->tableId
-  // TODO: May use hash table
-  void *pNameTableMap;
-} SMetaHandle;
 
 // ---- Operation on STable
 #define TSDB_TABLE_ID(pTable) ((pTable)->tableId)
@@ -84,10 +101,18 @@ SSchema *tsdbGetTableSchema(STable *pTable);
 #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
 
 // Create a new meta handle with configuration
-SMetaHandle * tsdbCreateMetaHandle (int32_t numOfTables);
+STsdbMeta *tsdbCreateMeta(int32_t maxTables);
-int32_t       tsdbFreeMetaHandle(SMetaHandle *pMetaHandle);
+int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 
 // Recover the meta handle from the file
-SMetaHandle * tsdbOpenMetaHandle(char *tsdbDir);
+STsdbMeta *tsdbOpenMetaHandle(char *tsdbDir);
 
-int32_t tsdbCreateTableImpl(SMetaHandle *pHandle, STableCfg *pCfg);
+int32_t tsdbCreateTableImpl(STsdbMeta *pHandle, STableCfg *pCfg);
+
+int32_t tsdbInsertDataImpl(STsdbMeta *pMeta, STableId tableId, char *pData);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // _TSDB_META_H_

src/vnode/tsdb/src/tsdb.c

@@ -1,204 +0,0 @@
-#include <pthread.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-// #include "taosdef.h"
-// #include "disk.h"
-#include "tsdbFile.h"
-#include "tsdb.h"
-#include "tsdbCache.h"
-#include "tsdbMeta.h"
-
-enum {
-  TSDB_REPO_STATE_ACTIVE,
-  TSDB_REPO_STATE_CLOSED,
-  TSDB_REPO_STATE_CONFIGURING
-};
-
-typedef struct _tsdb_repo {
-  // TSDB configuration
-  STSDBCfg *pCfg;
-
-  // The meter meta handle of this TSDB repository
-  SMetaHandle *pMetaHandle;
-
-  // The cache Handle
-  SCacheHandle *pCacheHandle;
-
-  // Disk tier handle for multi-tier storage
-  void *pDiskTier;
-
-  // File Store
-  void *pFileStore;
-
-  pthread_mutex_t tsdbMutex;
-
-  int8_t state;
-
-} STSDBRepo;
-
-#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
-#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
-#define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE)
-#define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED)
-
-// Check the correctness of the TSDB configuration
-static int32_t tsdbCheckCfg(STSDBCfg *pCfg) {
-  if (pCfg->rootDir == NULL) return -1;
-
-  if (access(pCfg->rootDir, F_OK|R_OK|W_OK) == -1) {
-    return -1;
-  }
-  // TODO
-  return 0;
-}
-
-static int32_t tsdbCreateFiles(STSDBRepo *pRepo) {
-  // TODO
-}
-
-static int32_t tsdbClearFiles(STSDBRepo *pRepo) {
-  // TODO
-}
-
-tsdb_repo_t *tsdbCreateRepo(STSDBCfg *pCfg) {
-
-  // Check the configuration
-  if (tsdbCheckCfg(pCfg) < 0) {
-    return NULL;
-  }
-
-  STSDBRepo *pRepo = (STSDBRepo *)malloc(sizeof(STSDBRepo));
-  if (pRepo == NULL) {
-    // TODO: deal with error
-    return NULL;
-  }
-
-  // TODO: Initailize pMetahandle
-  pRepo->pMetaHandle = tsdbCreateMetaHandle(pCfg->maxTables);
-  if (pRepo->pMetaHandle == NULL) {
-    // TODO: deal with error
-    free(pRepo);
-    return NULL;
-  }
-
-  // TODO: Initialize cache handle
-  pRepo->pCacheHandle = tsdbCreateCache(5);
-  if (pRepo->pCacheHandle == NULL) {
-    // TODO: free the object and return error
-    tsdbFreeMetaHandle(pRepo->pCacheHandle);
-    free(pRepo);
-    return NULL;
-  }
-
-  // Set configuration
-  pRepo->pCfg = pCfg;
-
-  // Create the Meta data file and data directory
-  if (tsdbCreateFiles(pRepo) < 0) {
-    // Failed to create and save files
-    tsdbFreeMetaHandle(pRepo->pCacheHandle);
-    free(pRepo);
-    return NULL;
-  }
-  
-  pRepo->state = TSDB_REPO_STATE_ACTIVE;
-
-  return (tsdb_repo_t *)pRepo;
-}
-
-int32_t tsdbDropRepo(tsdb_repo_t *repo) {
-  STSDBRepo *pRepo = (STSDBRepo *)repo;
-
-  pRepo->state = TSDB_REPO_STATE_CLOSED;
-
-  // Free the metaHandle
-  tsdbFreeMetaHandle(pRepo->pMetaHandle);
-
-  // Free the cache
-  tsdbFreeCache(pRepo->pCacheHandle);
-
-  tsdbClearFiles(pRepo);
-
-  return 0;
-}
-
-tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
-
-  if (access(tsdbDir, F_OK|W_OK|R_OK) < 0) {
-    return NULL;
-  }
-
-  STSDBRepo *pRepo = (STSDBRepo *)malloc(sizeof(STSDBRepo));
-  if (pRepo == NULL) {
-    return NULL;
-  }
-
-  // TODO: Initialize configuration from the file
-  pRepo->pMetaHandle = tsdbOpenMetaHandle();
-  if (pRepo->pMetaHandle == NULL) {
-    free(pRepo);
-    return NULL;
-  }
-
-  pRepo->pCacheHandle = tsdbCreateCache(5);
-  if (pRepo->pCacheHandle == NULL) {
-    // TODO: deal with error
-    return NULL;
-  }
-
-  pRepo->state = TSDB_REPO_STATE_ACTIVE;
-
-  return (tsdb_repo_t *)pRepo;
-}
-
-static int32_t tsdbFlushCache(STSDBRepo *pRepo) {
-  // TODO
-}
-
-int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
-  STSDBRepo *pRepo = (STSDBRepo *)repo;
-
-  tsdbFlushCache(pRepo);
-
-  pRepo->state = TSDB_REPO_STATE_CLOSED;
-
-  tsdbFreeMetaHandle(pRepo->pMetaHandle);
-
-  tsdbFreeCache(pRepo->pMetaHandle);
-
-  return 0;
-}
-
-int32_t tsdbConfigRepo(tsdb_repo_t *repo, STSDBCfg *pCfg) {
-  STSDBRepo *pRepo = (STSDBRepo *)repo;
-
-  pRepo->pCfg = pCfg;
-  // TODO
-  return 0;
-}
-
-STSDBRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
-  // TODO
-}
-
-int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
-  STSDBRepo *pRepo = (STSDBRepo *)repo;
-  return tsdbCreateTableImpl(pRepo->pMetaHandle, pCfg);
-}
-
-int32_t tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
-  // TODO
-}
-
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid, int32_t *error) {
-  // TODO
-}
-
-int32_t tsdbInsertData(tsdb_repo_t *pRepo, STableId tid, char *pData, int32_t *error) {
-  // TODO
-}

src/vnode/tsdb/src/tsdbCache.c

@@ -14,5 +14,4 @@ SCacheHandle *tsdbCreateCache(int32_t numOfBlocks) {
 
 }
 
-int32_t tsdbFreeCache(SCacheHandle *pHandle) {
+int32_t tsdbFreeCache(SCacheHandle *pHandle) { return 0; }
-}

src/vnode/tsdb/src/tsdbFileStore.c

@@ -1,6 +1,22 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
 #include "tsdbFile.h"
 
+const char *tsdbFileSuffix[] = {
+    ".head",  // TSDB_FILE_TYPE_HEAD
+    ".data",  // TSDB_FILE_TYPE_DATA
+    ".last",  // TSDB_FILE_TYPE_LAST
+    ".meta"   // TSDB_FILE_TYPE_META
+};
+
 char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type) {
-    // char *suffix = tsdbFileSuffix[type];
+  if (!IS_VALID_TSDB_FILE_TYPE(type)) return NULL;
-    // TODO 
+
+  char *fileName = (char *)malloc(strlen(dirName) + strlen(fname) + strlen(tsdbFileSuffix[type]) + 5);
+  if (fileName == NULL) return NULL;
+
+  sprintf(fileName, "%s/%s%s", dirName, fname, tsdbFileSuffix[type]);
+  return fileName;
 }

src/vnode/tsdb/src/tsdbMain.c

@@ -0,0 +1,294 @@
+#include <stdio.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+#include <dirent.h>
+
+// #include "taosdef.h"
+// #include "disk.h"
+#include "tsdb.h"
+#include "tsdbCache.h"
+#include "tsdbFile.h"
+#include "tsdbMeta.h"
+
+enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING };
+
+typedef struct _tsdb_repo {
+  char *rootDir;
+  // TSDB configuration
+  STsdbCfg config;
+
+  // The meter meta handle of this TSDB repository
+  STsdbMeta *tsdbMeta;
+
+  // The cache Handle
+  SCacheHandle *tsdbCache;
+
+  // Disk tier handle for multi-tier storage
+  void *diskTier;
+
+  // File Store
+  void *tsdbFiles;
+
+  pthread_mutex_t tsdbMutex;
+
+  // A limiter to monitor the resources used by tsdb
+  void *limiter;
+
+  int8_t state;
+
+} STsdbRepo;
+
+static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
+static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
+static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
+static int     tsdbOpenMetaFile(char *tsdbDir);
+static int     tsdbRecoverRepo(int fd, STsdbCfg *pCfg);
+
+#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
+#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
+#define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE)
+#define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED)
+
+tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
+
+  if (rootDir == NULL) return NULL;
+
+  if (access(rootDir, F_OK|R_OK|W_OK) == -1) return NULL;
+
+  if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) {
+    return NULL;
+  }
+
+  STsdbRepo *pRepo = (STsdbRepo *)malloc(sizeof(STsdbRepo));
+  if (pRepo == NULL) {
+    return NULL;
+  }
+
+  pRepo->rootDir = strdup(rootDir);
+  pRepo->config = *pCfg;
+  pRepo->limiter = limiter;
+
+  pRepo->tsdbMeta = tsdbCreateMeta(pCfg->maxTables);
+  if (pRepo->tsdbMeta == NULL) {
+    free(pRepo->rootDir);
+    free(pRepo);
+    return NULL;
+  }
+
+  pRepo->tsdbCache = tsdbCreateCache(5);
+  if (pRepo->tsdbCache == NULL) {
+    free(pRepo->rootDir);
+    tsdbFreeMeta(pRepo->tsdbMeta);
+    free(pRepo);
+    return NULL;
+  }
+
+  // Create the Meta data file and data directory
+  if (tsdbSetRepoEnv(pRepo) < 0) {
+    free(pRepo->rootDir);
+    tsdbFreeMeta(pRepo->tsdbMeta);
+    tsdbFreeCache(pRepo->tsdbCache);
+    free(pRepo);
+    return NULL;
+  }
+
+  pRepo->state = TSDB_REPO_STATE_ACTIVE;
+
+  return (tsdb_repo_t *)pRepo;
+}
+
+int32_t tsdbDropRepo(tsdb_repo_t *repo) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+  pRepo->state = TSDB_REPO_STATE_CLOSED;
+
+  // Free the metaHandle
+  tsdbFreeMeta(pRepo->tsdbMeta);
+
+  // Free the cache
+  tsdbFreeCache(pRepo->tsdbCache);
+
+  // Destroy the repository info
+  tsdbDestroyRepoEnv(pRepo);
+
+  free(pRepo->rootDir);
+  free(pRepo);
+
+  return 0;
+}
+
+tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
+  if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) {
+    return NULL;
+  }
+
+  STsdbRepo *pRepo = (STsdbRepo *)malloc(sizeof(STsdbRepo));
+  if (pRepo == NULL) {
+    return NULL;
+  }
+
+  int fd = tsdbOpenMetaFile(tsdbDir);
+  if (fd < 0) {
+    free(pRepo);
+    return NULL;
+  }
+
+  if (tsdbRecoverRepo(fd, &(pRepo->config)) < 0) {
+    close(fd);
+    free(pRepo);
+    return NULL;
+  }
+
+  pRepo->tsdbCache = tsdbCreateCache(5);
+  if (pRepo->tsdbCache == NULL) {
+    // TODO: deal with error
+    return NULL;
+  }
+
+  pRepo->rootDir = strdup(tsdbDir);
+  pRepo->state = TSDB_REPO_STATE_ACTIVE;
+
+  return (tsdb_repo_t *)pRepo;
+}
+
+static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
+  // TODO
+  return 0;
+}
+
+int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  if (pRepo == NULL) return 0;
+
+  pRepo->state = TSDB_REPO_STATE_CLOSED;
+
+  tsdbFlushCache(pRepo);
+
+  tsdbFreeMeta(pRepo->tsdbMeta);
+
+  tsdbFreeCache(pRepo->tsdbCache);
+
+  return 0;
+}
+
+int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+  pRepo->config = *pCfg;
+  // TODO
+  return 0;
+}
+
+STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
+  // TODO
+  return NULL;
+}
+
+int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
+}
+
+int32_t tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
+  // TODO
+  return 0;
+}
+
+int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tid) {
+  return 0;
+}
+
+STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid) {
+  // TODO
+  return NULL;
+}
+
+int32_t tsdbInsertData(tsdb_repo_t *repo, STableId tableId, char *pData) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+  tsdbInsertDataImpl(pRepo->tsdbMeta, tableId, pData);
+
+  return 0;
+}
+
+// Check the configuration and set default options
+static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
+  // TODO
+  return 0;
+}
+
+static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
+  char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META);
+
+  int fd = open(metaFname, O_WRONLY|O_CREAT);
+  if (fd < 0) {
+    return -1;
+  }
+
+  if (write(fd, (void *)(&(pRepo->config)), sizeof(STsdbCfg)) < 0) {
+    return -1;
+  }
+
+  // Create the data file
+  char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2);
+  if (dirName == NULL) {
+    return -1;
+  }
+
+  sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb");
+  if (mkdir(dirName, 0755) < 0) {
+    free(dirName);
+    return -1;
+  }
+
+  free(dirName);
+
+  return 0;
+}
+
+static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) {
+  char fname[128];
+  if (pRepo == NULL) return 0;
+  char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2);
+  if (dirName == NULL) {
+    return -1;
+  }
+
+  sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb");
+
+  DIR *dir = opendir(dirName);
+  if (dir == NULL) return -1;
+
+  struct dirent *dp;
+  while ((dp = readdir(dir)) != NULL) {
+    if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) continue;
+    sprintf(fname, "%s/%s", pRepo->rootDir, dp->d_name);
+    remove(fname);
+  }
+
+  closedir(dir);
+
+  rmdir(dirName);
+
+  char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META);
+  remove(metaFname);
+
+  return 0;
+}
+
+static int tsdbOpenMetaFile(char *tsdbDir) {
+  // TODO
+  return 0;
+}
+
+static int tsdbRecoverRepo(int fd, STsdbCfg *pCfg) {
+  // TODO: read tsdb configuration from file
+  // recover tsdb meta
+  return 0;
+}

src/vnode/tsdb/src/tsdbMeta.c

@@ -1,65 +1,182 @@
 #include <stdlib.h>
 
 // #include "taosdef.h"
+#include "hash.h"
+#include "tskiplist.h"
 #include "tsdb.h"
 #include "tsdbMeta.h"
 
-SMetaHandle *tsdbCreateMetaHandle(int32_t numOfTables) {
+#define TSDB_MIN_TABLES 10
-  SMetaHandle *pMetahandle = (SMetaHandle *)malloc(sizeof(SMetaHandle));
+#define TSDB_MAX_TABLES 100000
-  if (pMetahandle == NULL) {
+#define TSDB_DEFAULT_NSTABLES 10
+
+#define IS_VALID_MAX_TABLES(maxTables) (((maxTables) >= TSDB_MIN_TABLES) && ((maxTables) <= TSDB_MAX_TABLES))
+
+static int     tsdbFreeTable(STable *pTable);
+static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
+static STable *tsdbGetTableByUid(int64_t uid);
+static int     tsdbAddTable(STsdbMeta *pMeta, STable *pTable);
+static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
+static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
+
+STsdbMeta *tsdbCreateMeta(int32_t maxTables) {
+  if (!IS_VALID_MAX_TABLES(maxTables)) return NULL;
+
+  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
+  if (pMeta == NULL) {
     return NULL;
   }
 
-  pMetahandle->numOfTables = 0;
+  pMeta->maxTables = maxTables;
-  pMetahandle->numOfSuperTables = 0;
+  pMeta->stables = NULL;
-  pMetahandle->pTables = calloc(sizeof(STable *), numOfTables);
+  pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *));
-  if (pMetahandle->pTables == NULL) {
+  if (pMeta->tables == NULL) {
-    free(pMetahandle);
+    free(pMeta);
     return NULL;
   }
 
-  // TODO : initialize the map
+  pMeta->tableMap = taosInitHashTable(maxTables + maxTables / 10, taosGetDefaultHashFunction, false);
-  // pMetahandle->pNameTableMap = ;
+  if (pMeta->tableMap == NULL) {
-  if (pMetahandle->pNameTableMap == NULL) {
+    free(pMeta->tables);
-    free(pMetahandle->pTables);
+    free(pMeta);
-    free(pMetahandle);
     return NULL;
   }
 
-  return pMetahandle;
+  return pMeta;
 }
 
-int32_t tsdbFreeMetaHandle(SMetaHandle *pMetaHandle) {
+int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
-  // TODO
+  if (pMeta == NULL) return 0;
 
+  for (int i = 0; i < pMeta->maxTables; i++) {
+    if (pMeta->tables[i] != NULL) {
+      tsdbFreeTable(pMeta->tables[i]);
+    }
   }
 
-static int32_t tsdbCheckTableCfg(STableCfg *pCfg) { return 0; }
+  free(pMeta->tables);
 
-int32_t tsdbCreateTableImpl(SMetaHandle *pHandle, STableCfg *pCfg) {
+  STable *pTable = pMeta->stables;
+  while (pTable != NULL) {
+    STable *pTemp = pTable;
+    pTable = pTemp->next;
+    tsdbFreeTable(pTemp);
+  }
+
+  taosCleanUpHashTable(pMeta->tableMap);
+
+  free(pMeta);
+
+  return 0;
+}
+
+int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
   if (tsdbCheckTableCfg(pCfg) < 0) {
     return -1;
   }
 
-  // TODO:
+  STable *pSTable = NULL;
+
+  if (pCfg->stableUid > 0) {  // to create a TSDB_STABLE
+    pSTable = tsdbGetTableByUid(pCfg->stableUid);
+    if (pSTable == NULL) {  // super table not exists, try to create it
+      pSTable = (STable *)calloc(1, sizeof(STable));
+      if (pSTable == NULL) return -1;
+
+      pSTable->tableId.uid = pCfg->stableUid;
+      pSTable->tableId.tid = -1;
+      pSTable->type = TSDB_SUPER_TABLE;
+      pSTable->createdTime = pCfg->createdTime; // The created time is not required
+      pSTable->stableUid = -1;
+      pSTable->numOfCols = pCfg->numOfCols;
+      pSTable->pSchema = tdDupSchema(pCfg->schema);
+      // pSTable->content.pIndex = tSkipListCreate(5, 0, 10); // TODO: change here
+      tsdbAddTable(pMeta, pSTable);
+    } else {
+      if (pSTable->type != TSDB_SUPER_TABLE) return NULL;
+    }
+  }
+
   STable *pTable = (STable *)malloc(sizeof(STable));
   if (pTable == NULL) {
     return -1;
   }
 
-  pHandle->pTables[pCfg->tableId] = pTable;
+  pTable->tableId = pCfg->tableId;
+  pTable->createdTime = pCfg->createdTime;
+  if (1 /* */) { // TSDB_STABLE
+    pTable->type = TSDB_STABLE;
+    pTable->stableUid = pCfg->stableUid;
+    pTable->pTagVal = tdSDataRowDup(pCfg->tagValues);
+  } else { // TSDB_NTABLE
+    pTable->type = TSDB_NTABLE;
+    pTable->stableUid = -1;
+    pTable->pSchema = tdDupSchema(pCfg->schema);
+  }
+  // pTable->content.pData = tSkipListCreate(5, 0, 10); // TODO: change here
+
+  tsdbAddTable(pMeta, pTable);
 
-  // TODO: add name to it
   return 0;
 }
 
-SMetaHandle * tsdbOpenMetaHandle(char *tsdbDir) {
+STsdbMeta *tsdbOpenMetaHandle(char *tsdbDir) {
   // Open meta file for reading
 
-  SMetaHandle *pHandle = (SMetaHandle *)malloc(sizeof(SMetaHandle));
+  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
-  if (pHandle == NULL) {
+  if (pMeta == NULL) {
     return NULL;
   }
 
-  return pHandle;
+  return pMeta;
+}
+
+int32_t tsdbInsertDataImpl(STsdbMeta *pMeta, STableId tableId, char *pData) {
+  STable *pTable = pMeta->tables[tableId.tid];
+  if (pTable == NULL) {
+    // TODO: deal with the error here
+    return 0;
+  }
+
+  if (pTable->tableId.uid != tableId.uid) {
+    // TODO: deal with the error here
+    return 0;
+  }
+
+  return 0;
+}
+
+static int tsdbFreeTable(STable *pTable) { return 0; }
+
+static int32_t tsdbCheckTableCfg(STableCfg *pCfg) { return 0; }
+
+static STable *tsdbGetTableByUid(int64_t uid) { return NULL; }
+
+static int tsdbAddTable(STsdbMeta *pMeta, STable *pTable) {
+  if (pTable->type == TSDB_SUPER_TABLE) {
+    if (pMeta->stables == NULL) {
+      pMeta->stables = pTable;
+      pTable->next = NULL;
+    } else {
+      STable *pTemp = pMeta->stables;
+      pMeta->stables = pTable;
+      pTable->next = pTemp;
+    }
+  } else {
+    pMeta->tables[pTable->tableId.tid] = pTable;
+    if (pTable->type == TSDB_STABLE) {
+      tsdbAddTableIntoIndex(pMeta, pTable);
+    }
+  }
+
+  return tsdbAddTableIntoMap(pMeta, pTable);
+}
+
+static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) {
+  // TODO: add the table to the map
+  return 0;
+}
+static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) {
+  // TODO
+  return 0;
 }