From d352819f278a030d4b6e63de8c89c240a5294e33 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Wed, 11 Mar 2020 12:26:11 +0800
Subject: [PATCH 01/17] refactor code

---
 src/vnode/tsdb/inc/tsdb.h          | 40 +++++++-------
 src/vnode/tsdb/src/tsdbMain.c      | 20 +++++--
 src/vnode/tsdb/tests/tsdbTests.cpp | 86 +++++++++++++++---------------
 3 files changed, 78 insertions(+), 68 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index d5493fdee0..781ec973ed 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -32,6 +32,24 @@ extern "C" {
 
 typedef void tsdb_repo_t;  // use void to hide implementation details from outside
 
+// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
+typedef struct {
+  int8_t  precision;
+  int32_t tsdbId;
+  int32_t maxTables;            // maximum number of tables this repository can have
+  int32_t daysPerFile;          // day per file sharding policy
+  int32_t minRowsPerFileBlock;  // minimum rows per file block
+  int32_t maxRowsPerFileBlock;  // maximum rows per file block
+  int32_t keep;                 // day of data to keep
+  int64_t maxCacheSize;         // maximum cache size this TSDB can use
+} STsdbCfg;
+
+void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
+STsdbCfg *tsdbCreateDefaultCfg();
+void      tsdbFreeCfg(STsdbCfg *pCfg);
+
+// --------- TSDB REPOSITORY DEFINITION
+
 typedef struct {
   int64_t uid;  // the unique table ID
   int32_t tid;  // the table ID in the repository.
@@ -55,18 +73,6 @@ typedef struct {
 
 enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
 
-// the TSDB repository configuration
-typedef struct {
-  int8_t  precision;
-  int32_t tsdbId;
-  int32_t maxTables;            // maximum number of tables this repository can have
-  int32_t daysPerFile;          // day per file sharding policy
-  int32_t minRowsPerFileBlock;  // minimum rows per file block
-  int32_t maxRowsPerFileBlock;  // maximum rows per file block
-  int32_t keep;                 // day of data to keep
-  int64_t maxCacheSize;         // maximum cache size this TSDB can use
-} STsdbCfg;
-
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
   STsdbCfg tsdbCfg;
@@ -100,16 +106,6 @@ typedef struct {
   int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;
 
-/**
- * Create a configuration for TSDB default
- * @return a pointer to a configuration. the configuration must call tsdbFreeCfg to free memory after usage
- */
-STsdbCfg *tsdbCreateDefaultCfg();
-
-/**
- * Free
- */
-void tsdbFreeCfg(STsdbCfg *pCfg);
 
 /**
  * Create a new TSDB repository
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 935bf6281c..e7feeeaf2d 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -82,9 +82,11 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlock *pBlock);
 #define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE)
 #define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED)
 
-STsdbCfg *tsdbCreateDefaultCfg() {
-  STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg));
-  if (pCfg == NULL) return NULL;
+/**
+ * Set the default TSDB configuration
+ */
+void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
+  if (pCfg == NULL) return;
 
   pCfg->precision = -1;
   pCfg->tsdbId = 0;
@@ -94,6 +96,18 @@ STsdbCfg *tsdbCreateDefaultCfg() {
   pCfg->maxRowsPerFileBlock = -1;
   pCfg->keep = -1;
   pCfg->maxCacheSize = -1;
+}
+
+/**
+ * Create a configuration for TSDB default
+ * @return a pointer to a configuration. the configuration object 
+ *         must call tsdbFreeCfg to free memory after usage
+ */
+STsdbCfg *tsdbCreateDefaultCfg() {
+  STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg));
+  if (pCfg == NULL) return NULL;
+
+  tsdbSetDefaultCfg(pCfg);
 
   return pCfg;
 }
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 737deee3c5..366508c01a 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -5,52 +5,30 @@
 #include "dataformat.h"
 #include "tsdbMeta.h"
 
-TEST(TsdbTest, createTable) {
-  STsdbMeta *pMeta = tsdbCreateMeta(100);
-  ASSERT_NE(pMeta, nullptr);
-
-  STableCfg config;
-  config.tableId.tid = 0;
-  config.tableId.uid = 98868728187539L;
-  config.numOfCols = 5;
-  config.schema = tdNewSchema(config.numOfCols);
-  for (int i = 0; i < schemaNCols(config.schema); i++) {
-    STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
-    tdColCpy(schemaColAt(config.schema, i), pCol);
-    tdFreeCol(pCol);
-  }
-  config.tagValues = nullptr;
-
-  tsdbCreateTableImpl(pMeta, &config);
-
-  STable *pTable = tsdbGetTableByUid(pMeta, config.tableId.uid);
-  ASSERT_NE(pTable, nullptr);
-}
-
 TEST(TsdbTest, createRepo) {
-  STsdbCfg *pCfg = tsdbCreateDefaultCfg();
+  STsdbCfg config;
 
   // Create a tsdb repository
-  tsdb_repo_t *pRepo = tsdbCreateRepo("/root/mnt/test/vnode0", pCfg, NULL);
+  tsdbSetDefaultCfg(&config);
+  tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
   ASSERT_NE(pRepo, nullptr);
-  tsdbFreeCfg(pCfg);
 
-  // create a normal table in this repository
-  STableCfg config;
-  config.tableId.tid = 0;
-  config.tableId.uid = 98868728187539L;
-  config.numOfCols = 5;
-  config.schema = tdNewSchema(config.numOfCols);
-  STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
-  tdColCpy(schemaColAt(config.schema, 0), pCol);
-  tdFreeCol(pCol);
-  for (int i = 1; i < schemaNCols(config.schema); i++) {
-    pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
-    tdColCpy(schemaColAt(config.schema, i), pCol);
-    tdFreeCol(pCol);
-  }
+  // // create a normal table in this repository
+  // STableCfg config;
+  // config.tableId.tid = 0;
+  // config.tableId.uid = 98868728187539L;
+  // config.numOfCols = 5;
+  // config.schema = tdNewSchema(config.numOfCols);
+  // STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
+  // tdColCpy(schemaColAt(config.schema, 0), pCol);
+  // tdFreeCol(pCol);
+  // for (int i = 1; i < schemaNCols(config.schema); i++) {
+  //   pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
+  //   tdColCpy(schemaColAt(config.schema, i), pCol);
+  //   tdFreeCol(pCol);
+  // }
 
-  tsdbCreateTable(pRepo, &config);
+  // tsdbCreateTable(pRepo, &config);
   // Write some data
 
   // int32_t size = sizeof(SSubmitMsg) + sizeof(SSubmitBlock) + tdMaxRowDataBytes(config.schema) * 10 + sizeof(int32_t);
@@ -89,6 +67,28 @@ TEST(TsdbTest, createRepo) {
 
   // tdFreeDataRow(row);
 
-  tdFreeSchema(config.schema);
-  tsdbDropRepo(pRepo);
-}
\ No newline at end of file
+  // tdFreeSchema(config.schema);
+  // tsdbDropRepo(pRepo);
+}
+
+TEST(TsdbTest, DISABLED_createTable) {
+  STsdbMeta *pMeta = tsdbCreateMeta(100);
+  ASSERT_NE(pMeta, nullptr);
+
+  STableCfg config;
+  config.tableId.tid = 0;
+  config.tableId.uid = 98868728187539L;
+  config.numOfCols = 5;
+  config.schema = tdNewSchema(config.numOfCols);
+  for (int i = 0; i < schemaNCols(config.schema); i++) {
+    STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
+    tdColCpy(schemaColAt(config.schema, i), pCol);
+    tdFreeCol(pCol);
+  }
+  config.tagValues = nullptr;
+
+  tsdbCreateTableImpl(pMeta, &config);
+
+  STable *pTable = tsdbGetTableByUid(pMeta, config.tableId.uid);
+  ASSERT_NE(pTable, nullptr);
+}

From d022f5e83fff9ffd0b70c5cb49a771330d5dd935 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Wed, 11 Mar 2020 15:02:39 +0800
Subject: [PATCH 02/17] refactor

---
 src/vnode/tsdb/inc/tsdb.h          |  69 +++--------------
 src/vnode/tsdb/inc/tsdbCache.h     |   2 +-
 src/vnode/tsdb/inc/tsdbMeta.h      |   2 +-
 src/vnode/tsdb/src/tsdbCache.c     |   2 +-
 src/vnode/tsdb/src/tsdbMain.c      | 119 ++++++++++++++++++++++++-----
 src/vnode/tsdb/src/tsdbMeta.c      |   2 +-
 src/vnode/tsdb/tests/tsdbTests.cpp |   2 +-
 7 files changed, 118 insertions(+), 80 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index 781ec973ed..cf9c4502dd 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -30,9 +30,9 @@ extern "C" {
 #define TSDB_VERSION_MAJOR 1
 #define TSDB_VERSION_MINOR 0
 
-typedef void tsdb_repo_t;  // use void to hide implementation details from outside
-
 // --------- TSDB REPOSITORY CONFIGURATION DEFINITION
+enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
+
 typedef struct {
   int8_t  precision;
   int32_t tsdbId;
@@ -49,7 +49,15 @@ STsdbCfg *tsdbCreateDefaultCfg();
 void      tsdbFreeCfg(STsdbCfg *pCfg);
 
 // --------- TSDB REPOSITORY DEFINITION
+typedef void tsdb_repo_t;  // use void to hide implementation details from outside
 
+tsdb_repo_t *  tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
+int32_t        tsdbDropRepo(tsdb_repo_t *repo);
+tsdb_repo_t *  tsdbOpenRepo(char *tsdbDir);
+int32_t        tsdbCloseRepo(tsdb_repo_t *repo);
+int32_t        tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
+
+// --------- TSDB TABLE DEFINITION
 typedef struct {
   int64_t uid;  // the unique table ID
   int32_t tid;  // the table ID in the repository.
@@ -71,7 +79,6 @@ typedef struct {
   char     data[];
 } SSubmitBlock;
 
-enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
@@ -82,6 +89,8 @@ typedef struct STsdbRepoInfo {
   // TODO: Other informations to add
 } STsdbRepoInfo;
 
+STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
+
 // the meter configuration
 typedef struct {
   STableId tableId;
@@ -106,60 +115,6 @@ typedef struct {
   int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;
 
-
-/**
- * Create a new TSDB repository
- * @param rootDir the TSDB repository root directory
- * @param pCfg the TSDB repository configuration, upper layer to free the pointer
- *
- * @return a TSDB repository handle on success, NULL for failure and the error number is set
- */
-tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
-
-/**
- * Close and free all resources taken by the repository
- * @param repo the TSDB repository handle. The interface will free the handle too, so upper
- *              layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbDropRepo(tsdb_repo_t *repo);
-
-/**
- * Open an existing TSDB storage repository
- * @param tsdbDir the existing TSDB root directory
- *
- * @return a TSDB repository handle on success, NULL for failure and the error number is set
- */
-tsdb_repo_t *tsdbOpenRepo(char *tsdbDir);
-
-/**
- * Close a TSDB repository. Only free memory resources, and keep the files.
- * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
- *              layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbCloseRepo(tsdb_repo_t *repo);
-
-/**
- * Change the configuration of a repository
- * @param pCfg the repository configuration, the upper layer should free the pointer
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
-
-/**
- * Get the TSDB repository information, including some statistics
- * @param pRepo the TSDB repository handle
- * @param error the error number to set when failure occurs
- *
- * @return a info struct handle on success, NULL for failure and the error number is set. The upper
- *         layers should free the info handle themselves or memory leak will occur
- */
-STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
-
 // -- For table manipulation
 
 /**
diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h
index 8a78a6b19e..1821505eae 100644
--- a/src/vnode/tsdb/inc/tsdbCache.h
+++ b/src/vnode/tsdb/inc/tsdbCache.h
@@ -53,7 +53,7 @@ typedef struct STSDBCache {
 #define TSDB_NEXT_CACHE_BLOCK(pBlock) ((pBlock)->next)
 #define TSDB_PREV_CACHE_BLOCK(pBlock) ((pBlock)->prev)
 
-STsdbCache *tsdbCreateCache(int32_t numOfBlocks);
+STsdbCache *tsdbInitCache(int64_t maxSize);
 int32_t     tsdbFreeCache(STsdbCache *pCache);
 void *      tsdbAllocFromCache(STsdbCache *pCache, int64_t bytes);
 
diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h
index efab26e1db..73bafb112f 100644
--- a/src/vnode/tsdb/inc/tsdbMeta.h
+++ b/src/vnode/tsdb/inc/tsdbMeta.h
@@ -106,7 +106,7 @@ STSchema *tsdbGetTableSchema(STable *pTable);
 #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
 
 // Create a new meta handle with configuration
-STsdbMeta *tsdbCreateMeta(int32_t maxTables);
+STsdbMeta *tsdbInitMeta(int32_t maxTables);
 int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 
 // Recover the meta handle from the file
diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c
index dacb360253..165c561b5d 100644
--- a/src/vnode/tsdb/src/tsdbCache.c
+++ b/src/vnode/tsdb/src/tsdbCache.c
@@ -16,7 +16,7 @@
 
 #include "tsdbCache.h"
 
-STsdbCache *tsdbCreateCache(int32_t numOfBlocks) {
+STsdbCache *tsdbInitCache(int64_t maxSize) {
   STsdbCache *pCacheHandle = (STsdbCache *)malloc(sizeof(STsdbCache));
   if (pCacheHandle == NULL) {
     // TODO : deal with the error
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index e7feeeaf2d..5db2545c53 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -42,6 +42,9 @@
 #define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024)       // 4M
 #define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024)    // 1G
 
+#define TSDB_CFG_FILE_NAME "CONFIG"
+#define TSDB_DATA_DIR_NAME "data"
+
 enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING };
 
 typedef struct _tsdb_repo {
@@ -116,7 +119,15 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
   if (pCfg != NULL) free(pCfg);
 }
 
-tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
+/**
+ * Create a new TSDB repository
+ * @param rootDir the TSDB repository root directory
+ * @param pCfg the TSDB repository configuration, upper layer need to free the pointer
+ * @param limiter the limitation tracker will implement in the future, make it void now
+ *
+ * @return a TSDB repository handle on success, NULL for failure
+ */
+tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) {
   if (rootDir == NULL) return NULL;
 
   if (access(rootDir, F_OK | R_OK | W_OK) == -1) return NULL;
@@ -134,22 +145,26 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
   pRepo->config = *pCfg;
   pRepo->limiter = limiter;
 
-  pRepo->tsdbMeta = tsdbCreateMeta(pCfg->maxTables);
-  if (pRepo->tsdbMeta == NULL) {
+  // Initialize meta
+  STsdbMeta *pMeta = tsdbInitMeta(pCfg->maxTables);
+  if (pMeta == NULL) {
     free(pRepo->rootDir);
     free(pRepo);
     return NULL;
   }
+  pRepo->tsdbMeta = pMeta;
 
-  pRepo->tsdbCache = tsdbCreateCache(5);
-  if (pRepo->tsdbCache == NULL) {
+  // Initialize cache
+  STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize);
+  if (pCache == NULL) {
     free(pRepo->rootDir);
     tsdbFreeMeta(pRepo->tsdbMeta);
     free(pRepo);
     return NULL;
   }
+  pRepo->tsdbCache = pCache;
 
-  // Create the Meta data file and data directory
+  // Create the environment files and directories
   if (tsdbSetRepoEnv(pRepo) < 0) {
     free(pRepo->rootDir);
     tsdbFreeMeta(pRepo->tsdbMeta);
@@ -163,6 +178,13 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
   return (tsdb_repo_t *)pRepo;
 }
 
+/**
+ * Close and free all resources taken by the repository
+ * @param repo the TSDB repository handle. The interface will free the handle too, so upper
+ *              layer do NOT need to free the repo handle again.
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbDropRepo(tsdb_repo_t *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
@@ -183,6 +205,12 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) {
   return 0;
 }
 
+/**
+ * Open an existing TSDB storage repository
+ * @param tsdbDir the existing TSDB root directory
+ *
+ * @return a TSDB repository handle on success, NULL for failure and the error number is set
+ */
 tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
   if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) {
     return NULL;
@@ -205,7 +233,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
     return NULL;
   }
 
-  pRepo->tsdbCache = tsdbCreateCache(5);
+  pRepo->tsdbCache = tsdbInitCache(5);
   if (pRepo->tsdbCache == NULL) {
     // TODO: deal with error
     return NULL;
@@ -222,6 +250,13 @@ static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
   return 0;
 }
 
+/**
+ * Close a TSDB repository. Only free memory resources, and keep the files.
+ * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
+ *              layer do NOT need to free the repo handle again.
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   if (pRepo == NULL) return 0;
@@ -237,6 +272,12 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
   return 0;
 }
 
+/**
+ * Change the configuration of a repository
+ * @param pCfg the repository configuration, the upper layer should free the pointer
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
@@ -245,6 +286,14 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
   return 0;
 }
 
+/**
+ * Get the TSDB repository information, including some statistics
+ * @param pRepo the TSDB repository handle
+ * @param error the error number to set when failure occurs
+ *
+ * @return a info struct handle on success, NULL for failure and the error number is set. The upper
+ *         layers should free the info handle themselves or memory leak will occur
+ */
 STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
   // TODO
   return NULL;
@@ -299,7 +348,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   // Check tsdbId
   if (pCfg->tsdbId < 0) return -1;
 
-  // Check MaxTables
+  // Check maxTables
   if (pCfg->maxTables == -1) {
     pCfg->maxTables = TSDB_DEFAULT_TABLES;
   } else {
@@ -347,10 +396,18 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   return 0;
 }
 
-static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
-  char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META);
+static int32_t tsdbGetCfgFname(STsdbRepo *pRepo, char *fname) {
+  if (pRepo == NULL) return -1;
+  sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_CFG_FILE_NAME);
+  return 0;
+}
 
-  int fd = open(metaFname, O_WRONLY | O_CREAT);
+static int32_t tsdbSaveConfig(STsdbRepo *pRepo) {
+  char fname[128] = "\0"; // TODO: get rid of the literal 128
+
+  if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+
+  int fd = open(fname, O_WRONLY | O_CREAT, 0755);
   if (fd < 0) {
     return -1;
   }
@@ -359,20 +416,46 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
     return -1;
   }
 
-  // Create the data file
-  char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2);
-  if (dirName == NULL) {
+  close(fd);
+  return 0;
+}
+
+static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) {
+  char fname[128] = "\0";
+
+  if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+
+  int fd = open(fname, O_RDONLY);
+  if (fd < 0) {
     return -1;
   }
 
-  sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb");
+  if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) {
+    close(fd);
+    return -1;
+  }
+
+  close(fd);
+
+  return 0;
+}
+
+static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname) {
+  if (pRepo == NULL || pRepo->rootDir == NULL) return -1;
+  sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_DATA_DIR_NAME);
+  return 0;
+}
+
+static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
+  if (tsdbSaveConfig(pRepo) < 0) return -1;
+
+  char dirName[128] = "\0";
+  if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
+
   if (mkdir(dirName, 0755) < 0) {
-    free(dirName);
     return -1;
   }
 
-  free(dirName);
-
   return 0;
 }
 
diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index 6c9cc2404a..552950a57c 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -17,7 +17,7 @@ static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
 
-STsdbMeta *tsdbCreateMeta(int32_t maxTables) {
+STsdbMeta *tsdbInitMeta(int32_t maxTables) {
   STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
   if (pMeta == NULL) {
     return NULL;
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 366508c01a..9e88e156a9 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -72,7 +72,7 @@ TEST(TsdbTest, createRepo) {
 }
 
 TEST(TsdbTest, DISABLED_createTable) {
-  STsdbMeta *pMeta = tsdbCreateMeta(100);
+  STsdbMeta *pMeta = tsdbInitMeta(100);
   ASSERT_NE(pMeta, nullptr);
 
   STableCfg config;

From 6c825e2d81be8d6cfd44800a0b4f480a49a06af0 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Wed, 11 Mar 2020 18:38:10 +0800
Subject: [PATCH 03/17] add more

---
 src/vnode/tsdb/inc/tsdbMetaFile.h |  47 +++++++
 src/vnode/tsdb/src/tsdbMeta.c     |   1 +
 src/vnode/tsdb/src/tsdbMetaFile.c | 225 ++++++++++++++++++++++++++++++
 3 files changed, 273 insertions(+)
 create mode 100644 src/vnode/tsdb/inc/tsdbMetaFile.h
 create mode 100644 src/vnode/tsdb/src/tsdbMetaFile.c

diff --git a/src/vnode/tsdb/inc/tsdbMetaFile.h b/src/vnode/tsdb/inc/tsdbMetaFile.h
new file mode 100644
index 0000000000..758cb57958
--- /dev/null
+++ b/src/vnode/tsdb/inc/tsdbMetaFile.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TSDB_META_FILE_
+#define _TSDB_META_FILE_
+
+#include <stdint.h>
+
+#include "tsdbMeta.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TSDB_META_FILE_NAME "META"
+
+typedef struct {
+  int     fd;       // File descriptor
+  int     nDel;     // number of deletions
+  int     nRecord;  // Number of records
+  int64_t size;     // Total file size
+  void *  map;      // Map from uid ==> position
+} SMetaFile;
+
+SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables);
+int32_t    tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+int32_t    tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid);
+int32_t    tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+void       tsdbCloseMetaFile(SMetaFile *mfh);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // _TSDB_META_FILE_
\ No newline at end of file
diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index 552950a57c..23442ed1db 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -9,6 +9,7 @@
 #include "tsdbCache.h"
 
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
+#define TSDB_META_FILE_NAME "META"
 
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c
new file mode 100644
index 0000000000..828d722389
--- /dev/null
+++ b/src/vnode/tsdb/src/tsdbMetaFile.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "hash.h"
+#include "tsdbMetaFile.h"
+
+#define TSDB_META_FILE_HEADER_SIZE 512
+#define TSDB_META_HASH_FRACTION 1.1
+
+typedef struct {
+  int32_t offset;
+  int32_t size;
+} SRecordInfo;
+
+static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
+static int32_t tsdbCheckMetaHeader(int fd);
+static int32_t tsdbWriteMetaHeader(int fd);
+static int     tsdbCreateMetaFile(char *fname);
+static int     tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh);
+
+SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables) {
+  // TODO
+  char fname[128] = "\0";
+  if (tsdbGetMetaFileName(rootDir, fname) < 0) return NULL;
+
+  SMetaFile *mfh = (SMetaFile *)calloc(1, sizeof(SMetaFile));
+  if (mfh == NULL) return NULL;
+
+  // OPEN MAP
+  mfh->map =
+      taosInitHashTable(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  if (mfh->map == NULL) {
+    free(mfh);
+    return NULL;
+  }
+
+  // OPEN FILE
+  if (access(fname, F_OK) < 0) {  // file not exists
+    mfh->fd = tsdbCreateMetaFile(fname);
+    if (mfh->fd < 0) {
+      taosCleanUpHashTable(mfh->map);
+      free(mfh);
+      return NULL;
+    }
+  } else {  // file exists, recover from file
+    if (tsdbRestoreFromMetaFile(fname, mfh) < 0) {
+      taosCleanUpHashTable(mfh->map);
+      free(mfh);
+      return NULL;
+    }
+  }
+
+  return mfh;
+}
+
+int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+  if (taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
+    return -1;
+  }
+
+  SRecordInfo info;
+  info.offset = mfh->size;
+  info.size = contLen;  // TODO: Here is not correct
+
+  mfh->size += (contLen + sizeof(SRecordInfo));
+
+  if (taosAddToHashTable(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  // TODO: make below a function to implement
+  if (fseek(mfh->fd, info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, cont, contLen) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  mfh->nRecord++;
+
+  return 0;
+}
+
+int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
+  char *ptr = taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+  if (ptr == NULL) return -1;
+
+  SRecordInfo info = *(SRecordInfo *)ptr;
+
+  // Remove record from hash table
+  taosDeleteFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+
+  // Remove record from file
+
+  info.offset = -info.offset;
+  if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  mfh->nDel++;
+
+  return 0;
+}
+
+int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+  char *ptr = taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+  if (ptr == NULL) return -1;
+
+  SRecordInfo info = *(SRecordInfo *)ptr;
+  // Update the hash table
+  if (taosAddToHashTable(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  // Update record in file
+  if (info.size >= contLen) {  // Just update it in place
+    info.size = contLen;
+
+  } else {  // Just append to the end of file
+    info.offset = mfh->size;
+    info.size = contLen;
+
+    mfh->size += contLen;
+  }
+  if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  return 0;
+}
+
+void tsdbCloseMetaFile(SMetaFile *mfh) {
+  if (mfh == NULL) return;
+  close(mfh);
+
+  taosCleanUpHashTable(mfh->map);
+}
+
+static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {
+  if (rootDir == NULL) return -1;
+  sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME);
+  return 0;
+}
+
+static int32_t tsdbCheckMetaHeader(int fd) {
+  // TODO: write the meta file header check function
+  return 0;
+}
+
+static int32_t tsdbWriteMetaHeader(int fd) {
+  // TODO: write the meta file header to file
+  return 0;
+}
+
+static int tsdbCreateMetaFile(char *fname) {
+  int fd = open(fname, O_RDWR | O_CREAT, 0755);
+  if (fd < 0) return -1;
+
+  if (tsdbWriteMetaHeader(fd) < 0) {
+    close(fd);
+    return NULL;
+  }
+
+  return fd;
+}
+
+static int tsdbCheckMetaFileIntegrety(int fd) {
+  // TODO
+  return 0;
+}
+
+static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) {
+  int fd = open(fname, O_RDWR);
+  if (fd < 0) return -1;
+
+  if (tsdbCheckMetaFileIntegrety(fd) < 0) {
+    // TODO: decide if to auto-recover the file
+    close(fd);
+    return -1;
+  }
+
+  if (fseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) {
+    // TODO: deal with the error
+    close(fd);
+    return -1;
+  }
+
+  mfh->fd = fd;
+  // TODO: iterate to read the meta file to restore the meta data
+
+  return 0;
+}
\ No newline at end of file

From c80686b9614c31de6a7c243f64fc5caba760cdee Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 10:39:44 +0800
Subject: [PATCH 04/17] add more and refactor

---
 src/vnode/tsdb/inc/tsdbMeta.h     | 28 +++++++++-------
 src/vnode/tsdb/inc/tsdbMetaFile.h |  3 +-
 src/vnode/tsdb/src/tsdbMain.c     | 18 +++++------
 src/vnode/tsdb/src/tsdbMeta.c     | 53 ++++++++++++++++---------------
 src/vnode/tsdb/src/tsdbMetaFile.c |  2 +-
 5 files changed, 53 insertions(+), 51 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h
index 73bafb112f..a86a206950 100644
--- a/src/vnode/tsdb/inc/tsdbMeta.h
+++ b/src/vnode/tsdb/inc/tsdbMeta.h
@@ -20,6 +20,7 @@
 #include "tsdb.h"
 #include "dataformat.h"
 #include "tskiplist.h"
+#include "tsdbMetaFile.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -78,14 +79,24 @@ typedef struct STable {
 
 } STable;
 
+// ---------- TSDB META HANDLE DEFINITION
 typedef struct {
-  int32_t  maxTables;
-  int32_t  nTables;
-  STable **tables;    // array of normal tables
-  STable * stables;   // linked list of super tables // TODO use container to implement this
-  void *   tableMap;  // hash map of uid ==> STable *
+  int32_t maxTables;  // Max number of tables
+
+  int32_t nTables;  // Tables created
+
+  STable **tables;  // table array
+
+  STable *superList;  // super table list TODO: change  it to list container
+
+  void *map; // table map of (uid ===> table)
+
+  SMetaFile *mfh; // meta file handle
 } STsdbMeta;
 
+STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables);
+int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
+
 // ---- Operation on STable
 #define TSDB_TABLE_ID(pTable) ((pTable)->tableId)
 #define TSDB_TABLE_UID(pTable) ((pTable)->uid)
@@ -105,13 +116,6 @@ STSchema *tsdbGetTableSchema(STable *pTable);
 #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
 #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
 
-// Create a new meta handle with configuration
-STsdbMeta *tsdbInitMeta(int32_t maxTables);
-int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
-
-// Recover the meta handle from the file
-STsdbMeta *tsdbOpenMeta(char *tsdbDir);
-
 int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
 int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
 STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
diff --git a/src/vnode/tsdb/inc/tsdbMetaFile.h b/src/vnode/tsdb/inc/tsdbMetaFile.h
index 758cb57958..9fad703842 100644
--- a/src/vnode/tsdb/inc/tsdbMetaFile.h
+++ b/src/vnode/tsdb/inc/tsdbMetaFile.h
@@ -18,13 +18,12 @@
 
 #include <stdint.h>
 
-#include "tsdbMeta.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #define TSDB_META_FILE_NAME "META"
+#define TSDB_META_HASH_FRACTION 1.1
 
 typedef struct {
   int     fd;       // File descriptor
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 5db2545c53..c608353645 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -145,8 +145,15 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO
   pRepo->config = *pCfg;
   pRepo->limiter = limiter;
 
+  // Create the environment files and directories
+  if (tsdbSetRepoEnv(pRepo) < 0) {
+    free(pRepo->rootDir);
+    free(pRepo);
+    return NULL;
+  }
+
   // Initialize meta
-  STsdbMeta *pMeta = tsdbInitMeta(pCfg->maxTables);
+  STsdbMeta *pMeta = tsdbInitMeta(rootDir, pCfg->maxTables);
   if (pMeta == NULL) {
     free(pRepo->rootDir);
     free(pRepo);
@@ -164,15 +171,6 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO
   }
   pRepo->tsdbCache = pCache;
 
-  // Create the environment files and directories
-  if (tsdbSetRepoEnv(pRepo) < 0) {
-    free(pRepo->rootDir);
-    tsdbFreeMeta(pRepo->tsdbMeta);
-    tsdbFreeCache(pRepo->tsdbCache);
-    free(pRepo);
-    return NULL;
-  }
-
   pRepo->state = TSDB_REPO_STATE_ACTIVE;
 
   return (tsdb_repo_t *)pRepo;
diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index 23442ed1db..4f943c3b41 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -18,23 +18,33 @@ static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
 
-STsdbMeta *tsdbInitMeta(int32_t maxTables) {
+/**
+ * Initialize the meta handle
+ * ASSUMPTIONS: VALID PARAMETER
+ */
+STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
   STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
-  if (pMeta == NULL) {
-    return NULL;
-  }
+  if (pMeta == NULL) return NULL;
 
   pMeta->maxTables = maxTables;
   pMeta->nTables = 0;
-  pMeta->stables = NULL;
+  pMeta->superList = NULL;
   pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *));
   if (pMeta->tables == NULL) {
     free(pMeta);
     return NULL;
   }
 
-  pMeta->tableMap = taosInitHashTable(maxTables + maxTables / 10, taosGetDefaultHashFunction, false);
-  if (pMeta->tableMap == NULL) {
+  pMeta->map = taosInitHashTable(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  if (pMeta->map == NULL) {
+    free(pMeta->tables);
+    free(pMeta);
+    return NULL;
+  }
+
+  pMeta->mfh = tsdbInitMetaFile(rootDir, maxTables);
+  if (pMeta->mfh == NULL) {
+    taosCleanUpHashTable(pMeta->map);
     free(pMeta->tables);
     free(pMeta);
     return NULL;
@@ -46,6 +56,8 @@ STsdbMeta *tsdbInitMeta(int32_t maxTables) {
 int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
   if (pMeta == NULL) return 0;
 
+  tsdbCloseMetaFile(pMeta->mfh);
+
   for (int i = 0; i < pMeta->maxTables; i++) {
     if (pMeta->tables[i] != NULL) {
       tsdbFreeTable(pMeta->tables[i]);
@@ -54,14 +66,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 
   free(pMeta->tables);
 
-  STable *pTable = pMeta->stables;
+  STable *pTable = pMeta->superList;
   while (pTable != NULL) {
     STable *pTemp = pTable;
     pTable = pTemp->next;
     tsdbFreeTable(pTemp);
   }
 
-  taosCleanUpHashTable(pMeta->tableMap);
+  taosCleanUpHashTable(pMeta->map);
 
   free(pMeta);
 
@@ -126,17 +138,6 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
   return 0;
 }
 
-STsdbMeta *tsdbOpenMeta(char *tsdbDir) {
-  // TODO : Open meta file for reading
-
-  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
-  if (pMeta == NULL) {
-    return NULL;
-  }
-
-  return pMeta;
-}
-
 /**
  * Check if a table is valid to insert.
  * @return NULL for invalid and the pointer to the table if valid
@@ -206,7 +207,7 @@ static int32_t tsdbCheckTableCfg(STableCfg *pCfg) {
 }
 
 STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
-  void *ptr = taosGetDataFromHashTable(pMeta->tableMap, (char *)(&uid), sizeof(uid));
+  void *ptr = taosGetDataFromHashTable(pMeta->map, (char *)(&uid), sizeof(uid));
 
   if (ptr == NULL) return NULL;
 
@@ -216,12 +217,12 @@ STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
 static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable) {
   if (pTable->type == TSDB_SUPER_TABLE) { 
     // add super table to the linked list
-    if (pMeta->stables == NULL) {
-      pMeta->stables = pTable;
+    if (pMeta->superList == NULL) {
+      pMeta->superList = pTable;
       pTable->next = NULL;
     } else {
-      STable *pTemp = pMeta->stables;
-      pMeta->stables = pTable;
+      STable *pTemp = pMeta->superList;
+      pMeta->superList = pTable;
       pTable->next = pTemp;
     }
   } else {
@@ -245,7 +246,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
 static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) {
   // TODO: add the table to the map
   int64_t uid = pTable->tableId.uid;
-  if (taosAddToHashTable(pMeta->tableMap, (char *)(&uid), sizeof(uid), (void *)(&pTable), sizeof(pTable)) < 0) {
+  if (taosAddToHashTable(pMeta->map, (char *)(&uid), sizeof(uid), (void *)(&pTable), sizeof(pTable)) < 0) {
     return -1;
   }
   return 0;
diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c
index 828d722389..8f99afdcdd 100644
--- a/src/vnode/tsdb/src/tsdbMetaFile.c
+++ b/src/vnode/tsdb/src/tsdbMetaFile.c
@@ -15,11 +15,11 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "taosdef.h"
 #include "hash.h"
 #include "tsdbMetaFile.h"
 
 #define TSDB_META_FILE_HEADER_SIZE 512
-#define TSDB_META_HASH_FRACTION 1.1
 
 typedef struct {
   int32_t offset;

From e871eb8cf69d65cc3114b6aa6039cbab8c7e1ff8 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 12:17:44 +0800
Subject: [PATCH 05/17] add more code

---
 src/vnode/tsdb/inc/tsdbMeta.h |  7 +++--
 src/vnode/tsdb/src/tsdbMeta.c | 56 +++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h
index a86a206950..0a7f2a0bc6 100644
--- a/src/vnode/tsdb/inc/tsdbMeta.h
+++ b/src/vnode/tsdb/inc/tsdbMeta.h
@@ -39,6 +39,7 @@ typedef enum {
 
 #define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)
 
+// ---------- TSDB TABLE DEFINITION
 typedef struct STable {
   STableId        tableId;
   TSDB_TABLE_TYPE type;
@@ -79,6 +80,10 @@ typedef struct STable {
 
 } STable;
 
+void *  tsdbEncodeTable(STable *pTable, int *contLen);
+STable *tsdbDecodeTable(void *cont, int contLen);
+void *  tsdbFreeEncode(void *cont);
+
 // ---------- TSDB META HANDLE DEFINITION
 typedef struct {
   int32_t maxTables;  // Max number of tables
@@ -108,8 +113,6 @@ int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 #define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData)
 #define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex)
 
-STSchema *tsdbGetTableSchema(STable *pTable);
-
 // ---- Operation on SMetaHandle
 #define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables)
 #define TSDB_NUM_OF_SUPER_TABLES(pHandle) ((pHandle)->numOfSuperTables)
diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index 4f943c3b41..b0acd811ac 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -17,6 +17,57 @@ static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
+static int     tsdbEstimateTableEncodeSize(STable *pTable);
+
+/**
+ * Encode a TSDB table object as a binary content
+ * ASSUMPTIONS: VALID PARAMETERS
+ * 
+ * @param pTable table object to encode
+ * @param contLen the encoded binary content length
+ * 
+ * @return binary content for success
+ *         NULL fro failure
+ */
+void *tsdbEncodeTable(STable *pTable, int *contLen) {
+  if (pTable == NULL) return NULL;
+
+  *contLen = tsdbEstimateTableEncodeSize(pTable);
+  if (*contLen < 0) return NULL;
+
+  void *ret = malloc(*contLen);
+  if (ret == NULL) return NULL;
+
+  // TODO: encode the object to the memory
+  {}
+
+  return ret;
+}
+
+/**
+ * Decode from an encoded binary
+ * ASSUMPTIONS: valid parameters
+ * 
+ * @param cont binary object
+ * @param contLen binary length
+ * 
+ * @return TSDB table object for success
+ *         NULL for failure
+ */
+STable *tsdbDecodeTable(void *cont, int contLen) {
+  STable *pTable = (STable *)calloc(1, sizeof(STable));
+  if (pTable == NULL) return NULL;
+
+  {
+    // TODO recover from the binary content
+  }
+
+  return pTable;
+}
+
+void *tsdbFreeEncode(void *cont) {
+  if (cont != NULL) free(cont);
+}
 
 /**
  * Initialize the meta handle
@@ -261,4 +312,9 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   assert(pTable->type == TSDB_STABLE);
   // TODO
   return 0;
+}
+
+static int tsdbEstimateTableEncodeSize(STable *pTable) {
+  // TODO
+  return 0;
 }
\ No newline at end of file

From a614570ccfc998dd967c94e88af24402fcd4f132 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 15:45:24 +0800
Subject: [PATCH 06/17] refactor more

---
 src/vnode/tsdb/inc/tsdb.h     | 53 ++++++++++++---------
 src/vnode/tsdb/inc/tsdbMeta.h | 50 ++++----------------
 src/vnode/tsdb/src/tsdbMain.c | 87 +++++++++++++++++++++++++++++++++--
 src/vnode/tsdb/src/tsdbMeta.c | 74 ++++++++++++++---------------
 4 files changed, 163 insertions(+), 101 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index cf9c4502dd..f7c37009e3 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#if !defined(_TD_TSDB_H_)
+#ifndef _TD_TSDB_H_
 #define _TD_TSDB_H_
 
 #include <pthread.h>
@@ -30,8 +30,15 @@ extern "C" {
 #define TSDB_VERSION_MAJOR 1
 #define TSDB_VERSION_MINOR 0
 
+#define TSDB_INVALID_SUPER_TABLE_ID -1
+
 // --------- TSDB REPOSITORY CONFIGURATION DEFINITION
 enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
+typedef enum {
+  TSDB_SUPER_TABLE,  // super table
+  TSDB_NTABLE,       // table not created from super table
+  TSDB_STABLE        // table created from super table
+} TSDB_TABLE_TYPE;
 
 typedef struct {
   int8_t  precision;
@@ -63,6 +70,28 @@ typedef struct {
   int32_t tid;  // the table ID in the repository.
 } STableId;
 
+// --------- TSDB TABLE configuration
+typedef struct {
+  TSDB_TABLE_TYPE type;
+  STableId        tableId;
+  int64_t         superUid;
+  STSchema *      schema;
+  STSchema *      tagSchema;
+  SDataRow        tagValues;
+} STableCfg;
+
+int  tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid);
+int  tsdbTableSetSuperUid(STableCfg *config, int64_t uid);
+int  tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
+int  tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
+int  tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup);
+void tsdbClearTableCfg(STableCfg *config);
+
+int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
+int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
+int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
+
+
 // Submit message for this TSDB
 typedef struct {
   int32_t numOfTables;
@@ -88,25 +117,8 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-
 STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
 
-// the meter configuration
-typedef struct {
-  STableId tableId;
-
-  int64_t stableUid;
-  int64_t createdTime;
-
-  int32_t  numOfCols;  // number of columns. For table form super table, not includes the tag schema
-  STSchema *schema;     // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL
-                       // If numOfCols < schema->numOfCols, it is a table created from super table
-                       // assert(numOfCols <= schema->numOfCols);
-
-  SDataRow tagValues;  // NULL if it is normal table
-                       // otherwise, it contains the tag values.
-} STableCfg;
-
 // the meter information report structure
 typedef struct {
   STableCfg tableCfg;
@@ -114,6 +126,7 @@ typedef struct {
   int64_t   tableTotalDataSize;  // In bytes
   int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;
+STableInfo *   tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
 
 // -- For table manipulation
 
@@ -124,8 +137,6 @@ typedef struct {
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
-int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
 
 /**
  * Drop a table in a repository and free all the resources it takes
@@ -135,7 +146,6 @@ int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
 
 /**
  * Get the information of a table in the repository
@@ -145,7 +155,6 @@ int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
  *
  * @return a table information handle for success, NULL for failure and the error number is set
  */
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
 
 // -- FOR INSERT DATA
 /**
diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h
index 0a7f2a0bc6..be7c7d0406 100644
--- a/src/vnode/tsdb/inc/tsdbMeta.h
+++ b/src/vnode/tsdb/inc/tsdbMeta.h
@@ -31,53 +31,23 @@ extern "C" {
 // Initially, there are 4 tables
 #define TSDB_INIT_NUMBER_OF_SUPER_TABLE 4
 
-typedef enum {
-  TSDB_SUPER_TABLE,  // super table
-  TSDB_NTABLE,       // table not created from super table
-  TSDB_STABLE        // table created from super table
-} TSDB_TABLE_TYPE;
-
 #define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)
 
 // ---------- TSDB TABLE DEFINITION
 typedef struct STable {
-  STableId        tableId;
   TSDB_TABLE_TYPE type;
-
-  int64_t createdTime;
-
-  // super table UID -1 for normal table
-  int32_t stableUid;
-
-  int32_t numOfCols;
-
-  // Schema for this table
-  // For TSDB_SUPER_TABLE, it is the schema including tags
-  // For TSDB_NTABLE, it is only the schema, not including tags
-  // For TSDB_STABLE, it is NULL
-  STSchema *pSchema;
-
-  // Tag value for this table
-  // For TSDB_SUPER_TABLE and TSDB_NTABLE, it is NULL
-  // For TSDB_STABLE, it is the tag value string
-  SDataRow pTagVal;
-
-  // Object content;
-  // For TSDB_SUPER_TABLE, it is the index of tables created from it
-  // For TSDB_STABLE and TSDB_NTABLE, it is the cache data
+  STableId        tableId;
+  int32_t         superUid;  // Super table UID
+  STSchema *      schema;
+  STSchema *      tagSchema;
+  SDataRow        tagVal;
   union {
-    void *pData;
-    void *pIndex;
+    void *pData;   // For TSDB_NTABLE and TSDB_STABLE, it is the skiplist for cache data
+    void *pIndex;  // For TSDB_SUPER_TABLE, it is the skiplist index
   } content;
-
-  // A handle to deal with event
-  void *eventHandler;
-
-  // A handle to deal with stream
-  void *streamHandler;
-
-  struct STable *next;
-
+  void *         eventHandler;   // TODO
+  void *         streamHandler;  // TODO
+  struct STable *next; // TODO: remove the next
 } STable;
 
 void *  tsdbEncodeTable(STable *pTable, int *contLen);
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index c608353645..2842f40466 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -297,17 +297,17 @@ STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
   return NULL;
 }
 
-int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
+int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
 }
 
-int32_t tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
+int tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
   // TODO
   return 0;
 }
 
-int32_t tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
+int tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
   // TODO
   if (repo == NULL) return -1;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
@@ -334,6 +334,87 @@ int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
   return 0;
 }
 
+/**
+ * Initialize a table configuration
+ */
+int tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid) {
+  if (config == NULL) return -1;
+  if (type != TSDB_NTABLE && type != TSDB_STABLE) return -1;
+
+  memset((void *)config, 0, sizeof(STableCfg));
+
+  config->type = type;
+  config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
+  config->tableId.uid = uid;
+  config->tableId.tid = tid;
+  return -1;
+}
+
+/**
+ * Set the super table UID of the created table
+ */
+int tsdbTableSetSuperUid(STableCfg *config, int64_t uid) {
+  if (config->type != TSDB_STABLE) return -1;
+  if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1;
+
+  config->superUid = uid;
+  return 0;
+}
+
+/**
+ * Set the table schema in the configuration
+ * @param config the configuration to set
+ * @param pSchema the schema to set
+ * @param dup use the schema directly or duplicate one for use
+ * 
+ * @return 0 for success and -1 for failure
+ */
+int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+  if (dup) {
+    config->schema = tdDupSchema(pSchema);
+  } else {
+    config->schema = pSchema;
+  }
+  return 0;
+}
+
+/**
+ * Set the table schema in the configuration
+ * @param config the configuration to set
+ * @param pSchema the schema to set
+ * @param dup use the schema directly or duplicate one for use
+ * 
+ * @return 0 for success and -1 for failure
+ */
+int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+  if (config->type != TSDB_STABLE) return -1;
+
+  if (dup) {
+    config->tagSchema = tdDupSchema(pSchema);
+  } else {
+    config->tagSchema = pSchema;
+  }
+  return 0;
+}
+
+int tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup) {
+  if (config->type != TSDB_STABLE) return -1;
+
+  if (dup) {
+    config->tagValues = tdDataRowDup(row);
+  } else {
+    config->tagValues = row;
+  }
+
+  return 0;
+}
+
+void tsdbClearTableCfg(STableCfg *config) {
+  if (config->schema) tdFreeSchema(config->schema);
+  if (config->tagSchema) tdFreeSchema(config->tagSchema);
+  if (config->tagValues) tdFreeDataRow(config->tagValues);
+}
+
 // Check the configuration and set default options
 static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   // Check precision
diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index b0acd811ac..b654fd4739 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -132,59 +132,61 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 }
 
 int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
-  if (tsdbCheckTableCfg(pCfg) < 0) {
-    return -1;
-  }
+  if (tsdbCheckTableCfg(pCfg) < 0) return -1;
 
-  STable *pSTable = NULL;
+  STable *super = NULL;
   int newSuper = 0;
 
-  if (IS_CREATE_STABLE(pCfg)) {  // to create a TSDB_STABLE, check if super table exists
-    pSTable = tsdbGetTableByUid(pMeta, pCfg->stableUid);
-    if (pSTable == NULL) {  // super table not exists, try to create it
+  if (pCfg->type == TSDB_STABLE) { 
+    super = tsdbGetTableByUid(pMeta, pCfg->superUid);
+    if (super == NULL) {  // super table not exists, try to create it
       newSuper = 1;
-      pSTable = (STable *)calloc(1, sizeof(STable));
-      if (pSTable == NULL) return -1;
+      // TODO: use function to implement create table object
+      super = (STable *)calloc(1, sizeof(STable));
+      if (super == NULL) return -1;
 
-      pSTable->tableId.uid = pCfg->stableUid;
-      pSTable->tableId.tid = -1;
-      pSTable->type = TSDB_SUPER_TABLE;
-      // pSTable->createdTime = pCfg->createdTime; // The created time is not required
-      pSTable->stableUid = -1;
-      pSTable->numOfCols = pCfg->numOfCols;
-      pSTable->pSchema = tdDupSchema(pCfg->schema);
-      pSTable->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1,
+      super->type = TSDB_SUPER_TABLE;
+      super->tableId.uid = pCfg->superUid;
+      super->tableId.tid = -1;
+      super->superUid = TSDB_INVALID_SUPER_TABLE_ID;
+      super->schema = tdDupSchema(pCfg->schema);
+      super->tagSchema = tdDupSchema(pCfg->tagSchema);
+      super->tagVal = tdDataRowDup(pCfg->tagValues);
+      super->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1,
                                                 0, NULL);  // Allow duplicate key, no lock
-      if (pSTable->content.pIndex == NULL) {
-        free(pSTable);
+
+      if (super->content.pIndex == NULL) {
+        tdFreeSchema(super->schema);
+        tdFreeSchema(super->tagSchema);
+        tdFreeDataRow(super->tagVal);
+        free(super);
         return -1;
       }
     } else {
-      if (pSTable->type != TSDB_SUPER_TABLE) return -1;
+      if (super->type != TSDB_SUPER_TABLE) return -1;
     }
   }
 
-  STable *pTable = (STable *)malloc(sizeof(STable));
-  if (pTable == NULL) {
-    if (newSuper) tsdbFreeTable(pSTable);
+  STable *table = (STable *)malloc(sizeof(STable));
+  if (table == NULL) {
+    if (newSuper) tsdbFreeTable(super);
     return -1;
   }
 
-  pTable->tableId = pCfg->tableId;
-  pTable->createdTime = pCfg->createdTime;
+  table->tableId = pCfg->tableId;
   if (IS_CREATE_STABLE(pCfg)) { // TSDB_STABLE
-    pTable->type = TSDB_STABLE;
-    pTable->stableUid = pCfg->stableUid;
-    pTable->pTagVal = tdDataRowDup(pCfg->tagValues);
+    table->type = TSDB_STABLE;
+    table->superUid = pCfg->superUid;
+    table->tagVal = tdDataRowDup(pCfg->tagValues);
   } else { // TSDB_NTABLE
-    pTable->type = TSDB_NTABLE;
-    pTable->stableUid = -1;
-    pTable->pSchema = tdDupSchema(pCfg->schema);
+    table->type = TSDB_NTABLE;
+    table->superUid = -1;
+    table->schema = tdDupSchema(pCfg->schema);
   }
-  pTable->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, 0, 8, 0, 0, NULL);
+  table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, 0, 8, 0, 0, NULL);
 
-  if (newSuper) tsdbAddTableToMeta(pMeta, pSTable);
-  tsdbAddTableToMeta(pMeta, pTable);
+  if (newSuper) tsdbAddTableToMeta(pMeta, super);
+  tsdbAddTableToMeta(pMeta, table);
 
   return 0;
 }
@@ -236,9 +238,9 @@ int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) {
 static int tsdbFreeTable(STable *pTable) {
   // TODO: finish this function
   if (pTable->type == TSDB_STABLE) {
-    tdFreeDataRow(pTable->pTagVal);
+    tdFreeDataRow(pTable->tagVal);
   } else {
-    tdFreeSchema(pTable->pSchema);
+    tdFreeSchema(pTable->schema);
   }
 
   // Free content

From 14bbe488c7981cce0064161d3a179540178ba407 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 16:33:19 +0800
Subject: [PATCH 07/17] add more

---
 src/common/inc/dataformat.h        | 10 ++--
 src/common/src/dataformat.c        | 31 +++++++++-
 src/vnode/tsdb/CMakeLists.txt      |  2 +-
 src/vnode/tsdb/src/tsdbMain.c      |  2 +-
 src/vnode/tsdb/tests/tsdbTests.cpp | 94 ++++++------------------------
 5 files changed, 56 insertions(+), 83 deletions(-)

diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index cc13ab2eca..c263f4128b 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -51,18 +51,20 @@ void      tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes);
 
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
-  int32_t  numOfCols;
-  int32_t  padding;  // TODO: replace the padding for useful variable
+  int      numOfCols;  // Number of columns appended
+  int      totalCols;  // Total columns allocated
   STColumn columns[];
 } STSchema;
 
 #define schemaNCols(s) ((s)->numOfCols)
+#define schemaTCols(s) ((s)->totalCols)
 #define schemaColAt(s, i) ((s)->columns + i)
 
 STSchema *tdNewSchema(int32_t nCols);
+int       tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int16_t bytes);
 STSchema *tdDupSchema(STSchema *pSchema);
-void tdFreeSchema(STSchema *pSchema);
-void tdUpdateSchema(STSchema *pSchema);
+void      tdFreeSchema(STSchema *pSchema);
+void      tdUpdateSchema(STSchema *pSchema);
 
 // ----------------- Data row structure
 
diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c
index 064cb3ff29..ce07afd17d 100644
--- a/src/common/src/dataformat.c
+++ b/src/common/src/dataformat.c
@@ -89,13 +89,40 @@ void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
 STSchema *tdNewSchema(int32_t nCols) {
   int32_t  size = sizeof(STSchema) + sizeof(STColumn) * nCols;
 
-  STSchema *pSchema = (STSchema *)calloc(1, size);
+  STSchema *pSchema = (STSchema *)malloc(size);
   if (pSchema == NULL) return NULL;
-  pSchema->numOfCols = nCols;
+  pSchema->numOfCols = 0;
+  pSchema->totalCols = nCols;
 
   return pSchema;
 }
 
+/**
+ * Append a column to the schema
+ */
+int tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int16_t bytes) {
+  if (pSchema->numOfCols >= pSchema->totalCols) return -1;
+  if (!isValidDataType(type, 0)) return -1;
+
+  STColumn *pCol = schemaColAt(pSchema, schemaNCols(pSchema));
+  colSetType(pCol, type);
+  colSetColId(pCol, colId);
+  colSetOffset(pCol, -1);
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      colSetBytes(pCol, bytes);
+      break;
+    default:
+      colSetBytes(pCol, TYPE_BYTES[type]);
+      break;
+  }
+
+  pSchema->numOfCols++;
+
+  return 0;
+}
+
 /**
  * Duplicate the schema and return a new object
  */
diff --git a/src/vnode/tsdb/CMakeLists.txt b/src/vnode/tsdb/CMakeLists.txt
index 1317e32b51..8a7c7a1a51 100644
--- a/src/vnode/tsdb/CMakeLists.txt
+++ b/src/vnode/tsdb/CMakeLists.txt
@@ -15,5 +15,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   TARGET_LINK_LIBRARIES(tsdb common tutil)
 
   # Someone has no gtest directory, so comment it
-  #ADD_SUBDIRECTORY(tests)
+  ADD_SUBDIRECTORY(tests)
 ENDIF ()
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 2842f40466..147dcff2a5 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -347,7 +347,7 @@ int tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32
   config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
   config->tableId.uid = uid;
   config->tableId.tid = tid;
-  return -1;
+  return 0;
 }
 
 /**
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 9e88e156a9..757fd10017 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -3,92 +3,36 @@
 
 #include "tsdb.h"
 #include "dataformat.h"
-#include "tsdbMeta.h"
 
 TEST(TsdbTest, createRepo) {
   STsdbCfg config;
 
-  // Create a tsdb repository
+  // 1. Create a tsdb repository
   tsdbSetDefaultCfg(&config);
   tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
   ASSERT_NE(pRepo, nullptr);
 
-  // // create a normal table in this repository
-  // STableCfg config;
-  // config.tableId.tid = 0;
-  // config.tableId.uid = 98868728187539L;
-  // config.numOfCols = 5;
-  // config.schema = tdNewSchema(config.numOfCols);
-  // STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
-  // tdColCpy(schemaColAt(config.schema, 0), pCol);
-  // tdFreeCol(pCol);
-  // for (int i = 1; i < schemaNCols(config.schema); i++) {
-  //   pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
-  //   tdColCpy(schemaColAt(config.schema, i), pCol);
-  //   tdFreeCol(pCol);
-  // }
+  // 2. Create a normal table
+  STableCfg tCfg;
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NTABLE, 987607499877672L, 0), 0);
 
-  // tsdbCreateTable(pRepo, &config);
-  // Write some data
+  int nCols = 5;
+  STSchema *schema = tdNewSchema(nCols);
 
-  // int32_t size = sizeof(SSubmitMsg) + sizeof(SSubmitBlock) + tdMaxRowDataBytes(config.schema) * 10 + sizeof(int32_t);
-
-  // tdUpdateSchema(config.schema);
-
-  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(size);
-  // pMsg->numOfTables = 1;  // TODO: use api
-
-  // SSubmitBlock *pBlock = (SSubmitBlock *)pMsg->data;
-  // pBlock->tableId = {.uid = 98868728187539L, .tid = 0};
-  // pBlock->sversion = 0;
-  // pBlock->len = sizeof(SSubmitBlock);
-
-  // SDataRows rows = pBlock->data;
-  // dataRowsInit(rows);
-
-  // SDataRow row = tdNewDataRow(tdMaxRowDataBytes(config.schema));
-  // int64_t ttime = 1583508800000;
-  // for (int i = 0; i < 10; i++) {  // loop over rows
-  //   ttime += (10000 * i);
-  //   tdDataRowReset(row);
-  //   for (int j = 0; j < schemaNCols(config.schema); j++) {
-  //     if (j == 0) {  // set time stamp
-  //       tdAppendColVal(row, (void *)(&ttime), schemaColAt(config.schema, j), 40);
-  //     } else {       // set other fields
-  //       int32_t val = 10;
-  //       tdAppendColVal(row, (void *)(&val), schemaColAt(config.schema, j), 40);
-  //     }
-  //   }
-
-  //   tdDataRowsAppendRow(rows, row);
-  // }
-
-  // tsdbInsertData(pRepo, pMsg);
-
-  // tdFreeDataRow(row);
-
-  // tdFreeSchema(config.schema);
-  // tsdbDropRepo(pRepo);
-}
-
-TEST(TsdbTest, DISABLED_createTable) {
-  STsdbMeta *pMeta = tsdbInitMeta(100);
-  ASSERT_NE(pMeta, nullptr);
-
-  STableCfg config;
-  config.tableId.tid = 0;
-  config.tableId.uid = 98868728187539L;
-  config.numOfCols = 5;
-  config.schema = tdNewSchema(config.numOfCols);
-  for (int i = 0; i < schemaNCols(config.schema); i++) {
-    STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
-    tdColCpy(schemaColAt(config.schema, i), pCol);
-    tdFreeCol(pCol);
+  for (int i = 0; i < nCols; i++)
+  {
+    if (i == 0) {
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+    } else {
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+    }
   }
-  config.tagValues = nullptr;
 
-  tsdbCreateTableImpl(pMeta, &config);
+  tsdbTableSetSchema(&tCfg, schema, true);
 
-  STable *pTable = tsdbGetTableByUid(pMeta, config.tableId.uid);
-  ASSERT_NE(pTable, nullptr);
+  tsdbCreateTable(pRepo, &tCfg);
+
+  // 3. Loop to write some simple data
 }
+

From 7e0ece016a1a1636d63b0c422b8c3066a5700d45 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 17:20:52 +0800
Subject: [PATCH 08/17] refactor and add more

---
 src/common/inc/dataformat.h        |  7 +++++--
 src/common/src/dataformat.c        | 31 ++++++++++++++++++++++++++----
 src/common/src/ttypes.c            |  4 ++--
 src/vnode/tsdb/tests/tsdbTests.cpp |  3 +++
 4 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index c263f4128b..76b5317b35 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -79,14 +79,17 @@ void      tdUpdateSchema(STSchema *pSchema);
  */
 typedef void *SDataRow;
 
+#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
+
 #define dataRowLen(r) (*(int32_t *)(r))
-#define dataRowTuple(r) ((char *)(r) + sizeof(int32_t))
+#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
 #define dataRowIdx(r, i) ((char *)(r) + i)
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
 
 SDataRow tdNewDataRow(int32_t bytes);
-// SDataRow tdNewDdataFromSchema(SSchema *pSchema);
+int      tdMaxRowBytesFromSchema(STSchema *pSchema);
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
 // int32_t  tdAppendColVal(SDataRow row, void *value, SColumn *pCol, int32_t suffixOffset);
 void     tdDataRowCpy(void *dst, SDataRow row);
diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c
index ce07afd17d..c6fd4af284 100644
--- a/src/common/src/dataformat.c
+++ b/src/common/src/dataformat.c
@@ -178,10 +178,33 @@ SDataRow tdNewDataRow(int32_t bytes) {
   return row;
 }
 
-// SDataRow tdNewDdataFromSchema(SSchema *pSchema) {
-//   int32_t bytes = tdMaxRowDataBytes(pSchema);
-//   return tdNewDataRow(bytes);
-// }
+/**
+ * Get maximum bytes a data row from a schema
+ * ASSUMPTIONS: VALID PARAMETER
+ */
+int tdMaxRowBytesFromSchema(STSchema *pSchema) {
+  // TODO
+  int bytes = TD_DATA_ROW_HEAD_SIZE;
+  for (int i = 0; i < schemaNCols(pSchema); i++) {
+    STColumn *pCol = schemaColAt(pSchema, i);
+    bytes += TYPE_BYTES[pCol->type];
+
+    if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
+      bytes += pCol->bytes;
+    }
+  }
+
+  return bytes;
+}
+
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
+  int bytes = 0;
+  {
+    // TODO: estimiate size from schema
+  }
+
+  return tdNewDataRow(bytes);
+}
 
 /**
  * Free the SDataRow object
diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c
index 14b4d593fb..2f4aa6ab76 100644
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@@ -26,9 +26,9 @@ const int32_t TYPE_BYTES[11] = {
     sizeof(int64_t),  // TSDB_DATA_TYPE_BIGINT
     sizeof(float),    // TSDB_DATA_TYPE_FLOAT
     sizeof(double),   // TSDB_DATA_TYPE_DOUBLE
-    -1,               // TSDB_DATA_TYPE_BINARY
+    sizeof(int32_t),  // TSDB_DATA_TYPE_BINARY
     sizeof(TSKEY),    // TSDB_DATA_TYPE_TIMESTAMP
-    -1                // TSDB_DATA_TYPE_NCHAR
+    sizeof(int32_t)   // TSDB_DATA_TYPE_NCHAR
 };
 
 tDataTypeDescriptor tDataTypeDesc[11] = {
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 757fd10017..34a712dc0f 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -34,5 +34,8 @@ TEST(TsdbTest, createRepo) {
   tsdbCreateTable(pRepo, &tCfg);
 
   // 3. Loop to write some simple data
+  SDataRow row = tdNewDataRowFromSchema(schema);
+  for (int i = 0; i < nCols; i++) {
+  }
 }
 

From f3c92f8e5385a87f101fcfdd0b4643b880ae5fa7 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Thu, 12 Mar 2020 18:08:27 +0800
Subject: [PATCH 09/17] refactor and add more code

---
 src/common/inc/dataformat.h        |  2 +-
 src/vnode/tsdb/inc/tsdb.h          | 28 ++++++++++++------
 src/vnode/tsdb/src/tsdbMain.c      | 46 ++++++++++++++++++++++++------
 src/vnode/tsdb/tests/tsdbTests.cpp | 17 +++++++----
 4 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index 76b5317b35..08e49dd3c5 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -91,7 +91,7 @@ SDataRow tdNewDataRow(int32_t bytes);
 int      tdMaxRowBytesFromSchema(STSchema *pSchema);
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
-// int32_t  tdAppendColVal(SDataRow row, void *value, SColumn *pCol, int32_t suffixOffset);
+int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol, int32_t suffixOffset);
 void     tdDataRowCpy(void *dst, SDataRow row);
 void     tdDataRowReset(SDataRow row);
 SDataRow tdDataRowDup(SDataRow row);
diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index f7c37009e3..c41384114a 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -91,14 +91,6 @@ int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
 int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
 int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
 
-
-// Submit message for this TSDB
-typedef struct {
-  int32_t numOfTables;
-  int32_t compressed;
-  char    data[];
-} SSubmitMsg;
-
 // Submit message for one table
 typedef struct {
   STableId tableId;
@@ -106,8 +98,26 @@ typedef struct {
   int32_t  sversion;   // data schema version
   int32_t  len;        // message length
   char     data[];
-} SSubmitBlock;
+} SSubmitBlk;
 
+// Submit message for this TSDB
+typedef struct {
+  int32_t    length;
+  int32_t    compressed;
+  SSubmitBlk blocks[];
+} SSubmitMsg;
+
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+// SSubmitMsg Iterator
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  SSubmitBlk *pBlock;
+} SSubmitMsgIter;
+
+int           tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 147dcff2a5..37901ba588 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -78,7 +78,7 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
 static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
 static int     tsdbOpenMetaFile(char *tsdbDir);
 static int     tsdbRecoverRepo(int fd, STsdbCfg *pCfg);
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlock *pBlock);
+static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
 
 #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
 #define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@@ -322,14 +322,14 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) {
 
 // TODO: need to return the number of data inserted
 int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
-  SSubmitBlock *pBlock = (SSubmitBlock *)pMsg->data;
+  SSubmitBlk *pBlock = (SSubmitBlk *)pMsg->blocks;
 
-  for (int i = 0; i < pMsg->numOfTables; i++) {  // Loop to deal with the submit message
-    if (tsdbInsertDataToTable(repo, pBlock) < 0) {
-      return -1;
-    }
-    pBlock = (SSubmitBlock *)(((char *)pBlock) + sizeof(SSubmitBlock) + pBlock->len);
-  }
+  // for (int i = 0; i < pMsg->numOfTables; i++) {  // Loop to deal with the submit message
+  //   if (tsdbInsertDataToTable(repo, pBlock) < 0) {
+  //     return -1;
+  //   }
+  //   pBlock = (SSubmitBlk *)(((char *)pBlock) + sizeof(SSubmitBlk) + pBlock->len);
+  // }
 
   return 0;
 }
@@ -415,6 +415,34 @@ void tsdbClearTableCfg(STableCfg *config) {
   if (config->tagValues) tdFreeDataRow(config->tagValues);
 }
 
+int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+  if (pMsg == NULL || pIter == NULL) return -1;
+
+  pIter->totalLen = pMsg->length;
+  pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE;
+  if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
+    pIter->pBlock = NULL;
+  } else {
+    pIter->pBlock = pMsg->blocks;
+  }
+
+  return 0;
+}
+
+SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
+  SSubmitBlk *pBlock = pIter->pBlock;
+  if (pBlock == NULL) return NULL;
+
+  pIter->len += pBlock->len;
+  if (pIter->len >= pIter->totalLen) {
+    pIter->pBlock = NULL;
+  } else {
+    pIter->pBlock = (char *)pBlock + pBlock->len;
+  }
+
+  return pBlock;
+}
+
 // Check the configuration and set default options
 static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   // Check precision
@@ -601,7 +629,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   return 0;
 }
 
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlock *pBlock) {
+static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, pBlock->tableId);
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 34a712dc0f..1aa85c0808 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -17,11 +17,10 @@ TEST(TsdbTest, createRepo) {
   ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
   ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NTABLE, 987607499877672L, 0), 0);
 
-  int nCols = 5;
+  int       nCols = 5;
   STSchema *schema = tdNewSchema(nCols);
 
-  for (int i = 0; i < nCols; i++)
-  {
+  for (int i = 0; i < nCols; i++) {
     if (i == 0) {
       tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
     } else {
@@ -34,8 +33,14 @@ TEST(TsdbTest, createRepo) {
   tsdbCreateTable(pRepo, &tCfg);
 
   // 3. Loop to write some simple data
-  SDataRow row = tdNewDataRowFromSchema(schema);
-  for (int i = 0; i < nCols; i++) {
-  }
+  // int         size = tdMaxRowBytesFromSchema(schema);
+  // int         nrows = 100;
+  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk+ size * nrows);
+
+  // {
+  //   // TODO
+  // }
+
+  // tsdbInsertData(pRepo, pMsg);
 }
 

From c8ed7d15692d9e8c58d596298cc577a26bed80cb Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 10:19:48 +0800
Subject: [PATCH 10/17] resolve interface change conflict

---
 src/vnode/tsdb/src/tsdbMeta.c     |  6 +++---
 src/vnode/tsdb/src/tsdbMetaFile.c | 20 ++++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c
index 8dc00ade02..5c5c5c50f0 100644
--- a/src/vnode/tsdb/src/tsdbMeta.c
+++ b/src/vnode/tsdb/src/tsdbMeta.c
@@ -86,7 +86,7 @@ STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
     return NULL;
   }
 
-  pMeta->map = taosInitHashTable(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  pMeta->map = taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
   if (pMeta->map == NULL) {
     free(pMeta->tables);
     free(pMeta);
@@ -95,7 +95,7 @@ STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
 
   pMeta->mfh = tsdbInitMetaFile(rootDir, maxTables);
   if (pMeta->mfh == NULL) {
-    taosCleanUpHashTable(pMeta->map);
+    taosHashCleanup(pMeta->map);
     free(pMeta->tables);
     free(pMeta);
     return NULL;
@@ -260,7 +260,7 @@ static int32_t tsdbCheckTableCfg(STableCfg *pCfg) {
 }
 
 STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
-  void *ptr = taosHashGet(pMeta->tableMap, (char *)(&uid), sizeof(uid));
+  void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid));
 
   if (ptr == NULL) return NULL;
 
diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c
index 8f99afdcdd..ee173d7d71 100644
--- a/src/vnode/tsdb/src/tsdbMetaFile.c
+++ b/src/vnode/tsdb/src/tsdbMetaFile.c
@@ -42,7 +42,7 @@ SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables) {
 
   // OPEN MAP
   mfh->map =
-      taosInitHashTable(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+      taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
   if (mfh->map == NULL) {
     free(mfh);
     return NULL;
@@ -52,13 +52,13 @@ SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables) {
   if (access(fname, F_OK) < 0) {  // file not exists
     mfh->fd = tsdbCreateMetaFile(fname);
     if (mfh->fd < 0) {
-      taosCleanUpHashTable(mfh->map);
+      taosHashCleanup(mfh->map);
       free(mfh);
       return NULL;
     }
   } else {  // file exists, recover from file
     if (tsdbRestoreFromMetaFile(fname, mfh) < 0) {
-      taosCleanUpHashTable(mfh->map);
+      taosHashCleanup(mfh->map);
       free(mfh);
       return NULL;
     }
@@ -68,7 +68,7 @@ SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables) {
 }
 
 int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
-  if (taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
+  if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
     return -1;
   }
 
@@ -78,7 +78,7 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co
 
   mfh->size += (contLen + sizeof(SRecordInfo));
 
-  if (taosAddToHashTable(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+  if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
     return -1;
   }
 
@@ -103,13 +103,13 @@ int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co
 }
 
 int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
-  char *ptr = taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+  char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
   if (ptr == NULL) return -1;
 
   SRecordInfo info = *(SRecordInfo *)ptr;
 
   // Remove record from hash table
-  taosDeleteFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+  taosHashRemove(mfh->map, (char *)(&uid), sizeof(uid));
 
   // Remove record from file
 
@@ -130,12 +130,12 @@ int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
 }
 
 int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
-  char *ptr = taosGetDataFromHashTable(mfh->map, (char *)(&uid), sizeof(uid));
+  char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
   if (ptr == NULL) return -1;
 
   SRecordInfo info = *(SRecordInfo *)ptr;
   // Update the hash table
-  if (taosAddToHashTable(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+  if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
     return -1;
   }
 
@@ -166,7 +166,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) {
   if (mfh == NULL) return;
   close(mfh);
 
-  taosCleanUpHashTable(mfh->map);
+  taosHashCleanup(mfh->map);
 }
 
 static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {

From e80611c18a330d09c1babe6a2a3f28870a47ee2e Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 16:22:07 +0800
Subject: [PATCH 11/17] refactor and add more code

---
 src/common/inc/dataformat.h        |  24 ++++---
 src/common/src/dataformat.c        | 103 +++++++++++++----------------
 src/vnode/tsdb/inc/tsdb.h          |   2 +-
 src/vnode/tsdb/src/tsdbMain.c      |   2 +-
 src/vnode/tsdb/tests/tsdbTests.cpp |  30 ++++++---
 5 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index 08e49dd3c5..4589a0573a 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -69,31 +69,35 @@ void      tdUpdateSchema(STSchema *pSchema);
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * +---------+---------------------------------+
- * | int32_t |                                 |
- * +---------+---------------------------------+
- * |   len   |                row              |
- * +---------+---------------------------------+
+ * +----------+---------+---------------------------------+---------------------------------+
+ * | int32_t  | int32_t |                                 |                                 |
+ * +----------+---------+---------------------------------+---------------------------------+
+ * |   len    |  flen   |           First part            |             Second part         |
+ * +----------+---------+---------------------------------+---------------------------------+
+ * plen: first part length
  * len: the length including sizeof(row) + sizeof(len)
  * row: actual row data encoding
  */
 typedef void *SDataRow;
 
-#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
+#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
 
 #define dataRowLen(r) (*(int32_t *)(r))
+#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
 #define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
+#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
 #define dataRowIdx(r, i) ((char *)(r) + i)
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
+#define dataRowAt(r, idx) ((char *)(r) + (idx))
 
-SDataRow tdNewDataRow(int32_t bytes);
+void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 int      tdMaxRowBytesFromSchema(STSchema *pSchema);
+SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema);
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
-int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol, int32_t suffixOffset);
-void     tdDataRowCpy(void *dst, SDataRow row);
-void     tdDataRowReset(SDataRow row);
+int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
+void     tdDataRowReset(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
 /* Data rows definition, the format of it is like below:
diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c
index c6fd4af284..58530c5e3d 100644
--- a/src/common/src/dataformat.c
+++ b/src/common/src/dataformat.c
@@ -14,6 +14,8 @@
  */
 #include "dataformat.h"
 
+static int tdFLenFromSchema(STSchema *pSchema);
+
 /**
  * Create a new STColumn object
  * ASSUMPTIONS: VALID PARAMETERS
@@ -157,6 +159,14 @@ void tdUpdateSchema(STSchema *pSchema) {
   }
 }
 
+/**
+ * Initialize a data row
+ */
+void tdInitDataRow(SDataRow row, STSchema *pSchema) {
+  dataRowSetFLen(row, TD_DATA_ROW_HEAD_SIZE);
+  dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + tdFLenFromSchema(pSchema));
+}
+
 /**
  * Create a data row with maximum row length bytes.
  *
@@ -167,13 +177,13 @@ void tdUpdateSchema(STSchema *pSchema) {
  * @return SDataRow object for success
  *         NULL for failure
  */
-SDataRow tdNewDataRow(int32_t bytes) {
+SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema) {
   int32_t size = sizeof(int32_t) + bytes;
 
   SDataRow row = malloc(size);
   if (row == NULL) return NULL;
 
-  dataRowSetLen(row, sizeof(int32_t));
+  tdInitDataRow(row, pSchema);
 
   return row;
 }
@@ -197,14 +207,7 @@ int tdMaxRowBytesFromSchema(STSchema *pSchema) {
   return bytes;
 }
 
-SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
-  int bytes = 0;
-  {
-    // TODO: estimiate size from schema
-  }
-
-  return tdNewDataRow(bytes);
-}
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { return tdNewDataRow(tdMaxRowBytesFromSchema(pSchema), pSchema); }
 
 /**
  * Free the SDataRow object
@@ -214,62 +217,37 @@ void tdFreeDataRow(SDataRow row) {
 }
 
 /**
- * Append a column value to a SDataRow object.
- * NOTE: THE APPLICATION SHOULD MAKE SURE VALID PARAMETERS. THE FUNCTION ASSUMES
- * THE ROW OBJECT HAS ENOUGH SPACE TO HOLD THE VALUE.
- *
- * @param row the row to append value to
- * @param value value pointer to append
- * @param pSchema schema
- * @param colIdx column index
- * 
- * @return 0 for success and -1 for failure
+ * Append a column value to the data row
  */
-// int32_t tdAppendColVal(SDataRow row, void *value, SColumn *pCol, int32_t suffixOffset) {
-//   int32_t offset;
+int tdAppendColVal(SDataRow row, void *value, STColumn *pCol) {
+  switch (colType(pCol))
+  {
+  case TSDB_DATA_TYPE_BINARY:
+  case TSDB_DATA_TYPE_NCHAR:
+    *(int32_t *)dataRowAt(row, dataRowFLen(row)) = dataRowLen(row);
+    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+    memcpy((void *)dataRowAt(row, dataRowLen(row)), value, strlen(value));
+    dataRowLen(row) += strlen(value);
+    break;
+  default:
+    memcpy(dataRowAt(row, dataRowFLen(row)), value, TYPE_BYTES[colType(pCol)]);
+    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+    break;
+  }
+}
 
-//   switch (pCol->type) {
-//     case TD_DATATYPE_BOOL:
-//     case TD_DATATYPE_TINYINT:
-//     case TD_DATATYPE_SMALLINT:
-//     case TD_DATATYPE_INT:
-//     case TD_DATATYPE_BIGINT:
-//     case TD_DATATYPE_FLOAT:
-//     case TD_DATATYPE_DOUBLE:
-//     case TD_DATATYPE_TIMESTAMP:
-//       memcpy(dataRowIdx(row, pCol->offset + sizeof(int32_t)), value, rowDataLen[pCol->type]);
-//       if (dataRowLen(row) < suffixOffset + sizeof(int32_t))
-//         dataRowSetLen(row, dataRowLen(row) + rowDataLen[pCol->type]);
-//       break;
-//     case TD_DATATYPE_VARCHAR:
-//       offset = dataRowLen(row) > suffixOffset ? dataRowLen(row) : suffixOffset;
-//       memcpy(dataRowIdx(row, pCol->offset+sizeof(int32_t)), (void *)(&offset), sizeof(offset));
-//     case TD_DATATYPE_NCHAR:
-//     case TD_DATATYPE_BINARY:
-//       break;
-//     default:
-//       return -1;
-//   }
+void tdDataRowReset(SDataRow row, STSchema *pSchema) { tdInitDataRow(row, pSchema); }
 
-//   return 0;
-// }
-
-/**
- * Copy a data row to a destination
- * ASSUMPTIONS: dst has enough room for a copy of row
- */
-void tdDataRowCpy(void *dst, SDataRow row) { memcpy(dst, row, dataRowLen(row)); }
-void tdDataRowReset(SDataRow row) { dataRowSetLen(row, sizeof(int32_t)); }
 SDataRow tdDataRowDup(SDataRow row) {
-  SDataRow trow = tdNewDataRow(dataRowLen(row));
+  SDataRow trow = malloc(dataRowLen(row));
   if (trow == NULL) return NULL;
 
   dataRowCpy(trow, row);
-  return row;
+  return trow;
 }
 
 void tdDataRowsAppendRow(SDataRows rows, SDataRow row) {
-  tdDataRowCpy((void *)((char *)rows + dataRowsLen(rows)), row);
+  dataRowCpy((void *)((char *)rows + dataRowsLen(rows)), row);
   dataRowsSetLen(rows, dataRowsLen(rows) + dataRowLen(row));
 }
 
@@ -300,4 +278,17 @@ SDataRow tdDataRowsNext(SDataRowsIter *pIter) {
   }
 
   return row;
+}
+
+/**
+ * Return the first part length of a data row for a schema
+ */
+static int tdFLenFromSchema(STSchema *pSchema) {
+  int ret = 0;
+  for (int i = 0; i < schemaNCols(pSchema); i++) {
+    STColumn *pCol = schemaColAt(pSchema, i);
+    ret += TYPE_BYTES[pCol->type];
+  }
+
+  return ret;
 }
\ No newline at end of file
diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index c41384114a..57798b6a09 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -96,7 +96,7 @@ typedef struct {
   STableId tableId;
   int32_t  padding;    // TODO just for padding here
   int32_t  sversion;   // data schema version
-  int32_t  len;        // message length
+  int32_t  len;        // data part length, not including the SSubmitBlk head
   char     data[];
 } SSubmitBlk;
 
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index 37901ba588..f3aed75e79 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -621,7 +621,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   }
 
   pNode->level = level;
-  tdDataRowCpy(SL_GET_NODE_DATA(pNode), row);
+  dataRowCpy(SL_GET_NODE_DATA(pNode), row);
 
   // Insert the skiplist node into the data
   tsdbInsertRowToTableImpl(pNode, pTable);
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 1aa85c0808..eba7df3adb 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -32,15 +32,29 @@ TEST(TsdbTest, createRepo) {
 
   tsdbCreateTable(pRepo, &tCfg);
 
-  // 3. Loop to write some simple data
-  // int         size = tdMaxRowBytesFromSchema(schema);
-  // int         nrows = 100;
-  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk+ size * nrows);
+  // // 3. Loop to write some simple data
+  // int nRows = 10;
+  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * nRows);
 
-  // {
-  //   // TODO
+  // SSubmitBlk *pBlock = pMsg->blocks;
+  // pBlock->tableId = {.uid = 987607499877672L, .tid = 0};
+  // pBlock->sversion = 0;
+  // pBlock->len = 0;
+  // int64_t start_time = 1584081000000;
+  // for (int i = 0; i < nRows; i++) {
+  //   int64_t ttime = start_time + 1000 * i;
+  //   SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
+  //   dataRowInit(row);
+
+  //   for (int j; j < schemaNCols(schema); j++) {
+  //     if (j == 0) { // Just for timestamp
+  //       tdAppendColVal(row, (void *)(&time), schemaColAt(schema, i), );
+  //     } else { // For int
+
+  //     }
+  //   }
+
+  //   pBlock->len += dataRowLen(row);
   // }
-
-  // tsdbInsertData(pRepo, pMsg);
 }
 

From a3e12271a6b700d44309a54300f5fda9fec5f5a0 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 17:29:24 +0800
Subject: [PATCH 12/17] more

---
 src/vnode/tsdb/tests/tsdbTests.cpp | 43 +++++++++++++++++-------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index eba7df3adb..91b6d417a4 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -33,28 +33,33 @@ TEST(TsdbTest, createRepo) {
   tsdbCreateTable(pRepo, &tCfg);
 
   // // 3. Loop to write some simple data
-  // int nRows = 10;
-  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * nRows);
+  int nRows = 10;
+  SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * nRows);
 
-  // SSubmitBlk *pBlock = pMsg->blocks;
-  // pBlock->tableId = {.uid = 987607499877672L, .tid = 0};
-  // pBlock->sversion = 0;
-  // pBlock->len = 0;
-  // int64_t start_time = 1584081000000;
-  // for (int i = 0; i < nRows; i++) {
-  //   int64_t ttime = start_time + 1000 * i;
-  //   SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
-  //   dataRowInit(row);
+  SSubmitBlk *pBlock = pMsg->blocks;
+  pBlock->tableId = {.uid = 987607499877672L, .tid = 0};
+  pBlock->sversion = 0;
+  pBlock->len = 0;
+  int64_t start_time = 1584081000000;
+  for (int i = 0; i < nRows; i++) {
+    int64_t ttime = start_time + 1000 * i;
+    SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
+    tdInitDataRow(row, schema);
 
-  //   for (int j; j < schemaNCols(schema); j++) {
-  //     if (j == 0) { // Just for timestamp
-  //       tdAppendColVal(row, (void *)(&time), schemaColAt(schema, i), );
-  //     } else { // For int
+    for (int j; j < schemaNCols(schema); j++) {
+      if (j == 0) { // Just for timestamp
+        tdAppendColVal(row, (void *)(&time), schemaColAt(schema, i));
+      } else { // For int
+        int val = 10;
+        tdAppendColVal(row, (void *)(&val), schemaColAt(schema, i));
+      }
 
-  //     }
-  //   }
+      pBlock->len += dataRowLen(row);
+    }
 
-  //   pBlock->len += dataRowLen(row);
-  // }
+    pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
+  }
+
+  tsdbInsertData(pRepo, pMsg);
 }
 

From bedd0eb8b7faff748427721877a1042b73f40e19 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 17:33:10 +0800
Subject: [PATCH 13/17] make compile

---
 src/dnode/src/dnodeMgmt.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 63e4a290aa..79d2de7739 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -91,9 +91,9 @@ int32_t dnodeInitMgmt() {
   SVnodeObj *pVnode = dnodeGetVnode(cfg.cfg.vgId);
   dnodeDropVnode(pVnode);
 
-  dnodeCreateVnode(&cfg);
-  SVnodeObj *pVnode = dnodeGetVnode(cfg.cfg.vgId);
-  dnodeCleanupVnodes();
+  // dnodeCreateVnode(&cfg);
+  // SVnodeObj *pVnode = dnodeGetVnode(cfg.cfg.vgId);
+  // dnodeCleanupVnodes();
 
   dnodeOpenVnodes();
   dnodeCleanupVnodes();

From 36ade4200e263a73c55e384d000cca7d789f758f Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 19:14:47 +0800
Subject: [PATCH 14/17] fix bug

---
 src/vnode/tsdb/tests/tsdbTests.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 91b6d417a4..052ad38d01 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -46,12 +46,12 @@ TEST(TsdbTest, createRepo) {
     SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
     tdInitDataRow(row, schema);
 
-    for (int j; j < schemaNCols(schema); j++) {
+    for (int j = 0; j < schemaNCols(schema); j++) {
       if (j == 0) { // Just for timestamp
-        tdAppendColVal(row, (void *)(&time), schemaColAt(schema, i));
+        tdAppendColVal(row, (void *)(&time), schemaColAt(schema, j));
       } else { // For int
         int val = 10;
-        tdAppendColVal(row, (void *)(&val), schemaColAt(schema, i));
+        tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
       }
 
       pBlock->len += dataRowLen(row);

From d084ec804baa0c4035e50e5b6af5dfd670d69946 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 20:22:20 +0800
Subject: [PATCH 15/17] refact and add code

---
 src/vnode/tsdb/inc/tsdb.h          | 11 +++++-
 src/vnode/tsdb/src/tsdbMain.c      | 55 ++++++++++++++++++++----------
 src/vnode/tsdb/tests/tsdbTests.cpp |  4 +--
 3 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h
index 8c3fadf486..267b462b91 100644
--- a/src/vnode/tsdb/inc/tsdb.h
+++ b/src/vnode/tsdb/inc/tsdb.h
@@ -101,6 +101,15 @@ typedef struct {
   char     data[];
 } SSubmitBlk;
 
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SDataRow row;
+} SSubmitBlkIter;
+
+int      tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+
 // Submit message for this TSDB
 typedef struct {
   int32_t    length;
@@ -117,7 +126,7 @@ typedef struct {
   SSubmitBlk *pBlock;
 } SSubmitMsgIter;
 
-int           tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+int         tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
 SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
 
 // the TSDB repository info
diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c
index f3aed75e79..0f70875c63 100644
--- a/src/vnode/tsdb/src/tsdbMain.c
+++ b/src/vnode/tsdb/src/tsdbMain.c
@@ -322,14 +322,15 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) {
 
 // TODO: need to return the number of data inserted
 int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
-  SSubmitBlk *pBlock = (SSubmitBlk *)pMsg->blocks;
+  SSubmitMsgIter msgIter;
 
-  // for (int i = 0; i < pMsg->numOfTables; i++) {  // Loop to deal with the submit message
-  //   if (tsdbInsertDataToTable(repo, pBlock) < 0) {
-  //     return -1;
-  //   }
-  //   pBlock = (SSubmitBlk *)(((char *)pBlock) + sizeof(SSubmitBlk) + pBlock->len);
-  // }
+  tsdbInitSubmitMsgIter(pMsg, &msgIter);
+  SSubmitBlk *pBlock;
+  while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
+    if (tsdbInsertDataToTable(repo, pBlock) < 0) {
+      return -1;
+    }
+  }
 
   return 0;
 }
@@ -415,6 +416,28 @@ void tsdbClearTableCfg(STableCfg *config) {
   if (config->tagValues) tdFreeDataRow(config->tagValues);
 }
 
+int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+  if (pBlock->len <= 0) return -1;
+  pIter->totalLen = pBlock->len;
+  pIter->len = 0;
+  pIter->row = (SDataRow)(pBlock->data);
+  return 0;
+}
+
+SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  SDataRow row = pIter->row;
+  if (row == NULL) return NULL;
+
+  pIter->len += dataRowLen(row);
+  if (pIter->len >= pIter->totalLen) {
+    pIter->row = NULL;
+  } else {
+    pIter->row = (char *)row + dataRowLen(row);
+  }
+
+  return row;
+}
+
 int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL || pIter == NULL) return -1;
 
@@ -433,11 +456,11 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
   SSubmitBlk *pBlock = pIter->pBlock;
   if (pBlock == NULL) return NULL;
 
-  pIter->len += pBlock->len;
+  pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len;
   if (pIter->len >= pIter->totalLen) {
     pIter->pBlock = NULL;
   } else {
-    pIter->pBlock = (char *)pBlock + pBlock->len;
+    pIter->pBlock = (char *)pBlock + pBlock->len + sizeof(SSubmitBlk);
   }
 
   return pBlock;
@@ -633,19 +656,15 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, pBlock->tableId);
-  if (pTable == NULL) {
-    return -1;
-  }
+  if (pTable == NULL) return -1;
 
-  SDataRows     rows = pBlock->data;
-  SDataRowsIter rDataIter, *pIter;
-  pIter = &rDataIter;
+  SSubmitBlkIter blkIter;
   SDataRow row;
 
-  tdInitSDataRowsIter(rows, pIter);
-  while ((row = tdDataRowsNext(pIter)) != NULL) {
+  tsdbInitSubmitBlkIter(pBlock, &blkIter);
+  while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
     if (tdInsertRowToTable(pRepo, row, pTable) < 0) {
-      // TODO: deal with the error here
+      return -1;
     }
   }
 
diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp
index 052ad38d01..7b09fdfcde 100644
--- a/src/vnode/tsdb/tests/tsdbTests.cpp
+++ b/src/vnode/tsdb/tests/tsdbTests.cpp
@@ -54,11 +54,11 @@ TEST(TsdbTest, createRepo) {
         tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
       }
 
-      pBlock->len += dataRowLen(row);
     }
+    pBlock->len += dataRowLen(row);
 
-    pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
   }
+  pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
 
   tsdbInsertData(pRepo, pMsg);
 }

From 1f276412056f15513f5568c6373ff81e16b28aac Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Fri, 13 Mar 2020 22:53:27 +0800
Subject: [PATCH 16/17] add more code

---
 src/vnode/tsdb/inc/tsdbFile.h |  6 +++--
 src/vnode/tsdb/src/tsdbFile.c | 48 +++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h
index dbcec49651..7b3e19d0b9 100644
--- a/src/vnode/tsdb/inc/tsdbFile.h
+++ b/src/vnode/tsdb/inc/tsdbFile.h
@@ -16,7 +16,8 @@
 #define _TD_TSDB_FILE_H_
 
 #include <stdint.h>
-// #include "tstring.h"
+
+#include "taosdef.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,7 +35,8 @@ typedef enum {
 extern const char *tsdbFileSuffix[];
 
 typedef struct {
-  int64_t fileSize;
+  int64_t size;
+  int64_t tombSize;
 } SFileInfo;
 
 typedef struct {
diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c
index 6009d160e3..9cd9bd1d18 100644
--- a/src/vnode/tsdb/src/tsdbFile.c
+++ b/src/vnode/tsdb/src/tsdbFile.c
@@ -18,6 +18,54 @@
 
 #include "tsdbFile.h"
 
+typedef struct {
+  int64_t offset;
+} SCompHeader;
+
+typedef struct {
+  int64_t uid;
+  int64_t last : 1;
+  int64_t numOfBlocks : 63;
+  int32_t delimiter;
+} SCompInfo;
+
+typedef struct {
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  int32_t numOfBlocks;
+  int32_t offset;
+} SCompIdx;
+
+typedef struct {
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  int64_t offset;
+  int32_t len;
+  int32_t sversion;
+} SCompBlock;
+
+typedef struct {
+  int64_t uid;
+
+} SBlock;
+
+typedef struct {
+  int16_t colId;
+  int16_t bytes;
+  int32_t nNullPoints;
+  int32_t type:8;
+  int32_t offset:24;
+  int32_t len;
+  // fields for pre-aggregate
+  // TODO: pre-aggregation should be seperated
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIdx;
+  int16_t minIdx;
+  char    reserverd[20]:
+} SField;
+
 const char *tsdbFileSuffix[] = {
     ".head",  // TSDB_FILE_TYPE_HEAD
     ".data",  // TSDB_FILE_TYPE_DATA

From e85c8d516237a4538f3ca2dc59e1f5f0e9c8ccf2 Mon Sep 17 00:00:00 2001
From: hzcheng <hzcheng@taosdata.com>
Date: Sat, 14 Mar 2020 09:36:49 +0800
Subject: [PATCH 17/17] make compile

---
 src/vnode/tsdb/src/tsdbFile.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c
index 9cd9bd1d18..b977a51b51 100644
--- a/src/vnode/tsdb/src/tsdbFile.c
+++ b/src/vnode/tsdb/src/tsdbFile.c
@@ -46,7 +46,6 @@ typedef struct {
 
 typedef struct {
   int64_t uid;
-
 } SBlock;
 
 typedef struct {
@@ -63,7 +62,6 @@ typedef struct {
   int64_t min;
   int16_t maxIdx;
   int16_t minIdx;
-  char    reserverd[20]:
 } SField;
 
 const char *tsdbFileSuffix[] = {