Merge remote-tracking branch 'origin/3.0' into feature/dnode3

include/common/tdataformat.h

@@ -38,7 +38,6 @@ extern "C" {
     memcpy(varDataVal(x), (str), __len);          \
   } while (0);
 
-
 #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)                         \
   do {                                                                    \
     char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \
@@ -73,7 +72,8 @@ typedef struct {
 typedef struct {
   int version;    // version
   int numOfCols;  // Number of columns appended
-  int      tlen;       // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
+  int tlen;  // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
+             // (bytes))
   uint16_t flen;  // First part length in a SDataRow after the header part
   uint16_t vlen;  // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
@@ -202,7 +202,6 @@ void     tdFreeDataRow(SDataRow row);
 void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
-
 // offset here not include dataRow header length
 static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
                                            int32_t offset) {
@@ -228,7 +227,6 @@ static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool
   return 0;
 }
 
-
 // offset here not include dataRow header length
 static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
   return tdAppendDataColVal(row, value, true, type, offset);
@@ -269,7 +267,8 @@ static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSche
   setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
 }
 
-static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src, STSchema *pSrcSchema, int srcIdx) {
+static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src,
+                                                STSchema *pSrcSchema, int srcIdx) {
   int8_t type = pDstSchema->columns[dstIdx].type;
   assert(type == pSrcSchema->columns[srcIdx].type);
   void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
@@ -319,7 +318,6 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
   }
 }
 
-
 // ----------------- Data column structure
 typedef struct SDataCol {
   int8_t          type;       // column type
@@ -454,6 +452,7 @@ typedef struct {
 #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r))
 #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r)))
 #define kvRowKey(r) tdGetKey(kvRowTKey(r))
+#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t)))
 #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r))
 
 SKVRow tdKVRowDup(SKVRow row);
@@ -654,6 +653,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) {
 
 #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
 #define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
+#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
 #define memRowSetTKey(r, k)                 \
   do {                                      \
     if (isDataRow(r)) {                     \

include/common/tmsg.h

@@ -244,11 +244,29 @@ typedef struct SSubmitBlk {
 // Submit message for this TSDB
 typedef struct SSubmitMsg {
   SMsgHead header;
+  int64_t  version;
   int32_t  length;
   int32_t  numOfBlocks;
   char     blocks[];
 } SSubmitMsg;
 
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  SMemRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  void*   pMsg;
+} SSubmitMsgIter;
+
+int     tInitSubmitMsgIter(SSubmitMsg* pMsg, SSubmitMsgIter* pIter);
+int     tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
+int     tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
+SMemRow tGetSubmitBlkNext(SSubmitBlkIter* pIter);
+
 typedef struct {
   int32_t index;  // index of failed block in submit blocks
   int32_t vnode;  // vnode index of failed block
@@ -871,6 +889,7 @@ typedef struct {
 
 typedef struct {
   char        db[TSDB_DB_FNAME_LEN];
+  int64_t     uid;
   int32_t     vgVersion;
   int32_t     vgNum;
   int8_t      hashMethod;

include/dnode/vnode/tsdb/tsdb.h

@@ -22,21 +22,45 @@
 extern "C" {
 #endif
 
+typedef struct SDataStatis {
+  int16_t colId;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+} SDataStatis;
+
+typedef struct STable {
+  int32_t   tid;
+  uint64_t  uid;
+  STSchema *pSchema;
+} STable;
+
+#define TABLE_TID(t) (t)->tid
+#define TABLE_UID(t) (t)->uid
+
 // TYPES EXPOSED
 typedef struct STsdb STsdb;
 
 typedef struct STsdbCfg {
+  int8_t   precision;
   uint64_t lruCacheSize;
-  uint32_t keep0;
+  int32_t  daysPerFile;
-  uint32_t keep1;
+  int32_t  minRowsPerFileBlock;
-  uint32_t keep2;
+  int32_t  maxRowsPerFileBlock;
+  int32_t  keep;
+  int32_t  keep1;
+  int32_t  keep2;
+  int8_t   update;
 } STsdbCfg;
 
 // STsdb
 STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF);
 void   tsdbClose(STsdb *);
 void   tsdbRemove(const char *path);
-int    tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg);
+int    tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp);
 int    tsdbPrepareCommit(STsdb *pTsdb);
 int    tsdbCommit(STsdb *pTsdb);
 

include/dnode/vnode/tsdb2/tsdb.h

@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_TSDB_H_
+#define _TD_TSDB_H_
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "common.h"
+#include "taosdef.h"
+#include "tarray.h"
+#include "tdataformat.h"
+#include "thash.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "tmsg.h"
+#include "tname.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TSDB_VERSION_MAJOR 1
+#define TSDB_VERSION_MINOR 0
+
+#define TSDB_INVALID_SUPER_TABLE_ID -1
+
+#define TSDB_STATUS_COMMIT_START 1
+#define TSDB_STATUS_COMMIT_OVER 2
+#define TSDB_STATUS_COMMIT_NOBLOCK 3  // commit no block, need to be solved
+
+// TSDB STATE DEFINITION
+#define TSDB_STATE_OK 0x0
+#define TSDB_STATE_BAD_META 0x1
+#define TSDB_STATE_BAD_DATA 0x2
+
+typedef struct SDataStatis {
+  int16_t colId;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+} SDataStatis;
+
+// --------- TSDB APPLICATION HANDLE DEFINITION
+
+// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
+typedef struct {
+  int32_t tsdbId;
+  int32_t cacheBlockSize;
+  int32_t totalBlocks;
+  int32_t daysPerFile;  // day per file sharding policy
+  int32_t keep;         // day of data to keep
+  int32_t keep1;
+  int32_t keep2;
+  int32_t lruCacheSize;
+  int32_t minRowsPerFileBlock;  // minimum rows per file block
+  int32_t maxRowsPerFileBlock;  // maximum rows per file block
+  int8_t  precision;
+  int8_t  compression;
+  int8_t  update;
+  int8_t  cacheLastRow;  // 0:no cache, 1: cache last row, 2: cache last NULL column 3: 1&2
+} STsdbCfg;
+
+#define CACHE_NO_LAST(c) ((c)->cacheLastRow == 0)
+#define CACHE_LAST_ROW(c) (((c)->cacheLastRow & 1) > 0)
+#define CACHE_LAST_NULL_COLUMN(c) (((c)->cacheLastRow & 2) > 0)
+
+// --------- TSDB REPOSITORY USAGE STATISTICS
+typedef struct {
+  int64_t totalStorage;  // total bytes occupie
+  int64_t compStorage;
+  int64_t pointsWritten;  // total data points written
+} STsdbStat;
+
+typedef struct STsdb STsdb;
+
+STsdbCfg *tsdbGetCfg(const STsdb *repo);
+
+// --------- TSDB REPOSITORY DEFINITION
+// int32_t tsdbCreateRepo(int repoid);
+// int32_t tsdbDropRepo(int repoid);
+STsdb * tsdbOpen(STsdbCfg *pCfg, STsdbAppH *pAppH);
+int     tsdbClose(STsdb *repo, int toCommit);
+int32_t tsdbConfigRepo(STsdb *repo, STsdbCfg *pCfg);
+int     tsdbGetState(STsdb *repo);
+int8_t  tsdbGetCompactState(STsdb *repo);
+// --------- TSDB TABLE DEFINITION
+typedef struct {
+  uint64_t uid;  // the unique table ID
+  int32_t  tid;  // the table ID in the repository.
+} STableId;
+
+// --------- TSDB TABLE configuration
+typedef struct {
+  ETableType type;
+  char *     name;
+  STableId   tableId;
+  int32_t    sversion;
+  char *     sname;  // super table name
+  uint64_t   superUid;
+  STSchema * schema;
+  STSchema * tagSchema;
+  SKVRow     tagValues;
+  char *     sql;
+} STableCfg;
+
+void tsdbClearTableCfg(STableCfg *config);
+
+void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type);
+char *tsdbGetTableName(void *pTable);
+
+#define TSDB_TABLEID(_table) ((STableId *)(_table))
+#define TSDB_PREV_ROW 0x1
+#define TSDB_NEXT_ROW 0x2
+
+STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
+
+int tsdbCreateTable(STsdb *repo, STableCfg *pCfg);
+int tsdbDropTable(STsdb *pRepo, STableId tableId);
+int tsdbUpdateTableTagValue(STsdb *repo, SUpdateTableTagValMsg *pMsg);
+
+uint32_t tsdbGetFileInfo(STsdb *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
+
+// the TSDB repository info
+typedef struct STsdbRepoInfo {
+  STsdbCfg tsdbCfg;
+  uint64_t version;            // version of the repository
+  int64_t  tsdbTotalDataSize;  // the original inserted data size
+  int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
+  // TODO: Other informations to add
+} STsdbRepoInfo;
+STsdbRepoInfo *tsdbGetStatus(STsdb *pRepo);
+
+// the meter information report structure
+typedef struct {
+  STableCfg tableCfg;
+  uint64_t  version;
+  int64_t   tableTotalDataSize;  // In bytes
+  int64_t   tableTotalDiskSize;  // In bytes
+} STableInfo;
+
+// -- FOR INSERT DATA
+/**
+ * Insert data to a table in a repository
+ * @param pRepo the TSDB repository handle
+ * @param pData the data to insert (will give a more specific description)
+ *
+ * @return the number of points inserted, -1 for failure and the error number is set
+ */
+int32_t tsdbInsertData(STsdb *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
+
+// -- FOR QUERY TIME SERIES DATA
+
+typedef void *TsdbQueryHandleT;  // Use void to hide implementation details
+
+#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
+#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
+#define BLOCK_LOAD_TABLE_RR_ORDER 3
+
+// query condition to build multi-table data block iterator
+typedef struct STsdbQueryCond {
+  STimeWindow  twindow;
+  int32_t      order;   // desc|asc order to iterate the data block
+  int64_t      offset;  // skip offset put down to tsdb
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} STsdbQueryCond;
+
+typedef struct STableData STableData;
+typedef struct {
+  T_REF_DECLARE()
+  SRWLatch     latch;
+  TSKEY        keyFirst;
+  TSKEY        keyLast;
+  int64_t      numOfRows;
+  int32_t      maxTables;
+  STableData **tData;
+  SList *      actList;
+  SList *      extraBuffList;
+  SList *      bufBlockList;
+  int64_t      pointsAdd;   // TODO
+  int64_t      storageAdd;  // TODO
+} SMemTable;
+
+typedef struct {
+  SMemTable *mem;
+  SMemTable *imem;
+  SMemTable  mtable;
+  SMemTable *omem;
+} SMemSnapshot;
+
+typedef struct SMemRef {
+  int32_t      ref;
+  SMemSnapshot snapshot;
+} SMemRef;
+
+#if 0
+typedef struct SFileBlockInfo {
+  int32_t numBlocksOfStep;
+} SFileBlockInfo;
+
+typedef struct {
+  void *pTable;
+  TSKEY lastKey;
+} STableKeyInfo;
+
+typedef struct {
+  uint32_t  numOfTables;
+  SArray *  pGroupList;
+  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
+} STableGroupInfo;
+
+#define TSDB_BLOCK_DIST_STEP_ROWS 16
+typedef struct {
+  uint16_t rowSize;
+  uint16_t numOfFiles;
+  uint32_t numOfTables;
+  uint64_t totalSize;
+  uint64_t totalRows;
+  int32_t  maxRows;
+  int32_t  minRows;
+  int32_t  firstSeekTimeUs;
+  uint32_t numOfRowsInMemTable;
+  uint32_t numOfSmallBlocks;
+  SArray * dataBlockInfos;
+} STableBlockDist;
+
+/**
+ * Get the data block iterator, starting from position according to the query condition
+ *
+ * @param tsdb       tsdb handle
+ * @param pCond      query condition, including time window, result set order, and basic required columns for each block
+ * @param tableInfoGroup  table object list in the form of set, grouped into different sets according to the
+ *                        group by condition
+ * @param qinfo      query info handle from query processor
+ * @return
+ */
+TsdbQueryHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
+                                  SMemRef *pRef);
+
+/**
+ * Get the last row of the given query time window for all the tables in STableGroupInfo object.
+ * Note that only one data block with only row will be returned while invoking retrieve data block function for
+ * all tables in this group.
+ *
+ * @param tsdb   tsdb handle
+ * @param pCond  query condition, including time window, result set order, and basic required columns for each block
+ * @param tableInfo  table list.
+ * @return
+ */
+TsdbQueryHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
+                                  SMemRef *pRef);
+
+TsdbQueryHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId,
+                                    SMemRef *pMemRef);
+
+bool isTsdbCacheLastRow(TsdbQueryHandleT *pQueryHandle);
+
+/**
+ * get the queried table object list
+ * @param pHandle
+ * @return
+ */
+SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
+
+/**
+ * get the group list according to table id from client
+ * @param tsdb
+ * @param pCond
+ * @param groupList
+ * @param qinfo
+ * @return
+ */
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+                                               uint64_t qId, SMemRef *pRef);
+
+/**
+ * get num of rows in mem table
+ *
+ * @param pHandle
+ * @return row size
+ */
+
+int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT *pHandle);
+
+/**
+ * move to next block if exists
+ *
+ * @param pQueryHandle
+ * @return
+ */
+bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle);
+
+/**
+ * Get current data block information
+ *
+ * @param pQueryHandle
+ * @param pBlockInfo
+ * @return
+ */
+void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo);
+
+/**
+ *
+ * Get the pre-calculated information w.r.t. current data block.
+ *
+ * In case of data block in cache, the pBlockStatis will always be NULL.
+ * If a block is not completed loaded from disk, the pBlockStatis will be NULL.
+
+ * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
+ * @return
+ */
+int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
+
+/**
+ *
+ * The query condition with primary timestamp is passed to iterator during its constructor function,
+ * the returned data block must be satisfied with the time window condition in any cases,
+ * which means the SData data block is not actually the completed disk data blocks.
+ *
+ * @param pQueryHandle      query handle
+ * @param pColumnIdList     required data columns id list
+ * @return
+ */
+SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
+
+/**
+ * Get the qualified table id for a super table according to the tag query expression.
+ * @param stableid. super table sid
+ * @param pTagCond. tag query condition
+ */
+int32_t tsdbQuerySTableByTagCond(STsdb *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
+                                 STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols);
+
+/**
+ * destroy the created table group list, which is generated by tag query
+ * @param pGroupList
+ */
+void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
+
+/**
+ * create the table group result including only one table, used to handle the normal table query
+ *
+ * @param tsdb        tsdbHandle
+ * @param uid         table uid
+ * @param pGroupInfo  the generated result
+ * @return
+ */
+int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+
+/**
+ *
+ * @param tsdb
+ * @param pTableIdList
+ * @param pGroupInfo
+ * @return
+ */
+int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
+
+/**
+ * clean up the query handle
+ * @param queryHandle
+ */
+void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
+
+void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond);
+
+void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo *groupList);
+
+int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT *queryHandle, STableBlockDist *pTableBlockInfo);
+
+// obtain queryHandle attribute
+int64_t tsdbSkipOffset(TsdbQueryHandleT queryHandle);
+
+/**
+ * get the statistics of repo usage
+ * @param repo. point to the tsdbrepo
+ * @param totalPoints. total data point written
+ * @param totalStorage. total bytes took by the tsdb
+ * @param compStorage. total bytes took by the tsdb after compressed
+ */
+void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
+
+int  tsdbInitCommitQueue();
+void tsdbDestroyCommitQueue();
+int  tsdbSyncCommit(STsdb *repo);
+void tsdbIncCommitRef(int vgId);
+void tsdbDecCommitRef(int vgId);
+void tsdbSwitchTable(TsdbQueryHandleT pQueryHandle);
+
+// For TSDB file sync
+int tsdbSyncSend(void *pRepo, SOCKET socketFd);
+int tsdbSyncRecv(void *pRepo, SOCKET socketFd);
+
+// // For TSDB Compact
+// int tsdbCompact(STsdb *pRepo);
+
+// For TSDB Health Monitor
+
+// // no problem return true
+// bool tsdbNoProblem(STsdb *pRepo);
+// // unit of walSize: MB
+// int tsdbCheckWal(STsdb *pRepo, uint32_t walSize);
+
+// // for json tag
+// void *getJsonTagValueElment(void *data, char *key, int32_t keyLen, char *out, int16_t bytes);
+// void  getJsonTagValueAll(void *data, void *dst, int16_t bytes);
+// char *parseTagDatatoJson(void *p);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // _TD_TSDB_H_

include/libs/catalog/catalog.h

@@ -48,8 +48,22 @@ typedef struct SMetaData {
 typedef struct SCatalogCfg {
   uint32_t maxTblCacheNum;
   uint32_t maxDBCacheNum;
+  uint32_t dbRentSec;
+  uint32_t stableRentSec;
 } SCatalogCfg;
 
+typedef struct SSTableMetaVersion {
+  uint64_t suid;
+  int16_t  sversion;
+  int16_t  tversion;  
+} SSTableMetaVersion;
+
+typedef struct SDbVgVersion {
+  int64_t dbId;
+  int32_t vgVersion;
+} SDbVgVersion;
+
+
 int32_t catalogInit(SCatalogCfg *cfg);
 
 /**
@@ -60,19 +74,27 @@ int32_t catalogInit(SCatalogCfg *cfg);
  */
 int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle);
 
+/**
+ * Free a cluster's all catalog info, usually it's not necessary, until the application is closing. 
+ * no current or future usage should be guaranteed by application
+ * @param pCatalog (input, NO more usage)
+ * @return error code
+ */
+void catalogFreeHandle(struct SCatalog* pCatalog);
+
 int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
 
 /**
  * Get a DB's all vgroup info.
  * @param pCatalog (input, got with catalogGetHandle)
- * @param pRpc (input, rpc object)
+ * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
  * @param pDBName (input, full db name)
  * @param forceUpdate (input, force update db vgroup info from mnode) 
  * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
  * @return error code
  */
-int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
+int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
 
 int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
 
@@ -87,15 +109,28 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB
  */
 int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
 
+/**
+ * Get a super table's meta data. 
+ * @param pCatalog (input, got with catalogGetHandle)
+ * @param pTransporter (input, rpc object)
+ * @param pMgmtEps (input, mnode EPs)
+ * @param pTableName (input, table name, NOT including db name)
+ * @param pTableMeta(output, table meta data, NEED to free it by calller)
+ * @return error code
+ */
+int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
+
+
 /**
  * Force renew a table's local cached meta data. 
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
  * @param pTableName (input, table name, NOT including db name)
+ * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) 
  * @return error code
  */
-int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName);
+  int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable);
 
 /**
  * Force renew a table's local cached meta data and get the new one. 
@@ -104,21 +139,23 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, co
  * @param pMgmtEps (input, mnode EPs)
  * @param pTableName (input, table name, NOT including db name)
  * @param pTableMeta(output, table meta data, NEED to free it by calller) 
+ * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) 
  * @return error code
  */
-int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
+  int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable);
+
 
 
 /**
  * Get a table's actual vgroup, for stable it's all possible vgroup list.
  * @param pCatalog (input, got with catalogGetHandle)
- * @param pRpc (input, rpc object)
+ * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
  * @param pTableName (input, table name, NOT including db name)
  * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
  * @return error code
  */
-int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList);
+int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList);
 
 /**
  * Get a table's vgroup from its name's hash value.
@@ -135,17 +172,20 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter
 /**
  * Get all meta data required in pReq.
  * @param pCatalog (input, got with catalogGetHandle)
- * @param pRpc (input, rpc object)
+ * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
  * @param pReq (input, reqest info)
  * @param pRsp (output, response data)
  * @return error code 
  */
-int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
+int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
 
-int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList);
+int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
 
+int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);
+
+int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
 
 
 /**

include/libs/parser/parser.h

@@ -24,7 +24,6 @@ extern "C" {
 
 typedef struct SParseContext {
   SParseBasicCtx   ctx;
-  int8_t           schemaAttached; // denote if submit block is built with table schema or not
   const char      *pSql;           // sql string
   size_t           sqlLen;         // length of the sql string
   char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
@@ -41,8 +40,17 @@ typedef struct SParseContext {
  */
 int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
 
-bool qIsDdlQuery(const SQueryNode* pQuery);
+/**
+ * Return true if it is a ddl/dcl sql statement
+ * @param pQuery
+ * @return
+ */
+bool qIsDdlQuery(const SQueryNode* pQueryNode);
 
+/**
+ * Destroy logic query plan
+ * @param pQueryNode
+ */
 void qDestroyQuery(SQueryNode* pQueryNode);
 
 /**

include/libs/planner/planner.h

@@ -141,6 +141,10 @@ struct SQueryNode;
 
  /**
   * Create the physical plan for the query, according to the AST.
+  * @param pQueryInfo
+  * @param pDag
+  * @param requestId
+  * @return
   */
 int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId);
 

include/libs/qcom/query.h

@@ -76,6 +76,7 @@ typedef struct STableMeta {
 
 typedef struct SDBVgroupInfo {
   SRWLatch  lock;
+  int64_t   dbId;
   int32_t   vgVersion;  
   int8_t    hashMethod;
   SHashObj *vgInfo;  //key:vgId, value:SVgroupInfo
@@ -86,8 +87,15 @@ typedef struct SUseDbOutput {
   SDBVgroupInfo dbVgroup;
 } SUseDbOutput;
 
+enum {
+  META_TYPE_NON_TABLE = 1,
+  META_TYPE_CTABLE,
+  META_TYPE_TABLE,
+  META_TYPE_BOTH_TABLE,
+};
+
 typedef struct STableMetaOutput {
-  int32_t     metaNum;
+  int32_t     metaType;
   char        ctbFname[TSDB_TABLE_FNAME_LEN];
   char        tbFname[TSDB_TABLE_FNAME_LEN];
   SCTableMeta ctbMeta;
@@ -149,6 +157,11 @@ void initQueryModuleMsgHandle();
 extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
 extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
 
+#define SET_META_TYPE_NONE(t) (t) = META_TYPE_NON_TABLE
+#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
+#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
+#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
+
 #define qFatal(...)  do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
 #define qError(...)  do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
 #define qWarn(...)   do { if (qDebugFlag & DEBUG_WARN)  { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }}  while(0)

include/libs/scheduler/scheduler.h

@@ -75,6 +75,12 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void
  */
 int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob);
 
+/**
+ * Fetch query result from the remote query executor
+ * @param pJob
+ * @param data
+ * @return
+ */
 int32_t scheduleFetchRows(void *pJob, void **data);
 
 
@@ -85,6 +91,10 @@ int32_t scheduleFetchRows(void *pJob, void **data);
  */
 int32_t scheduleCancelJob(void *pJob);
 
+/**
+ * Free the query job
+ * @param pJob
+ */
 void scheduleFreeJob(void *pJob);
 
 void schedulerDestroy(void);

include/libs/tfs/tfs.h

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFS_H
+#define TD_TFS_H
+
+#include "tglobal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int level;
+  int id;
+} SDiskID;
+
+#define TFS_UNDECIDED_LEVEL -1
+#define TFS_UNDECIDED_ID -1
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+#define TFS_MIN_LEVEL 0
+#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1)
+
+// FS APIs ====================================
+typedef struct {
+  int64_t tsize;
+  int64_t used;
+  int64_t avail;
+} SFSMeta;
+
+typedef struct {
+  int64_t size;
+  int64_t used;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+
+int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+void tfsDestroy();
+void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
+void tfsGetMeta(SFSMeta *pMeta);
+void tfsAllocDisk(int expLevel, int *level, int *id);
+
+const char *TFS_PRIMARY_PATH();
+const char *TFS_DISK_PATH(int level, int id);
+
+// TFILE APIs ====================================
+typedef struct {
+  int  level;
+  int  id;
+  char rname[TSDB_FILENAME_LEN];  // REL name
+  char aname[TSDB_FILENAME_LEN];  // ABS name
+} TFILE;
+
+#define TFILE_LEVEL(pf) ((pf)->level)
+#define TFILE_ID(pf) ((pf)->id)
+#define TFILE_NAME(pf) ((pf)->aname)
+#define TFILE_REL_NAME(pf) ((pf)->rname)
+
+#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
+#define tfsclose(fd) close(fd)
+#define tfsremove(pf) remove(TFILE_NAME(pf))
+#define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
+
+void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
+int   tfsEncodeFile(void **buf, TFILE *pf);
+void *tfsDecodeFile(void *buf, TFILE *pf);
+void  tfsbasename(const TFILE *pf, char *dest);
+void  tfsdirname(const TFILE *pf, char *dest);
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id);
+int tfsMkdirRecurAt(const char *rname, int level, int id);
+int tfsMkdir(const char *rname);
+int tfsRmdir(const char *rname);
+int tfsRename(char *orname, char *nrname);
+
+typedef struct TDIR TDIR;
+
+TDIR *       tfsOpendir(const char *rname);
+const TFILE *tfsReaddir(TDIR *tdir);
+void         tfsClosedir(TDIR *tdir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/os/os.h

@@ -24,10 +24,13 @@ extern "C" {
 #include <ctype.h>
 #include <dirent.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <float.h>
 #include <inttypes.h>
+#include <limits.h>
 #include <locale.h>
 #include <math.h>
+#include <regex.h>
 #include <sched.h>
 #include <setjmp.h>
 #include <signal.h>
@@ -44,6 +47,8 @@ extern "C" {
 #include <unistd.h>
 #include <wchar.h>
 #include <wctype.h>
+#include <wordexp.h>
+#include <libgen.h>
 
 #include <sys/mman.h>
 

include/os/osDef.h

@@ -73,6 +73,12 @@ extern "C" {
   
 #endif
 
+#ifndef WINDOWS
+  #ifndef O_BINARY
+    #define O_BINARY 0
+  #endif
+#endif
+
 #define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
 #define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
 

include/util/thash.h

@@ -124,6 +124,9 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
  */
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
 
+int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
+
+
 /**
  * return the payload data with the specified key
  *

include/util/tlog.h

@@ -44,7 +44,6 @@ extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
-extern int32_t ctgDebugFlag;
 
 #define DEBUG_FATAL 1U
 #define DEBUG_ERROR DEBUG_FATAL

source/client/inc/clientInt.h

@@ -62,6 +62,7 @@ typedef struct SAppInstInfo {
   SList            *pConnList;  // STscObj linked list
   int64_t           clusterId;
   void             *pTransporter;
+  SHeartBeatInfo hb;
 } SAppInstInfo;
 
 typedef struct SAppInfo {
@@ -70,7 +71,7 @@ typedef struct SAppInfo {
   char          *ep;
   int32_t        pid;
   int32_t        numOfThreads;
-  SHeartBeatInfo hb;
+
   SHashObj      *pInstMap;
 } SAppInfo;
 

source/client/src/clientImpl.c

@@ -181,7 +181,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
     if (pDcl->msgType == TDMT_VND_SHOW_TABLES) {
       SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
       if (pShowReqInfo->pArray == NULL) {
-        pShowReqInfo->currentIndex = 0;
+        pShowReqInfo->currentIndex = 0;  // set the first vnode/ then iterate the next vnode
         pShowReqInfo->pArray = pDcl->pExtension;
       }
     }

source/client/src/clientMain.c

@@ -1,11 +1,12 @@
 #include "os.h"
+#include "tref.h"
+#include "trpc.h"
 #include "clientInt.h"
 #include "clientLog.h"
 #include "query.h"
 #include "tmsg.h"
 #include "tglobal.h"
-#include "tref.h"
+#include "catalog.h"
-#include "trpc.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
@@ -46,6 +47,7 @@ void taos_cleanup(void) {
   taosCloseRef(id);
 
   rpcCleanup();
+  catalogDestroy();
   taosCloseLog();
 
   tscInfo("all local resources released");

source/client/test/clientTests.cpp

@@ -48,7 +48,7 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) { taos_init(); }
-
+#if 0
 TEST(testCase, connect_Test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
@@ -485,6 +485,22 @@ TEST(testCase, drop_stable_Test) {
   taos_close(pConn);
 }
 
+TEST(testCase, generated_request_id_test) {
+  SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+
+  for (int32_t i = 0; i < 50000; ++i) {
+    uint64_t v = generateRequestId();
+    void*    result = taosHashGet(phash, &v, sizeof(v));
+    if (result != nullptr) {
+      printf("0x%lx, index:%d\n", v, i);
+    }
+    assert(result == nullptr);
+    taosHashPut(phash, &v, sizeof(v), NULL, 0);
+  }
+
+  taosHashCleanup(phash);
+}
+
 // TEST(testCase, create_topic_Test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  assert(pConn != NULL);
@@ -518,46 +534,55 @@ TEST(testCase, drop_stable_Test) {
 //  tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
 //  taos_close(pConn);
 //}
-TEST(testCase, generated_request_id_test) {
+//TEST(testCase, insert_test) {
-  SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
-
-  for (int32_t i = 0; i < 50000; ++i) {
-    uint64_t v = generateRequestId();
-    void*    result = taosHashGet(phash, &v, sizeof(v));
-    if (result != nullptr) {
-      printf("0x%lx, index:%d\n", v, i);
-    }
-    assert(result == nullptr);
-    taosHashPut(phash, &v, sizeof(v), NULL, 0);
-  }
-
-  taosHashCleanup(phash);
-}
-
-// TEST(testCase, projection_query_tables) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  ASSERT_EQ(pConn, nullptr);
 //
 //  TAOS_RES* pRes = taos_query(pConn, "use abc1");
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "select * from t_2");
+//  pRes = taos_query(pConn, "insert into t_2 values(now, 1)");
 //  if (taos_errno(pRes) != 0) {
 //    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
 //    taos_free_result(pRes);
 //    ASSERT_TRUE(false);
 //  }
 //
-//  TAOS_ROW pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
-//  }
-//
 //  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
+#endif
+
+TEST(testCase, projection_query_tables) {
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(pConn, nullptr);
+
+  TAOS_RES* pRes = taos_query(pConn, "use test1");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to use db, reason:%s", taos_errstr(pRes));
+    taos_free_result(pRes);
+    return;
+  }
+
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "select * from tm0");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    ASSERT_TRUE(false);
+  }
+
+  TAOS_ROW    pRow = NULL;
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  int32_t     numOfFields = taos_num_fields(pRes);
+
+  char str[512] = {0};
+  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    printf("%s\n", str);
+  }
+
+  taos_free_result(pRes);
+  taos_close(pConn);
+}

source/common/src/tglobal.c

@@ -927,16 +927,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "ctgDebugFlag";
-  cfg.ptr = &ctgDebugFlag;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
-  cfg.maxValue = 255;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   cfg.option = "enableRecordSql";
   cfg.ptr = &tsTscEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT8;

source/common/src/tmsg.c

@@ -27,6 +27,64 @@
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
+int tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+  if (pMsg == NULL) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    return -1;
+  }
+
+  pIter->totalLen = pMsg->length;
+  pIter->len = 0;
+  pIter->pMsg = pMsg;
+  if (pMsg->length <= sizeof(SSubmitMsg)) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    return -1;
+  }
+
+  return 0;
+}
+
+int tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
+  if (pIter->len == 0) {
+    pIter->len += sizeof(SSubmitMsg);
+  } else {
+    SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+    pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
+  }
+
+  if (pIter->len > pIter->totalLen) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    *pPBlock = NULL;
+    return -1;
+  }
+
+  *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+
+  return 0;
+}
+
+int tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+  if (pBlock->dataLen <= 0) return -1;
+  pIter->totalLen = pBlock->dataLen;
+  pIter->len = 0;
+  pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen);
+  return 0;
+}
+
+SMemRow tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  SMemRow row = pIter->row;
+
+  if (pIter->len >= pIter->totalLen) {
+    return NULL;
+  } else {
+    pIter->len += memRowTLen(row);
+    if (pIter->len < pIter->totalLen) {
+      pIter->row = POINTER_SHIFT(row, memRowTLen(row));
+    }
+    return row;
+  }
+}
+
 int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
   int tlen = 0;
   tlen += taosEncodeSClientHbKey(buf, &pReq->connKey);

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -554,7 +554,7 @@ static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) {
   pCfg->ttl = 4;
   pCfg->keep = pCreate->daysToKeep0;
   pCfg->isWeak = true;
-  pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep = pCreate->daysToKeep0;
   pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
   pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;

source/dnode/mgmt/impl/test/stb/stb.cpp

@@ -137,9 +137,9 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     pRsp->numOfColumns = htonl(pRsp->numOfColumns);
     pRsp->sversion = htonl(pRsp->sversion);
     pRsp->tversion = htonl(pRsp->tversion);
-    pRsp->suid = htobe64(pRsp->suid);
+    pRsp->suid = be64toh(pRsp->suid);
-    pRsp->tuid = htobe64(pRsp->tuid);
+    pRsp->tuid = be64toh(pRsp->tuid);
-    pRsp->vgId = htobe64(pRsp->vgId);
+    pRsp->vgId = be64toh(pRsp->vgId);
     for (int32_t i = 0; i < pRsp->numOfTags + pRsp->numOfColumns; ++i) {
       SSchema* pSchema = &pRsp->pSchema[i];
       pSchema->colId = htonl(pSchema->colId);
@@ -156,7 +156,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     EXPECT_EQ(pRsp->sversion, 1);
     EXPECT_EQ(pRsp->tversion, 0);
     EXPECT_GT(pRsp->suid, 0);
-    EXPECT_EQ(pRsp->tuid, 0);
+    EXPECT_GT(pRsp->tuid, 0);
     EXPECT_EQ(pRsp->vgId, 0);
 
     {

source/dnode/mnode/impl/src/mndDb.c

@@ -917,6 +917,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
   }
 
   memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN);
+  pRsp->uid = htobe64(pDb->uid);
   pRsp->vgVersion = htonl(pDb->vgVersion);
   pRsp->vgNum = htonl(vindex);
   pRsp->hashMethod = pDb->hashMethod;

source/dnode/mnode/impl/src/mndStb.c

@@ -769,7 +769,8 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
   pMeta->tableType = TSDB_SUPER_TABLE;
   pMeta->update = pDb->cfg.update;
   pMeta->sversion = htonl(pStb->version);
-  pMeta->suid = htonl(pStb->uid);
+  pMeta->suid = htobe64(pStb->uid);
+  pMeta->tuid = htobe64(pStb->uid);
 
   for (int32_t i = 0; i < totalCols; ++i) {
     SSchema *pSchema = &pMeta->pSchema[i];

source/dnode/mnode/impl/src/mndSync.c

@@ -91,6 +91,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
     if (sdbWriteFile(pSdb) != 0) {
       goto WAL_RESTORE_OVER;
     }
+  }
 
   if (walBeginSnapshot(pWal, sdbVer) < 0) {
     goto WAL_RESTORE_OVER;
@@ -99,7 +100,6 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
   if (walEndSnapshot(pWal) < 0) {
     goto WAL_RESTORE_OVER;
   }
-  }
 
   code = 0;
 

source/dnode/vnode/impl/inc/vnodeCommit.h

@@ -23,8 +23,8 @@ extern "C" {
 #endif
 
 #define vnodeShouldCommit vnodeBufPoolIsFull
+int vnodeSyncCommit(SVnode *pVnode);
 int vnodeAsyncCommit(SVnode *pVnode);
-int vnodeCommit(void *arg);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/src/vnodeCommit.c

@@ -17,8 +17,12 @@
 
 static int  vnodeStartCommit(SVnode *pVnode);
 static int  vnodeEndCommit(SVnode *pVnode);
+static int  vnodeCommit(void *arg);
+static void vnodeWaitCommit(SVnode *pVnode);
 
 int vnodeAsyncCommit(SVnode *pVnode) {
+  vnodeWaitCommit(pVnode);
+
   vnodeBufPoolSwitch(pVnode);
   SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask));
 
@@ -33,7 +37,13 @@ int vnodeAsyncCommit(SVnode *pVnode) {
   return 0;
 }
 
-int vnodeCommit(void *arg) {
+int vnodeSyncCommit(SVnode *pVnode) {
+  vnodeAsyncCommit(pVnode);
+  vnodeWaitCommit(pVnode);
+  return 0;
+}
+
+static int vnodeCommit(void *arg) {
   SVnode *pVnode = (SVnode *)arg;
 
   metaCommit(pVnode->pMeta);
@@ -42,7 +52,6 @@ int vnodeCommit(void *arg) {
 
   vnodeBufPoolRecycle(pVnode);
   tsem_post(&(pVnode->canCommit));
-  // TODO
   return 0;
 }
 
@@ -55,3 +64,5 @@ static int vnodeEndCommit(SVnode *pVnode) {
   // TODO
   return 0;
 }
+
+static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); }

source/dnode/vnode/impl/src/vnodeMain.c

@@ -137,6 +137,7 @@ static int vnodeOpenImpl(SVnode *pVnode) {
 }
 
 static void vnodeCloseImpl(SVnode *pVnode) {
+  // vnodeSyncCommit(pVnode);
   if (pVnode) {
     vnodeCloseBufPool(pVnode);
     metaClose(pVnode->pMeta);

source/dnode/vnode/impl/src/vnodeQuery.c

@@ -105,6 +105,9 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   if (pTbCfg->type == META_CHILD_TABLE) {
     strcpy(pTbMetaMsg->stbFname, pStbCfg->name);
     pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid);
+  } else if (pTbCfg->type == META_SUPER_TABLE) {
+    strcpy(pTbMetaMsg->stbFname, pTbCfg->name);
+    pTbMetaMsg->suid = htobe64(uid);
   }
   pTbMetaMsg->numOfTags = htonl(nTagCols);
   pTbMetaMsg->numOfColumns = htonl(nCols);

source/dnode/vnode/impl/src/vnodeWrite.c

@@ -92,7 +92,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       // }
       break;
     case TDMT_VND_SUBMIT:
-      if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {
+      if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr, NULL) < 0) {
         // TODO: handle error
       }
       break;

source/dnode/vnode/tsdb/CMakeLists.txt

@@ -10,6 +10,9 @@ else(0)
         "src/tsdbMemTable.c"
         "src/tsdbOptions.c"
         "src/tsdbWrite.c"
+        "src/tsdbReadImpl.c"
+        "src/tsdbFile.c"
+        "src/tsdbFS.c"
     )
 endif(0)
 
@@ -25,4 +28,5 @@ target_link_libraries(
     PUBLIC util
     PUBLIC common
     PUBLIC tkv
+    PUBLIC tfs
 )

source/dnode/vnode/tsdb/inc/tsdbCommit.h

@@ -16,7 +16,6 @@
 #ifndef _TD_TSDB_COMMIT_H_
 #define _TD_TSDB_COMMIT_H_
 
-#if 0
 typedef struct {
   int   minFid;
   int   midFid;
@@ -30,9 +29,19 @@ typedef struct {
   int64_t  size;
 } SKVRecord;
 
+void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
+
+static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
+  if (key < 0) {
+    return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
+  } else {
+    return (int)((key / tsTickPerDay[precision] / days));
+  }
+}
+
+#if 0
 #define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
 
-void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
 int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
 void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
 void *tsdbCommitData(STsdbRepo *pRepo);

source/dnode/vnode/tsdb/inc/tsdbDef.h

@@ -17,27 +17,43 @@
 #define _TD_TSDB_DEF_H_
 
 #include "mallocator.h"
-#include "tmsg.h"
+#include "tcompression.h"
-#include "tlist.h"
+#include "tglobal.h"
 #include "thash.h"
+#include "tlist.h"
+#include "tmsg.h"
 #include "tskiplist.h"
+#include "ttime.h"
 
 #include "tsdb.h"
+#include "tsdbCommit.h"
+#include "tsdbFS.h"
+#include "tsdbFile.h"
+#include "tsdbLog.h"
 #include "tsdbMemTable.h"
+#include "tsdbMemory.h"
 #include "tsdbOptions.h"
+#include "tsdbReadImpl.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct STsdb {
+  int32_t               vgId;
   char *                path;
-  STsdbCfg              options;
+  STsdbCfg              config;
   STsdbMemTable *       mem;
   STsdbMemTable *       imem;
+  SRtn                  rtn;
   SMemAllocatorFactory *pmaf;
+  STsdbFS               fs;
 };
 
+#define REPO_ID(r) 0
+#define REPO_CFG(r) (&(r)->config)
+#define REPO_FS(r) (&(r)->fs)
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/tsdb/inc/tsdbFS.h

@@ -16,7 +16,7 @@
 #ifndef _TD_TSDB_FS_H_
 #define _TD_TSDB_FS_H_
 
-#if 0
+#include "tsdbFile.h"
 
 #define TSDB_FS_VERSION 0
 
@@ -39,8 +39,6 @@ typedef struct {
 // ==================
 typedef struct {
   STsdbFSMeta meta;  // FS meta
-  SMFile*     pmf;   // meta file pointer
-  SMFile      mf;    // meta file
   SArray *    df;    // data file array
 } SFSStatus;
 
@@ -74,19 +72,19 @@ typedef struct {
 
 STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
 void *   tsdbFreeFS(STsdbFS *pfs);
-int      tsdbOpenFS(STsdbRepo *pRepo);
+int      tsdbOpenFS(STsdb *pRepo);
-void     tsdbCloseFS(STsdbRepo *pRepo);
+void     tsdbCloseFS(STsdb *pRepo);
-void     tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd);
+void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
-int      tsdbEndFSTxn(STsdbRepo *pRepo);
+int      tsdbEndFSTxn(STsdb *pRepo);
 int      tsdbEndFSTxnWithError(STsdbFS *pfs);
 void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
-void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+// void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
 int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
 
 void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
 void       tsdbFSIterSeek(SFSIter *pIter, int fid);
 SDFileSet *tsdbFSIterNext(SFSIter *pIter);
-int        tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta);
+int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
 
 static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
   int code = pthread_rwlock_rdlock(&(pFs->lock));
@@ -115,6 +113,4 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
   return 0;
 }
 
-#endif
-
 #endif /* _TD_TSDB_FS_H_ */

source/dnode/vnode/tsdb/inc/tsdbFile.h

@@ -16,7 +16,8 @@
 #ifndef _TS_TSDB_FILE_H_
 #define _TS_TSDB_FILE_H_
 
-#if 0
+#include "tchecksum.h"
+#include "tfs.h"
 
 #define TSDB_FILE_HEAD_SIZE 512
 #define TSDB_FILE_DELIMITER 0xF00AFA0F
@@ -34,7 +35,7 @@
 #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
 #define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
 #define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
-#define TSDB_FILE_FSYNC(tf) taosFsync(TSDB_FILE_FD(tf))
+#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
 #define TSDB_FILE_STATE(tf) ((tf)->state)
 #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
 #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
@@ -42,6 +43,7 @@
 
 typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
 
+#if 0
 // =============== SMFile
 typedef struct {
   int64_t  size;
@@ -68,7 +70,7 @@ int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
 int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
 int   tsdbUpdateMFileHeader(SMFile* pMFile);
 int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
-int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
+int   tsdbScanAndTryFixMFile(STsdb* pRepo);
 int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
 void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
 
@@ -96,7 +98,7 @@ static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
 static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
   ASSERT(TSDB_FILE_OPENED(pMFile));
 
-  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
+  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pMFile), offset, whence);
   if (loffset < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -108,7 +110,7 @@ static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int wh
 static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pMFile));
 
-  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
+  int64_t nwrite = taosWriteFile(pMFile->fd, buf, nbyte);
   if (nwrite < nbyte) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -150,7 +152,7 @@ static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_
 static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pMFile));
 
-  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
+  int64_t nread = taosReadFile(pMFile->fd, buf, nbyte);
   if (nread < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -159,6 +161,8 @@ static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nby
   return nread;
 }
 
+#endif
+
 // =============== SDFile
 typedef struct {
   uint32_t magic;
@@ -210,7 +214,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
 static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
+  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
   if (loffset < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -222,7 +226,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
 static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
+  int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
   if (nwrite < nbyte) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -264,7 +268,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_
 static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
+  int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
   if (nread < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -316,7 +320,7 @@ void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
 int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
 int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
 int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
-int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
 
 static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
   for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
@@ -366,5 +370,4 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
   return true;
 }
 
-#endif
 #endif /* _TS_TSDB_FILE_H_ */

source/dnode/vnode/tsdb/inc/tsdbLog.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+#include "tlog.h"
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */

source/dnode/vnode/tsdb/inc/tsdbMemTable.h

@@ -24,9 +24,9 @@ extern "C" {
 
 typedef struct STsdbMemTable STsdbMemTable;
 
-STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF);
+STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
-void           tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable);
+void           tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
-int            tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg);
+int            tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/tsdb/inc/tsdbMemory.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMORY_H_
+#define _TD_TSDB_MEMORY_H_
+
+static void * taosTMalloc(size_t size);
+static void * taosTCalloc(size_t nmemb, size_t size);
+static void * taosTRealloc(void *ptr, size_t size);
+static void * taosTZfree(void *ptr);
+static size_t taosTSizeof(void *ptr);
+static void   taosTMemset(void *ptr, int c);
+
+static FORCE_INLINE void *taosTMalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = malloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = taosTMalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  taosTMemset(ret, 0);
+  return ret;
+}
+
+static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
+
+static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return taosTMalloc(size);
+
+  if (size <= taosTSizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  void* tptr1 = realloc(tptr, tsize);
+  if (tptr1 == NULL) return NULL;
+  tptr = tptr1;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+static FORCE_INLINE void* taosTZfree(void* ptr) {
+  if (ptr) {
+    free((void*)((char*)ptr - sizeof(size_t)));
+  }
+  return NULL;
+}
+
+
+#endif /* _TD_TSDB_MEMORY_H_ */

source/dnode/vnode/tsdb/inc/tsdbReadImpl.h

@@ -15,14 +15,14 @@
 
 #ifndef _TD_TSDB_READ_IMPL_H_
 #define _TD_TSDB_READ_IMPL_H_
-#if 0
 
+#include "os.h"
 #include "tfs.h"
 #include "tsdb.h"
-#include "os.h"
 #include "tsdbFile.h"
 #include "tskiplist.h"
-#include "tsdbMeta.h"
+#include "tsdbMemory.h"
+#include "common.h"
 
 typedef struct SReadH SReadH;
 
@@ -91,7 +91,7 @@ typedef struct {
 } SBlockData;
 
 struct SReadH {
-  STsdbRepo * pRepo;
+  STsdb *     pRepo;
   SDFileSet   rSet;     // FSET to read
   SArray *    aBlkIdx;  // SBlockIdx array
   STable *    pTable;   // table to read
@@ -116,7 +116,7 @@ struct SReadH {
 
 #define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))
 
-int   tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo);
+int   tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
 void  tsdbDestroyReadH(SReadH *pReadh);
 int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
 void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
@@ -151,6 +151,4 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
   return 0;
 }
 
-#endif
-
 #endif /*_TD_TSDB_READ_IMPL_H_*/

source/dnode/vnode/tsdb/src/tsdbCommit.c

@@ -32,6 +32,23 @@ int tsdbCommit(STsdb *pTsdb) {
   return 0;
 }
 
+void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+  TSKEY     minKey, midKey, maxKey, now;
+
+  now = taosGetTimestamp(pCfg->precision);
+  minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision];
+  midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
+  maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision];
+
+  pRtn->minKey = minKey;
+  pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
+  pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
+  pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
+  tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey,
+            pRtn->minFid, pRtn->midFid, pRtn->maxFid);
+}
+
 #if 0
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
@@ -420,23 +437,6 @@ void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) {
   return buf;
 }
 
-void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) {
-  STsdbCfg *pCfg = REPO_CFG(pRepo);
-  TSKEY     minKey, midKey, maxKey, now;
-
-  now = taosGetTimestamp(pCfg->precision);
-  minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision];
-  midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
-  maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision];
-
-  pRtn->minKey = minKey;
-  pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
-  pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
-  pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
-  tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey,
-            pRtn->minFid, pRtn->midFid, pRtn->maxFid);
-}
-
 static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) {
   char      buf[64] = "\0";
   void *    pBuf = buf;

source/dnode/vnode/tsdb/src/tsdbFS.c

@@ -13,9 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "tsdbint.h"
 #include <regex.h>
+#include "os.h"
+#include "tsdbDef.h"
 
 typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
 static const char *tsdbTxnFname[] = {"current.t", "current"};
@@ -26,16 +26,24 @@ static void tsdbResetFSStatus(SFSStatus *pStatus);
 static int  tsdbSaveFSStatus(SFSStatus *pStatus, int vid);
 static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo);
 static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]);
-static int  tsdbOpenFSFromCurrent(STsdbRepo *pRepo);
+static int  tsdbOpenFSFromCurrent(STsdb *pRepo);
-static int  tsdbScanAndTryFixFS(STsdbRepo *pRepo);
+static int  tsdbScanAndTryFixFS(STsdb *pRepo);
-static int  tsdbScanRootDir(STsdbRepo *pRepo);
+static int  tsdbScanRootDir(STsdb *pRepo);
-static int  tsdbScanDataDir(STsdbRepo *pRepo);
+static int  tsdbScanDataDir(STsdb *pRepo);
 static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf);
-static int  tsdbRestoreCurrent(STsdbRepo *pRepo);
+static int  tsdbRestoreCurrent(STsdb *pRepo);
 static int  tsdbComparTFILE(const void *arg1, const void *arg2);
-static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired);
+static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
-static int  tsdbProcessExpiredFS(STsdbRepo *pRepo);
+// static int  tsdbProcessExpiredFS(STsdb *pRepo);
-static int  tsdbCreateMeta(STsdbRepo *pRepo);
+// static int  tsdbCreateMeta(STsdb *pRepo);
+
+static void tsdbGetRootDir(int repoid, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid);
+}
+
+static void tsdbGetDataDir(int repoid, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid);
+}
 
 // For backward compatibility
 // ================== CURRENT file header info
@@ -104,11 +112,11 @@ static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) {
 }
 
 static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) {
-  ASSERT(pStatus->pmf);
+  // ASSERT(pStatus->pmf);
 
   int tlen = 0;
 
-  tlen += tsdbEncodeSMFile(buf, pStatus->pmf);
+  // tlen += tsdbEncodeSMFile(buf, pStatus->pmf);
   tlen += tsdbEncodeDFileSetArray(buf, pStatus->df);
 
   return tlen;
@@ -117,9 +125,9 @@ static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) {
 static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) {
   tsdbResetFSStatus(pStatus);
 
-  pStatus->pmf = &(pStatus->mf);
+  // pStatus->pmf = &(pStatus->mf);
 
-  buf = tsdbDecodeSMFile(buf, pStatus->pmf);
+  // buf = tsdbDecodeSMFile(buf, pStatus->pmf);
   buf = tsdbDecodeDFileSetArray(buf, pStatus->df);
 
   return buf;
@@ -132,7 +140,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) {
     return NULL;
   }
 
-  TSDB_FILE_SET_CLOSED(&(pStatus->mf));
+  // TSDB_FILE_SET_CLOSED(&(pStatus->mf));
 
   pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet));
   if (pStatus->df == NULL) {
@@ -158,18 +166,18 @@ static void tsdbResetFSStatus(SFSStatus *pStatus) {
     return;
   }
 
-  TSDB_FILE_SET_CLOSED(&(pStatus->mf));
+  // TSDB_FILE_SET_CLOSED(&(pStatus->mf));
 
-  pStatus->pmf = NULL;
+  // pStatus->pmf = NULL;
   taosArrayClear(pStatus->df);
 }
 
-static void tsdbSetStatusMFile(SFSStatus *pStatus, const SMFile *pMFile) {
+// static void tsdbSetStatusMFile(SFSStatus *pStatus, const SMFile *pMFile) {
-  ASSERT(pStatus->pmf == NULL);
+//   ASSERT(pStatus->pmf == NULL);
 
-  pStatus->pmf = &(pStatus->mf);
+//   pStatus->pmf = &(pStatus->mf);
-  tsdbInitMFileEx(pStatus->pmf, (SMFile *)pMFile);
+//   tsdbInitMFileEx(pStatus->pmf, (SMFile *)pMFile);
-}
+// }
 
 static int tsdbAddDFileSetToStatus(SFSStatus *pStatus, const SDFileSet *pSet) {
   if (taosArrayPush(pStatus->df, (void *)pSet) == NULL) {
@@ -240,63 +248,63 @@ void *tsdbFreeFS(STsdbFS *pfs) {
   return NULL;
 }
 
-static int tsdbProcessExpiredFS(STsdbRepo *pRepo) {
+// static int tsdbProcessExpiredFS(STsdb *pRepo) {
-  tsdbStartFSTxn(pRepo, 0, 0);
+//   tsdbStartFSTxn(pRepo, 0, 0);
-  if (tsdbCreateMeta(pRepo) < 0) {
+//   // if (tsdbCreateMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+//   //   tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+//   //   return -1;
-  }
+//   // }
 
-  if (tsdbApplyRtn(pRepo) < 0) {
+//   if (tsdbApplyRtn(pRepo) < 0) {
-    tsdbEndFSTxnWithError(REPO_FS(pRepo));
+//     tsdbEndFSTxnWithError(REPO_FS(pRepo));
-    tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
+//     tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+//     return -1;
-  }
+//   }
-  if (tsdbEndFSTxn(pRepo) < 0) {
+//   if (tsdbEndFSTxn(pRepo) < 0) {
-    tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
+//     tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+//     return -1;
-  }
+//   }
-  return 0;
+//   return 0;
-}
+// }
 
-static int tsdbCreateMeta(STsdbRepo *pRepo) {
+// static int tsdbCreateMeta(STsdb *pRepo) {
-  STsdbFS *pfs = REPO_FS(pRepo);
+//   STsdbFS *pfs = REPO_FS(pRepo);
-  SMFile * pOMFile = pfs->cstatus->pmf;
+//   SMFile * pOMFile = pfs->cstatus->pmf;
-  SMFile   mf;
+//   SMFile   mf;
-  SDiskID  did;
+//   SDiskID  did;
 
-  if (pOMFile != NULL) {
+//   if (pOMFile != NULL) {
-    // keep the old meta file
+//     // keep the old meta file
-    tsdbUpdateMFile(pfs, pOMFile);
+//     tsdbUpdateMFile(pfs, pOMFile);
-    return 0;
+//     return 0;
-  }
+//   }
 
-  // Create a new meta file
+//   // Create a new meta file
-  did.level = TFS_PRIMARY_LEVEL;
+//   did.level = TFS_PRIMARY_LEVEL;
-  did.id = TFS_PRIMARY_ID;
+//   did.id = TFS_PRIMARY_ID;
-  tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
+//   tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
 
-  if (tsdbCreateMFile(&mf, true) < 0) {
+//   if (tsdbCreateMFile(&mf, true) < 0) {
-    tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
+//     tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+//     return -1;
-  }
+//   }
 
-  tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
+//   tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
 
-  if (tsdbUpdateMFileHeader(&mf) < 0) {
+//   if (tsdbUpdateMFileHeader(&mf) < 0) {
-    tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
+//     tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
-    tsdbApplyMFileChange(&mf, pOMFile);
+//     tsdbApplyMFileChange(&mf, pOMFile);
-    return -1;
+//     return -1;
-  }
+//   }
 
-  TSDB_FILE_FSYNC(&mf);
+//   TSDB_FILE_FSYNC(&mf);
-  tsdbCloseMFile(&mf);
+//   tsdbCloseMFile(&mf);
-  tsdbUpdateMFile(pfs, &mf);
+//   tsdbUpdateMFile(pfs, &mf);
 
-  return 0;
+//   return 0;
-}
+// }
 
-int tsdbOpenFS(STsdbRepo *pRepo) {
+int tsdbOpenFS(STsdb *pRepo) {
   STsdbFS *pfs = REPO_FS(pRepo);
   char     current[TSDB_FILENAME_LEN] = "\0";
   int      nExpired = 0;
@@ -313,9 +321,9 @@ int tsdbOpenFS(STsdbRepo *pRepo) {
     }
 
     tsdbScanAndTryFixDFilesHeader(pRepo, &nExpired);
-    if (nExpired > 0) {
+    // if (nExpired > 0) {
-      tsdbProcessExpiredFS(pRepo);
+    //   tsdbProcessExpiredFS(pRepo);
-    }
+    // }
   } else {
     // should skip expired fileset inside of the function
     if (tsdbRestoreCurrent(pRepo) < 0) {
@@ -329,39 +337,39 @@ int tsdbOpenFS(STsdbRepo *pRepo) {
     return -1;
   }
 
-  // Load meta cache if has meta file
+  // // Load meta cache if has meta file
-  if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) {
+  // if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) {
-    tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno));
+  //   tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+  //   return -1;
-  }
+  // }
 
   return 0;
 }
 
-void tsdbCloseFS(STsdbRepo *pRepo) {
+void tsdbCloseFS(STsdb *pRepo) {
   // Do nothing
 }
 
 // Start a new transaction to modify the file system
-void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd) {
+void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) {
   STsdbFS *pfs = REPO_FS(pRepo);
   ASSERT(pfs->intxn == false);
 
   pfs->intxn = true;
   tsdbResetFSStatus(pfs->nstatus);
   pfs->nstatus->meta = pfs->cstatus->meta;
-  if (pfs->cstatus->pmf == NULL) {
+  // if (pfs->cstatus->pmf == NULL) {
   pfs->nstatus->meta.version = 0;
-  } else {
+  // } else {
-    pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1;
+  //   pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1;
-  }
+  // }
   pfs->nstatus->meta.totalPoints = pfs->cstatus->meta.totalPoints + pointsAdd;
   pfs->nstatus->meta.totalStorage = pfs->cstatus->meta.totalStorage += storageAdd;
 }
 
 void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta) { pfs->nstatus->meta = *pMeta; }
 
-int tsdbEndFSTxn(STsdbRepo *pRepo) {
+int tsdbEndFSTxn(STsdb *pRepo) {
   STsdbFS *pfs = REPO_FS(pRepo);
   ASSERT(FS_IN_TXN(pfs));
   SFSStatus *pStatus;
@@ -393,7 +401,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) {
   return 0;
 }
 
-void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile) { tsdbSetStatusMFile(pfs->nstatus, pMFile); }
+// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile) { tsdbSetStatusMFile(pfs->nstatus, pMFile); }
 
 int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); }
 
@@ -415,8 +423,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   }
 
   fsheader.version = TSDB_FS_VERSION;
-  if (pStatus->pmf == NULL) {
+  if (taosArrayGetSize(pStatus->df) == 0) {
-    ASSERT(taosArrayGetSize(pStatus->df) == 0);
     fsheader.len = 0;
   } else {
     fsheader.len = tsdbEncodeFSStatus(NULL, pStatus) + sizeof(TSCKSUM);
@@ -429,7 +436,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
 
   taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
 
-  if (taosWrite(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     close(fd);
     remove(tfname);
@@ -448,7 +455,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
     tsdbEncodeFSStatus(&ptr, pStatus);
     taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
 
-    if (taosWrite(fd, pBuf, fsheader.len) < fsheader.len) {
+    if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       close(fd);
       (void)remove(tfname);
@@ -458,7 +465,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   }
 
   // fsync, close and rename
-  if (taosFsync(fd) < 0) {
+  if (taosFsyncFile(fd) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     close(fd);
     remove(tfname);
@@ -467,7 +474,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   }
 
   (void)close(fd);
-  (void)taosRename(tfname, cfname);
+  (void)taosRenameFile(tfname, cfname);
   taosTZfree(pBuf);
 
   return 0;
@@ -484,7 +491,7 @@ static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo) {
   sizeTo = taosArrayGetSize(pTo->df);
 
   // Apply meta file change
-  (void)tsdbApplyMFileChange(pFrom->pmf, pTo->pmf);
+  // (void)tsdbApplyMFileChange(pFrom->pmf, pTo->pmf);
 
   // Apply SDFileSet change
   if (ifrom >= sizeFrom) {
@@ -642,7 +649,7 @@ static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) {
   snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]);
 }
 
-static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) {
+static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
   STsdbFS * pfs = REPO_FS(pRepo);
   int       fd = -1;
   void *    buffer = NULL;
@@ -664,7 +671,7 @@ static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) {
     goto _err;
   }
 
-  int nread = (int)taosRead(fd, buffer, TSDB_FILE_HEAD_SIZE);
+  int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE);
   if (nread < 0) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
               strerror(errno));
@@ -698,7 +705,7 @@ static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) {
       goto _err;
     }
 
-    nread = (int)taosRead(fd, buffer, fsheader.len);
+    nread = (int)taosReadFile(fd, buffer, fsheader.len);
     if (nread < 0) {
       tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -737,14 +744,14 @@ _err:
 }
 
 // Scan and try to fix incorrect files
-static int tsdbScanAndTryFixFS(STsdbRepo *pRepo) {
+static int tsdbScanAndTryFixFS(STsdb *pRepo) {
   STsdbFS *  pfs = REPO_FS(pRepo);
   SFSStatus *pStatus = pfs->cstatus;
 
-  if (tsdbScanAndTryFixMFile(pRepo) < 0) {
+  // if (tsdbScanAndTryFixMFile(pRepo) < 0) {
-    tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno));
+  //   tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+  //   return -1;
-  }
+  // }
 
   size_t size = taosArrayGetSize(pStatus->df);
 
@@ -763,143 +770,143 @@ static int tsdbScanAndTryFixFS(STsdbRepo *pRepo) {
   return 0;
 }
 
-int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
+// int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta) {
-  char      tbuf[128];
+//   char      tbuf[128];
-  STsdbFS * pfs = REPO_FS(pRepo);
+//   STsdbFS * pfs = REPO_FS(pRepo);
-  SMFile    mf;
+//   SMFile    mf;
-  SMFile *  pMFile = &mf;
+//   SMFile *  pMFile = &mf;
-  void *    pBuf = NULL;
+//   void *    pBuf = NULL;
-  SKVRecord rInfo;
+//   SKVRecord rInfo;
-  int64_t   maxBufSize = 0;
+//   int64_t   maxBufSize = 0;
-  SMFInfo   minfo;
+//   SMFInfo   minfo;
 
-  taosHashClear(pfs->metaCache);
+//   taosHashClear(pfs->metaCache);
 
-  // No meta file, just return
+//   // No meta file, just return
-  if (pfs->cstatus->pmf == NULL) return 0;
+//   if (pfs->cstatus->pmf == NULL) return 0;
 
-  mf = pfs->cstatus->mf;
+//   mf = pfs->cstatus->mf;
-  // Load cache first
+//   // Load cache first
-  if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
+//   if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
-    return -1;
+//     return -1;
-  }
+//   }
 
-  if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
+//   if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
-    tsdbCloseMFile(pMFile);
+//     tsdbCloseMFile(pMFile);
-    return -1;
+//     return -1;
-  }
+//   }
 
-  while (true) {
+//   while (true) {
-    int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
+//     int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
-    if (tsize == 0) break;
+//     if (tsize == 0) break;
 
-    if (tsize < 0) {
+//     if (tsize < 0) {
-      tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
+//       tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-      return -1;
+//       return -1;
-    }
+//     }
 
-    if (tsize < sizeof(SKVRecord)) {
+//     if (tsize < sizeof(SKVRecord)) {
-      tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
+//       tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
-                TSDB_FILE_FULL_NAME(pMFile));
+//                 TSDB_FILE_FULL_NAME(pMFile));
-      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+//       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-      tsdbCloseMFile(pMFile);
+//       tsdbCloseMFile(pMFile);
-      return -1;
+//       return -1;
-    }
+//     }
 
-    void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
+//     void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
-    ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
+//     ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
-    // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
+//     // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
 
-    if (rInfo.offset < 0) {
+//     if (rInfo.offset < 0) {
-      taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
+//       taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
-#if 0
+// #if 0
-      pStore->info.size += sizeof(SKVRecord);
+//       pStore->info.size += sizeof(SKVRecord);
-      pStore->info.nRecords--;
+//       pStore->info.nRecords--;
-      pStore->info.nDels++;
+//       pStore->info.nDels++;
-      pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
+//       pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-#endif
+// #endif
-    } else {
+//     } else {
-      ASSERT(rInfo.offset > 0 && rInfo.size > 0);
+//       ASSERT(rInfo.offset > 0 && rInfo.size > 0);
-      if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
+//       if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
-        tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
+//         tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
-                  TSDB_FILE_FULL_NAME(pMFile));
+//                   TSDB_FILE_FULL_NAME(pMFile));
-        terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+//         terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-      maxBufSize = MAX(maxBufSize, rInfo.size);
+//       maxBufSize = MAX(maxBufSize, rInfo.size);
 
-      if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
+//       if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
-        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+//         tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-                  tstrerror(terrno));
+//                   tstrerror(terrno));
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-#if 0
+// #if 0
-      pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
+//       pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
-      pStore->info.nRecords++;
+//       pStore->info.nRecords++;
-#endif
+// #endif
-    }
+//     }
-  }
+//   }
 
-  if (recoverMeta) {
+//   if (recoverMeta) {
-    pBuf = malloc((size_t)maxBufSize);
+//     pBuf = malloc((size_t)maxBufSize);
-    if (pBuf == NULL) {
+//     if (pBuf == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+//       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbCloseMFile(pMFile);
+//       tsdbCloseMFile(pMFile);
-      return -1;
+//       return -1;
-    }
+//     }
 
-    SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
+//     SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
-    while (pRecord) {
+//     while (pRecord) {
-      if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
+//       if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
-        tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+//         tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-                  tstrerror(terrno));
+//                   tstrerror(terrno));
-        tfree(pBuf);
+//         tfree(pBuf);
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-      int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
+//       int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
-      if (nread < 0) {
+//       if (nread < 0) {
-        tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+//         tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-                  tstrerror(terrno));
+//                   tstrerror(terrno));
-        tfree(pBuf);
+//         tfree(pBuf);
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-      if (nread < pRecord->size) {
+//       if (nread < pRecord->size) {
-        tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
+//         tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
-                  REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
+//                   REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
-        terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-        tfree(pBuf);
+//         tfree(pBuf);
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-      if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
+//       if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
-        tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
+//         tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
-                  tstrerror(terrno));
+//                   tstrerror(terrno));
-        tfree(pBuf);
+//         tfree(pBuf);
-        tsdbCloseMFile(pMFile);
+//         tsdbCloseMFile(pMFile);
-        return -1;
+//         return -1;
-      }
+//       }
 
-      pRecord = taosHashIterate(pfs->metaCache, pRecord);
+//       pRecord = taosHashIterate(pfs->metaCache, pRecord);
-    }
+//     }
 
-    tsdbOrgMeta(pRepo);
+//     tsdbOrgMeta(pRepo);
-  }
+//   }
 
-  tsdbCloseMFile(pMFile);
+//   tsdbCloseMFile(pMFile);
-  tfree(pBuf);
+//   tfree(pBuf);
-  return 0;
+//   return 0;
-}
+// }
 
-static int tsdbScanRootDir(STsdbRepo *pRepo) {
+static int tsdbScanRootDir(STsdb *pRepo) {
   char         rootDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
   STsdbFS *    pfs = REPO_FS(pRepo);
@@ -920,9 +927,9 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) {
       continue;
     }
 
-    if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
+    // if (/*pfs->cstatus->pmf && */ tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
-      continue;
+    //   continue;
-    }
+    // }
 
     (void)tfsremove(pf);
     tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf));
@@ -933,7 +940,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) {
   return 0;
 }
 
-static int tsdbScanDataDir(STsdbRepo *pRepo) {
+static int tsdbScanDataDir(STsdb *pRepo) {
   char         dataDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
   STsdbFS *    pfs = REPO_FS(pRepo);
@@ -977,128 +984,128 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
   return false;
 }
 
-static int tsdbRestoreMeta(STsdbRepo *pRepo) {
+// static int tsdbRestoreMeta(STsdb *pRepo) {
-  char         rootDir[TSDB_FILENAME_LEN];
+//   char         rootDir[TSDB_FILENAME_LEN];
-  char         bname[TSDB_FILENAME_LEN];
+//   char         bname[TSDB_FILENAME_LEN];
-  TDIR *       tdir = NULL;
+//   TDIR *       tdir = NULL;
-  const TFILE *pf = NULL;
+//   const TFILE *pf = NULL;
-  const char * pattern = "^meta(-ver[0-9]+)?$";
+//   const char * pattern = "^meta(-ver[0-9]+)?$";
-  regex_t      regex;
+//   regex_t      regex;
-  STsdbFS *    pfs = REPO_FS(pRepo);
+//   STsdbFS *    pfs = REPO_FS(pRepo);
 
-  regcomp(&regex, pattern, REG_EXTENDED);
+//   regcomp(&regex, pattern, REG_EXTENDED);
 
-  tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
+//   tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
 
-  tsdbGetRootDir(REPO_ID(pRepo), rootDir);
+//   tsdbGetRootDir(REPO_ID(pRepo), rootDir);
 
-  tdir = tfsOpendir(rootDir);
+//   tdir = tfsOpendir(rootDir);
-  if (tdir == NULL) {
+//   if (tdir == NULL) {
-    tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
+//     tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
-    regfree(&regex);
+//     regfree(&regex);
-    return -1;
+//     return -1;
-  }
+//   }
 
-  while ((pf = tfsReaddir(tdir))) {
+//   while ((pf = tfsReaddir(tdir))) {
-    tfsbasename(pf, bname);
+//     tfsbasename(pf, bname);
 
-    if (strcmp(bname, "data") == 0) {
+//     if (strcmp(bname, "data") == 0) {
-      // Skip the data/ directory
+//       // Skip the data/ directory
-      continue;
+//       continue;
-    }
+//     }
 
-    if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
+//     if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
-      // Skip current.t file
+//       // Skip current.t file
-      tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
+//       tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
-      (void)tfsremove(pf);
+//       (void)tfsremove(pf);
-      continue;
+//       continue;
-    }
+//     }
 
-    int code = regexec(&regex, bname, 0, NULL, 0);
+//     int code = regexec(&regex, bname, 0, NULL, 0);
-    if (code == 0) {
+//     if (code == 0) {
-      // Match
+//       // Match
-      if (pfs->cstatus->pmf != NULL) {
+//       if (pfs->cstatus->pmf != NULL) {
-        tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
+//         tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
-                  TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf));
+//                   TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf));
-        terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-        tfsClosedir(tdir);
+//         tfsClosedir(tdir);
-        regfree(&regex);
+//         regfree(&regex);
-        return -1;
+//         return -1;
-      } else {
+//       } else {
-        uint32_t _version = 0;
+//         uint32_t _version = 0;
-        if (strcmp(bname, "meta") != 0) {
+//         if (strcmp(bname, "meta") != 0) {
-          sscanf(bname, "meta-ver%" PRIu32, &_version);
+//           sscanf(bname, "meta-ver%" PRIu32, &_version);
-          pfs->cstatus->meta.version = _version;
+//           pfs->cstatus->meta.version = _version;
-        }
+//         }
 
-        pfs->cstatus->pmf = &(pfs->cstatus->mf);
+//         pfs->cstatus->pmf = &(pfs->cstatus->mf);
-        pfs->cstatus->pmf->f = *pf;
+//         pfs->cstatus->pmf->f = *pf;
-        TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
+//         TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
 
-        if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
+//         if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
-          tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-          tfsClosedir(tdir);
+//           tfsClosedir(tdir);
-          regfree(&regex);
+//           regfree(&regex);
-          return -1;
+//           return -1;
-        }
+//         }
 
-        if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
+//         if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
-          tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-          tsdbCloseMFile(pfs->cstatus->pmf);
+//           tsdbCloseMFile(pfs->cstatus->pmf);
-          tfsClosedir(tdir);
+//           tfsClosedir(tdir);
-          regfree(&regex);
+//           regfree(&regex);
-          return -1;
+//           return -1;
-        }
+//         }
 
-        if (tsdbForceKeepFile) {
+//         if (tsdbForceKeepFile) {
-          struct stat tfstat;
+//           struct stat tfstat;
 
-          // Get real file size
+//           // Get real file size
-          if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) {
+//           if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) {
-            terrno = TAOS_SYSTEM_ERROR(errno);
+//             terrno = TAOS_SYSTEM_ERROR(errno);
-            tsdbCloseMFile(pfs->cstatus->pmf);
+//             tsdbCloseMFile(pfs->cstatus->pmf);
-            tfsClosedir(tdir);
+//             tfsClosedir(tdir);
-            regfree(&regex);
+//             regfree(&regex);
-            return -1;
+//             return -1;
-          }
+//           }
 
-          if (pfs->cstatus->pmf->info.size != tfstat.st_size) {
+//           if (pfs->cstatus->pmf->info.size != tfstat.st_size) {
-            int64_t tfsize = pfs->cstatus->pmf->info.size;
+//             int64_t tfsize = pfs->cstatus->pmf->info.size;
-            pfs->cstatus->pmf->info.size = tfstat.st_size;
+//             pfs->cstatus->pmf->info.size = tfstat.st_size;
-            tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
+//             tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
-                     TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size);
+//                      TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size);
-          }
+//           }
-        }
+//         }
 
-        tsdbCloseMFile(pfs->cstatus->pmf);
+//         tsdbCloseMFile(pfs->cstatus->pmf);
-      }
+//       }
-    } else if (code == REG_NOMATCH) {
+//     } else if (code == REG_NOMATCH) {
-      // Not match
+//       // Not match
-      tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
+//       tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
-      tfsremove(pf);
+//       tfsremove(pf);
-      continue;
+//       continue;
-    } else {
+//     } else {
-      // Has other error
+//       // Has other error
-      tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
+//       tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
-      terrno = TAOS_SYSTEM_ERROR(code);
+//       terrno = TAOS_SYSTEM_ERROR(code);
-      tfsClosedir(tdir);
+//       tfsClosedir(tdir);
-      regfree(&regex);
+//       regfree(&regex);
-      return -1;
+//       return -1;
-    }
+//     }
-  }
+//   }
 
-  if (pfs->cstatus->pmf) {
+//   if (pfs->cstatus->pmf) {
-    tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
+//     tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
-  } else {
+//   } else {
-    tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
+//     tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
-  }
+//   }
 
-  tfsClosedir(tdir);
+//   tfsClosedir(tdir);
-  regfree(&regex);
+//   regfree(&regex);
-  return 0;
+//   return 0;
-}
+// }
 
-static int tsdbRestoreDFileSet(STsdbRepo *pRepo) {
+static int tsdbRestoreDFileSet(STsdb *pRepo) {
   char         dataDir[TSDB_FILENAME_LEN];
   char         bname[TSDB_FILENAME_LEN];
   TDIR *       tdir = NULL;
@@ -1222,7 +1229,8 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) {
       pDFile->f = *pf;
 
       if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) {
-        tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno));
+        tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+                  tstrerror(terrno));
         taosArrayDestroy(fArray);
         return -1;
       }
@@ -1266,12 +1274,12 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) {
   return 0;
 }
 
-static int tsdbRestoreCurrent(STsdbRepo *pRepo) {
+static int tsdbRestoreCurrent(STsdb *pRepo) {
-  // Loop to recover mfile
+  // // Loop to recover mfile
-  if (tsdbRestoreMeta(pRepo) < 0) {
+  // if (tsdbRestoreMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno));
+  //   tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
+  //   return -1;
-  }
+  // }
 
   // Loop to recover dfile set
   if (tsdbRestoreDFileSet(pRepo) < 0) {
@@ -1279,7 +1287,7 @@ static int tsdbRestoreCurrent(STsdbRepo *pRepo) {
     return -1;
   }
 
-  if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) {
+  if (tsdbSaveFSStatus(pRepo->fs.cstatus, REPO_ID(pRepo)) < 0) {
     tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno));
     return -1;
   }
@@ -1317,7 +1325,7 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) {
   }
 }
 
-static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired) {
+static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) {
   STsdbFS *  pfs = REPO_FS(pRepo);
   SFSStatus *pStatus = pfs->cstatus;
   SDFInfo    info;

source/dnode/vnode/tsdb/src/tsdbFile.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbint.h"
+#include "tsdbDef.h"
 
 static const char *TSDB_FNAME_SUFFIX[] = {
     "head",  // TSDB_FILE_HEAD
@@ -24,11 +24,12 @@ static const char *TSDB_FNAME_SUFFIX[] = {
 };
 
 static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
-static int   tsdbRollBackMFile(SMFile *pMFile);
+// static int   tsdbRollBackMFile(SMFile *pMFile);
 static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
 static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
 static int   tsdbRollBackDFile(SDFile *pDFile);
 
+#if 0
 // ============== SMFile
 void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
   char fname[TSDB_FILENAME_LEN];
@@ -185,7 +186,7 @@ int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
   return 0;
 }
 
-int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) {
+int tsdbScanAndTryFixMFile(STsdb *pRepo) {
   SMFile *    pMFile = pRepo->fs->cstatus->pmf;
   struct stat mfstat;
   SMFile      mf;
@@ -291,6 +292,8 @@ static int tsdbRollBackMFile(SMFile *pMFile) {
   return 0;
 }
 
+#endif
+
 // ============== Operations on SDFile
 void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) {
   char fname[TSDB_FILENAME_LEN];
@@ -397,7 +400,7 @@ int tsdbUpdateDFileHeader(SDFile *pDFile) {
   }
 
   void *ptr = buf;
-  taosEncodeFixedU32(&ptr, TSDB_FS_VERSION);
+  taosEncodeFixedU32(&ptr, 0);
   tsdbEncodeDFInfo(&ptr, &(pDFile->info));
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
@@ -433,7 +436,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
   return 0;
 }
 
-static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
+static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
   struct stat dfstat;
   SDFile      df;
 
@@ -442,7 +445,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
   if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
     tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
               TSDB_FILE_FULL_NAME(pDFile));
-    pRepo->state |= TSDB_STATE_BAD_DATA;
+    // pRepo->state |= TSDB_STATE_BAD_DATA;
     TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
     return 0;
   }
@@ -457,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
       return -1;
     }
 
-    if (taosFtruncate(df.fd, df.info.size) < 0) {
+    if (taosFtruncateFile(df.fd, df.info.size) < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       tsdbCloseDFile(&df);
       return -1;
@@ -474,7 +477,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
   } else if (pDFile->info.size > dfstat.st_size) {
     tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
               REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
-    pRepo->state |= TSDB_STATE_BAD_DATA;
+    // pRepo->state |= TSDB_STATE_BAD_DATA;
     TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
     return 0;
@@ -538,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
     return -1;
   }
 
-  if (taosFtruncate(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
+  if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tsdbCloseDFile(&df);
     return -1;
@@ -651,7 +654,7 @@ int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
   return 0;
 }
 
-int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) {
+int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet) {
   for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
     if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
       return -1;

source/dnode/vnode/tsdb/src/tsdbMain.c

@@ -72,7 +72,7 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorF
   }
 
   pTsdb->path = strdup(path);
-  tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg);
+  tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg);
   pTsdb->pmaf = pMAF;
 
   return pTsdb;

source/dnode/vnode/tsdb/src/tsdbMemTable.c

@@ -15,22 +15,13 @@
 
 #include "tsdbDef.h"
 
-#if 1
 typedef struct STbData {
-  TD_SLIST_NODE(STbData);
+  tb_uid_t   uid;
-  SSubmitMsg *pMsg;
-} STbData;
-#else
-typedef struct STbData {
-  TD_SLIST_NODE(STbData);
-  uint64_t   uid;  // TODO: change here as tb_uid_t
   TSKEY      keyMin;
   TSKEY      keyMax;
-  uint64_t   nRows;
+  int64_t    nrows;
-  SSkipList *pData;  // Here need a container, may not use the SL
+  SSkipList *pData;
-  T_REF_DECLARE()
 } STbData;
-#endif
 
 struct STsdbMemTable {
   T_REF_DECLARE()
@@ -40,45 +31,315 @@ struct STsdbMemTable {
   uint64_t       nRow;
   SMemAllocator *pMA;
   // Container
+#if 1
+  SSkipList *pSlIdx;  // SSkiplist<STbData>
+  SHashObj * pHashIdx;
+#else
   TD_SLIST(STbData) list;
+#endif
 };
 
-STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) {
+static int      tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitMsg *pMsg);
-  STsdbMemTable *pMemTable;
+static int      tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows);
-  SMemAllocator *pMA;
+static STbData *tsdbNewTbData(tb_uid_t uid);
+static void     tsdbFreeTbData(STbData *pTbData);
+static char *   tsdbGetTsTupleKey(const void *data);
+static int      tsdbTbDataComp(const void *arg1, const void *arg2);
+static char *   tsdbTbDataGetUid(const void *arg);
 
-  pMA = (*pMAF->create)(pMAF);
+STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) {
-  ASSERT(pMA != NULL);
+  STsdbMemTable *pMemTable = (STsdbMemTable *)calloc(1, sizeof(*pMemTable));
-
-  pMemTable = (STsdbMemTable *)TD_MA_MALLOC(pMA, sizeof(*pMemTable));
   if (pMemTable == NULL) {
-    (*pMAF->destroy)(pMAF, pMA);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
   T_REF_INIT_VAL(pMemTable, 1);
   taosInitRWLatch(&(pMemTable->latch));
-  pMemTable->keyMin = TSKEY_MAX;
   pMemTable->keyMax = TSKEY_MIN;
+  pMemTable->keyMin = TSKEY_MAX;
   pMemTable->nRow = 0;
-  pMemTable->pMA = pMA;
+  pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf);
-  TD_SLIST_INIT(&(pMemTable->list));
+  if (pMemTable->pMA == NULL) {
+    free(pMemTable);
+    return NULL;
+  }
+
+  // Initialize the container
+  pMemTable->pSlIdx =
+      tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
+  if (pMemTable->pSlIdx == NULL) {
+    pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
+    free(pMemTable);
+    return NULL;
+  }
+
+  pMemTable->pHashIdx = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (pMemTable->pHashIdx == NULL) {
+    pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
+    tSkipListDestroy(pMemTable->pSlIdx);
+    free(pMemTable);
+    return NULL;
+  }
 
-  // TODO
   return pMemTable;
 }
 
-void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) {
+void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) {
-  SMemAllocator *pMA = pMemTable->pMA;
+  if (pMemTable) {
-
+    taosHashCleanup(pMemTable->pHashIdx);
-  if (TD_MA_FREE_FUNC(pMA) != NULL) {
+    tSkipListDestroy(pMemTable->pSlIdx);
-    // TODO
+    if (pMemTable->pMA) {
-    ASSERT(0);
+      pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
+    }
+    free(pMemTable);
+  }
 }
 
-  (*pMAF->destroy)(pMAF, pMA);
+int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+  SSubmitBlk *   pBlock = NULL;
+  SSubmitMsgIter msgIter = {0};
+  int32_t        affectedrows = 0, numOfRows = 0;
+
+  if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
+    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+      tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    }
+    return -1;
   }
 
+  tInitSubmitMsgIter(pMsg, &msgIter);
+  while (true) {
+    tGetSubmitMsgNext(&msgIter, &pBlock);
+    if (pBlock == NULL) break;
+    if (tsdbMemTableInsertTbData(pTsdb, pBlock, &affectedrows) < 0) {
+      return -1;
+    }
+
+    numOfRows += pBlock->numOfRows;
+  }
+
+  if (pRsp != NULL) {
+    pRsp->affectedRows = htonl(affectedrows);
+    pRsp->numOfRows = htonl(numOfRows);
+  }
+
+  return 0;
+}
+
+static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitMsg *pMsg) {
+  ASSERT(pMsg != NULL);
+  // STsdbMeta *    pMeta = pTsdb->tsdbMeta;
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlk *   pBlock = NULL;
+  SSubmitBlkIter blkIter = {0};
+  SMemRow        row = NULL;
+  TSKEY          now = taosGetTimestamp(pTsdb->config.precision);
+  TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep;
+  TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.daysPerFile;
+
+  terrno = TSDB_CODE_SUCCESS;
+  pMsg->length = htonl(pMsg->length);
+  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  while (true) {
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (pBlock == NULL) break;
+
+    pBlock->uid = htobe64(pBlock->uid);
+    pBlock->tid = htonl(pBlock->tid);
+    pBlock->sversion = htonl(pBlock->sversion);
+    pBlock->dataLen = htonl(pBlock->dataLen);
+    pBlock->schemaLen = htonl(pBlock->schemaLen);
+    pBlock->numOfRows = htons(pBlock->numOfRows);
+
+#if 0
+    if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+      return -1;
+    }
+
+    STable *pTable = pMeta->tables[pBlock->tid];
+    if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+      return -1;
+    }
+
+    if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+      tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable));
+      terrno = TSDB_CODE_TDB_INVALID_ACTION;
+      return -1;
+    }
+
+    // Check schema version and update schema if needed
+    if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) {
+      if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+        continue;
+      } else {
+        return -1;
+      }
+    }
+
+    tsdbInitSubmitBlkIter(pBlock, &blkIter);
+    while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+      if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) {
+        return -1;
+      }
+    }
+#endif
+  }
+
+  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  return 0;
+}
+
+static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) {
+  // STsdbMeta       *pMeta = pRepo->tsdbMeta;
+  // int32_t          points = 0;
+  // STable          *pTable = NULL;
+  SSubmitBlkIter blkIter = {0};
+  STsdbMemTable *pMemTable = pTsdb->mem;
+  void *         tptr;
+  STbData *      pTbData;
+  SMemRow        row;
+  TSKEY          keyMin;
+  TSKEY          keyMax;
+
+  // SMemTable       *pMemTable = NULL;
+  // STableData      *pTableData = NULL;
+  // STsdbCfg        *pCfg = &(pRepo->config);
+
+  tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid));
+  if (tptr == NULL) {
+    pTbData = tsdbNewTbData(pBlock->uid);
+    if (pTbData == NULL) {
+      return -1;
+    }
+
+    // Put into hash
+    taosHashPut(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid), &(pTbData), sizeof(pTbData));
+
+    // Put into skiplist
+    tSkipListPut(pMemTable->pSlIdx, pTbData);
+  } else {
+    pTbData = *(STbData **)tptr;
+  }
+
+  tInitSubmitBlkIter(pBlock, &blkIter);
+  if (blkIter.row == NULL) return 0;
+  keyMin = memRowKey(blkIter.row);
+
+  tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
+
+  // Set statistics
+  keyMax = memRowKey(blkIter.row);
+
+  pTbData->nrows += pBlock->numOfRows;
+  if (pTbData->keyMin > keyMin) pTbData->keyMin = keyMin;
+  if (pTbData->keyMax < keyMax) pTbData->keyMax = keyMax;
+
+  pMemTable->nRow += pBlock->numOfRows;
+  if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin;
+  if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax;
+
+  // SMemRow lastRow = NULL;
+  // int64_t osize = SL_SIZE(pTableData->pData);
+  // tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
+  // tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
+  // int64_t dsize = SL_SIZE(pTableData->pData) - osize;
+  // (*pAffectedRows) += points;
+
+  // if(lastRow != NULL) {
+  //   TSKEY lastRowKey = memRowKey(lastRow);
+  //   if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey;
+  //   pMemTable->numOfRows += dsize;
+
+  //   if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey;
+  //   pTableData->numOfRows += dsize;
+  //   if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey;
+  //   if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey;
+  //   if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) {
+  //     return -1;
+  //   }
+  // }
+
+  // STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion, -1);
+  // pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
+  // pRepo->stat.totalStorage += points * schemaVLen(pSchema);
+
+  return 0;
+}
+
+static STbData *tsdbNewTbData(tb_uid_t uid) {
+  STbData *pTbData = (STbData *)calloc(1, sizeof(*pTbData));
+  if (pTbData == NULL) {
+    return NULL;
+  }
+
+  pTbData->uid = uid;
+  pTbData->keyMin = TSKEY_MAX;
+  pTbData->keyMax = TSKEY_MIN;
+  pTbData->nrows = 0;
+
+  // uint8_t skipListCreateFlags;
+  // if (pCfg->update == TD_ROW_DISCARD_UPDATE)
+  //   skipListCreateFlags = SL_DISCARD_DUP_KEY;
+  // else
+  //   skipListCreateFlags = SL_UPDATE_DUP_KEY;
+
+  // pTableData->pData =
+  //     tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
+  //                     tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey);
+  // if (pTableData->pData == NULL) {
+  //   terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  //   free(pTableData);
+  //   return NULL;
+  // }
+
+  pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY,
+                                   tsdbGetTsTupleKey);
+  if (pTbData->pData == NULL) {
+    free(pTbData);
+    return NULL;
+  }
+
+  return pTbData;
+}
+
+static void tsdbFreeTbData(STbData *pTbData) {
+  if (pTbData) {
+    tSkipListDestroy(pTbData->pData);
+    free(pTbData);
+  }
+}
+
+static char *tsdbGetTsTupleKey(const void *data) { return memRowKeys((SMemRow)data); }
+
+static int tsdbTbDataComp(const void *arg1, const void *arg2) {
+  STbData *pTbData1 = (STbData *)arg1;
+  STbData *pTbData2 = (STbData *)arg2;
+
+  if (pTbData1->uid > pTbData2->uid) {
+    return 1;
+  } else if (pTbData1->uid == pTbData2->uid) {
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+static char *tsdbTbDataGetUid(const void *arg) {
+  STbData *pTbData = (STbData *)arg;
+  return (char *)(&(pTbData->uid));
+}
+
+/* ------------------------ REFACTORING ------------------------ */
+#if 0
 int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) {
   SMemAllocator *pMA = pMemTable->pMA;
   STbData *      pTbData = (STbData *)TD_MA_MALLOC(pMA, sizeof(*pTbData));
@@ -91,4 +352,609 @@ int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) {
   return 0;
 }
 
-/* ------------------------ STATIC METHODS ------------------------ */
+#include "tdataformat.h"
+#include "tfunctional.h"
+#include "tsdbRowMergeBuf.h"
+#include "tsdbint.h"
+#include "tskiplist.h"
+
+#define TSDB_DATA_SKIPLIST_LEVEL 5
+#define TSDB_MAX_INSERT_BATCH 512
+
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SMemRow  row;
+} SSubmitBlkIter;
+
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  void *  pMsg;
+} SSubmitMsgIter;
+
+static SMemTable *  tsdbNewMemTable(STsdbRepo *pRepo);
+static void         tsdbFreeMemTable(SMemTable *pMemTable);
+static STableData*  tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
+static void         tsdbFreeTableData(STableData *pTableData);
+static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
+static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row);
+static int          tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+static SMemRow      tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static int          tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
+static int          tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
+static int          tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+static int          tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
+static int          tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
+static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row);
+
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
+                                          TSKEY now);
+
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
+  if (pMemTable == NULL) return 0;
+  int ref = T_REF_INC(pMemTable);
+	tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref);
+  return 0;
+}
+
+// Need to lock the repository
+int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
+  if (pMemTable == NULL) return 0;
+
+	int ref = T_REF_DEC(pMemTable);
+	tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref);
+  if (ref == 0) {
+    STsdbBufPool *pBufPool = pRepo->pPool;
+
+    SListNode *pNode = NULL;
+    bool addNew = false;
+    if (tsdbLockRepo(pRepo) < 0) return -1;
+    while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) {
+      if (pBufPool->nRecycleBlocks > 0) {
+        tsdbRecycleBufferBlock(pBufPool, pNode, false);
+        pBufPool->nRecycleBlocks -= 1;
+      } else {
+        if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) {
+          tsdbRecycleBufferBlock(pBufPool, pNode, true);
+        } else {
+          tdListAppendNode(pBufPool->bufBlockList, pNode);
+          addNew = true;
+        }
+      }      
+    }
+    if (addNew) {
+      int code = pthread_cond_signal(&pBufPool->poolNotEmpty);
+      if (code != 0) {
+        if (tsdbUnlockRepo(pRepo) < 0) return -1;
+        tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code));
+        terrno = TAOS_SYSTEM_ERROR(code);
+        return -1;
+      }
+    }
+
+    if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+    for (int i = 0; i < pMemTable->maxTables; i++) {
+      if (pMemTable->tData[i] != NULL) {
+        tsdbFreeTableData(pMemTable->tData[i]);
+      }
+    }
+
+    tdListDiscard(pMemTable->actList);
+    tdListDiscard(pMemTable->bufBlockList);
+    tsdbFreeMemTable(pMemTable);
+  }
+  return 0;
+}
+
+int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATable) {
+  memset(pSnapshot, 0, sizeof(*pSnapshot));
+
+  if (tsdbLockRepo(pRepo) < 0) return -1;
+
+  pSnapshot->omem = pRepo->mem;
+  pSnapshot->imem = pRepo->imem;
+  tsdbRefMemTable(pRepo, pRepo->mem);
+  tsdbRefMemTable(pRepo, pRepo->imem);
+
+  if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+  if (pSnapshot->omem) {
+    taosRLockLatch(&(pSnapshot->omem->latch));
+
+    pSnapshot->mem = &(pSnapshot->mtable);
+
+    pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *));
+    if (pSnapshot->mem->tData == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      taosRUnLockLatch(&(pSnapshot->omem->latch));
+      tsdbUnRefMemTable(pRepo, pSnapshot->omem);
+      tsdbUnRefMemTable(pRepo, pSnapshot->imem);
+      pSnapshot->mem = NULL;
+      pSnapshot->imem = NULL;
+      pSnapshot->omem = NULL;
+      return -1;
+    }
+
+    pSnapshot->mem->keyFirst = pSnapshot->omem->keyFirst;
+    pSnapshot->mem->keyLast = pSnapshot->omem->keyLast;
+    pSnapshot->mem->numOfRows = pSnapshot->omem->numOfRows;
+    pSnapshot->mem->maxTables = pSnapshot->omem->maxTables;
+
+    for (size_t i = 0; i < taosArrayGetSize(pATable); i++) {
+      STable *    pTable = *(STable **)taosArrayGet(pATable, i);
+      int32_t     tid = TABLE_TID(pTable);
+      STableData *pTableData = (tid < pSnapshot->omem->maxTables) ? pSnapshot->omem->tData[tid] : NULL;
+
+      if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue;
+
+      pSnapshot->mem->tData[tid] = pTableData;
+      T_REF_INC(pTableData);
+    }
+
+    taosRUnLockLatch(&(pSnapshot->omem->latch));
+  }
+
+  tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem);
+  return 0;
+}
+
+void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) {
+  tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem);
+
+  if (pSnapshot->mem) {
+    ASSERT(pSnapshot->omem != NULL);
+
+    for (size_t i = 0; i < pSnapshot->mem->maxTables; i++) {
+      STableData *pTableData = pSnapshot->mem->tData[i];
+      if (pTableData) {
+        tsdbFreeTableData(pTableData);
+      }
+    }
+    tfree(pSnapshot->mem->tData);
+
+    tsdbUnRefMemTable(pRepo, pSnapshot->omem);
+  }
+
+  tsdbUnRefMemTable(pRepo, pSnapshot->imem);
+
+  pSnapshot->mem = NULL;
+  pSnapshot->imem = NULL;
+  pSnapshot->omem = NULL;
+}
+
+int tsdbSyncCommitConfig(STsdbRepo* pRepo) {
+  ASSERT(pRepo->config_changed == true);
+  tsem_wait(&(pRepo->readyToCommit));
+
+  if (pRepo->code != TSDB_CODE_SUCCESS) {
+    tsdbWarn("vgId:%d try to commit config when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno));
+  }
+
+  if (tsdbLockRepo(pRepo) < 0) return -1;
+  tsdbScheduleCommit(pRepo, COMMIT_CONFIG_REQ);
+  if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+  tsem_wait(&(pRepo->readyToCommit));
+  tsem_post(&(pRepo->readyToCommit));
+
+  if (pRepo->code != TSDB_CODE_SUCCESS) {
+    terrno = pRepo->code;
+    return -1;
+  }
+
+  terrno = TSDB_CODE_SUCCESS;
+  return 0;
+}
+
+/**
+ * This is an important function to load data or try to load data from memory skiplist iterator.
+ * 
+ * This function load memory data until:
+ * 1. iterator ends
+ * 2. data key exceeds maxKey
+ * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
+ * 4. operations in pCols not exceeds its max capacity if pCols is given
+ * 
+ * The function tries to procceed AS MUCH AS POSSIBLE.
+ */
+int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
+                          TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
+  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
+  if (pIter == NULL) return 0;
+  STSchema * pSchema = NULL;
+  TSKEY      rowKey = 0;
+  TSKEY      fKey = 0;
+  bool       isRowDel = false;
+  int        filterIter = 0;
+  SMemRow    row = NULL;
+  SMergeInfo mInfo;
+
+  if (pMergeInfo == NULL) pMergeInfo = &mInfo;
+
+  memset(pMergeInfo, 0, sizeof(*pMergeInfo));
+  pMergeInfo->keyFirst = INT64_MAX;
+  pMergeInfo->keyLast = INT64_MIN;
+  if (pCols) tdResetDataCols(pCols);
+
+  row = tsdbNextIterRow(pIter);
+  if (row == NULL || memRowKey(row) > maxKey) {
+    rowKey = INT64_MAX;
+    isRowDel = false;
+  } else {
+    rowKey = memRowKey(row);
+    isRowDel = memRowDeleted(row);
+  }
+
+  if (filterIter >= nFilterKeys) {
+    fKey = INT64_MAX;
+  } else {
+    fKey = tdGetKey(filterKeys[filterIter]);
+  }
+
+  while (true) {
+    if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
+
+    if (fKey < rowKey) {
+      pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+      pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+    } else if (fKey > rowKey) {
+      if (isRowDel) {
+        pMergeInfo->rowsDeleteFailed++;
+      } else {
+        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsInserted++;
+        pMergeInfo->nOperations++;
+        pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+        pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || memRowKey(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = memRowKey(row);
+        isRowDel = memRowDeleted(row);
+      }
+    } else {
+      if (isRowDel) {
+        ASSERT(!keepDup);
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsDeleteSucceed++;
+        pMergeInfo->nOperations++;
+        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+      } else {
+        if (keepDup) {
+          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+          pMergeInfo->rowsUpdated++;
+          pMergeInfo->nOperations++;
+          pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+          pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+          tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+        } else {
+          pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+          pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+        }
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || memRowKey(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = memRowKey(row);
+        isRowDel = memRowDeleted(row);
+      }
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+    }
+  }
+
+  return 0;
+}
+
+// ---------------- LOCAL FUNCTIONS ----------------
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) {
+  if (pCols) {
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row), (int8_t)memRowType(row));
+      if (*ppSchema == NULL) {
+        ASSERT(false);
+        return -1;
+      }
+    }
+
+    tdAppendMemRowToDataCol(row, *ppSchema, pCols, true, 0);
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
+                                          TSKEY now) {
+  TSKEY rowKey = memRowKey(row);
+  if (rowKey < minKey || rowKey > maxKey) {
+    tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
+              " maxKey %" PRId64 " row key %" PRId64,
+              REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey,
+              rowKey);
+    terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
+    return -1;
+  }
+
+  return 0;
+}
+
+
+//row1 has higher priority
+static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo,
+                                     STSchema **ppSchema1, STSchema **ppSchema2,
+                                     STable* pTable, int32_t* pPoints, SMemRow* pLastRow) {
+
+  //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows!
+  if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) {
+    (*pPoints)++;
+    return NULL;
+  }
+
+  tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+            "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
+            memRowKey(row1));
+
+  if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) {
+    void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1));
+    if(pMem == NULL) return NULL;
+    memRowCpy(pMem, row1);
+    (*pPoints)++;
+    *pLastRow = pMem;
+    return pMem;
+  }
+
+  STSchema *pSchema1 = *ppSchema1;
+  STSchema *pSchema2 = *ppSchema2;
+  SMergeBuf * pBuf = &pRepo->mergeBuf;
+  int dv1 = memRowVersion(row1);
+  int dv2 = memRowVersion(row2);
+  if(pSchema1 == NULL || schemaVersion(pSchema1) != dv1) {
+    if(pSchema2 != NULL && schemaVersion(pSchema2) == dv1) {
+      *ppSchema1 = pSchema2;
+    } else {
+      *ppSchema1 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row1), (int8_t)memRowType(row1));
+    }
+    pSchema1 = *ppSchema1;
+  }
+
+  if(pSchema2 == NULL || schemaVersion(pSchema2) != dv2) {
+    if(schemaVersion(pSchema1) == dv2) {
+      pSchema2 = pSchema1;
+    } else {
+      *ppSchema2 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row2), (int8_t)memRowType(row2));
+      pSchema2 = *ppSchema2;
+    }
+  }
+
+  SMemRow tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2);
+
+  void* pMem = tsdbAllocBytes(pRepo, memRowTLen(tmp));
+  if(pMem == NULL) return NULL;
+  memRowCpy(pMem, tmp);
+
+  (*pPoints)++;
+  *pLastRow = pMem;
+  return pMem;
+}
+
+static void* tsdbInsertDupKeyMergePacked(void** args) {
+  return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]);
+}
+
+static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) {
+
+  if(pSkipList->insertHandleFn == NULL) {
+    tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9);
+    dupHandleSavedFunc->args[2] = pRepo;
+    dupHandleSavedFunc->args[3] = NULL;
+    dupHandleSavedFunc->args[4] = NULL;
+    dupHandleSavedFunc->args[5] = pTable;
+    pSkipList->insertHandleFn = dupHandleSavedFunc;
+  }
+  pSkipList->insertHandleFn->args[6] = pPoints;
+  pSkipList->insertHandleFn->args[7] = pLastRow;
+}
+
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
+  ASSERT(pTable != NULL);
+
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1, -1);
+  int       sversion = schemaVersion(pSchema);
+
+  if (pBlock->sversion == sversion) {
+    return 0;
+  } else {
+    if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {  // stream table is not allowed to change schema
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      return -1;
+    }
+  }
+
+  if (pBlock->sversion > sversion) {  // may need to update table schema
+    if (pBlock->schemaLen > 0) {
+      tsdbDebug(
+          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
+          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+      ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
+      int       numOfCols = pBlock->schemaLen / sizeof(STColumn);
+      STColumn *pTCol = (STColumn *)pBlock->data;
+
+      STSchemaBuilder schemaBuilder = {0};
+      if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+                  tstrerror(terrno));
+        return -1;
+      }
+
+      for (int i = 0; i < numOfCols; i++) {
+        if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
+          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+          tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+                    tstrerror(terrno));
+          tdDestroyTSchemaBuilder(&schemaBuilder);
+          return -1;
+        }
+      }
+
+      STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+      if (pNSchema == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tdDestroyTSchemaBuilder(&schemaBuilder);
+        return -1;
+      }
+
+      tdDestroyTSchemaBuilder(&schemaBuilder);
+      tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
+    } else {
+      tsdbDebug(
+          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
+          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+      terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
+      return -1;
+    }
+  } else {
+    ASSERT(pBlock->sversion >= 0);
+    if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion, -1) == NULL) {
+      tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
+                pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
+  tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data,
+            memRowVersion(row));
+
+  STSchema* pSchema = tsdbGetTableLatestSchema(pTable);
+  if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) {
+    return;
+  }
+
+  pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row), (int8_t)memRowType(row));
+  if (pSchema == NULL) {
+    return;
+  }
+
+  SDataCol *pLatestCols = pTable->lastCols;
+  int32_t kvIdx = 0;
+
+  for (int16_t j = 0; j < schemaNCols(pSchema); j++) {
+    STColumn *pTCol = schemaColAt(pSchema, j);
+    // ignore not exist colId
+    int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pTCol->colId);
+    if (idx == -1) {
+      continue;
+    }
+
+    void *value = NULL;
+
+    value = tdGetMemRowDataOfColEx(row, pTCol->colId, (int8_t)pTCol->type,
+                                   TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset, &kvIdx);
+
+    if ((value == NULL) || isNull(value, pTCol->type)) {
+      continue;
+    }
+    // lock
+    TSDB_WLOCK_TABLE(pTable); 
+    SDataCol *pDataCol = &(pLatestCols[idx]);
+    if (pDataCol->pData == NULL) {
+      pDataCol->pData = malloc(pTCol->bytes);
+      pDataCol->bytes = pTCol->bytes;
+    } else if (pDataCol->bytes < pTCol->bytes) {
+      pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes);
+      pDataCol->bytes = pTCol->bytes;
+    }
+    // the actual value size
+    uint16_t bytes = IS_VAR_DATA_TYPE(pTCol->type) ? varDataTLen(value) : pTCol->bytes;
+    // the actual data size CANNOT larger than column size
+    assert(pTCol->bytes >= bytes);
+    memcpy(pDataCol->pData, value, bytes);
+    //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData);
+    pDataCol->ts = memRowKey(row);
+    // unlock
+    TSDB_WUNLOCK_TABLE(pTable); 
+  }
+}
+
+static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
+  STsdbCfg *pCfg = &pRepo->config;
+
+  // if cacheLastRow config has been reset, free the lastRow
+  if (!pCfg->cacheLastRow && pTable->lastRow != NULL) {
+    SMemRow cachedLastRow = pTable->lastRow;
+    TSDB_WLOCK_TABLE(pTable);
+    pTable->lastRow = NULL;
+    TSDB_WUNLOCK_TABLE(pTable);
+    taosTZfree(cachedLastRow);
+  }
+
+  if (tsdbGetTableLastKeyImpl(pTable) <= memRowKey(row)) {
+    if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) {
+      SMemRow nrow = pTable->lastRow;
+      if (taosTSizeof(nrow) < memRowTLen(row)) {
+        SMemRow orow = nrow;
+        nrow = taosTMalloc(memRowTLen(row));
+        if (nrow == NULL) {
+          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+          return -1;
+        }
+
+        memRowCpy(nrow, row);
+        TSDB_WLOCK_TABLE(pTable);
+        pTable->lastKey = memRowKey(row);
+        pTable->lastRow = nrow;
+        TSDB_WUNLOCK_TABLE(pTable);
+        taosTZfree(orow);
+      } else {
+        TSDB_WLOCK_TABLE(pTable);
+        pTable->lastKey = memRowKey(row);
+        memRowCpy(nrow, row);
+        TSDB_WUNLOCK_TABLE(pTable);
+      }
+    } else {
+      pTable->lastKey = memRowKey(row);
+    }
+
+    if (CACHE_LAST_NULL_COLUMN(pCfg)) {
+      updateTableLatestColumn(pRepo, pTable, row);
+    }
+  }
+
+  pTable->cacheLastConfigVersion = pRepo->cacheLastConfigVersion;
+
+  return 0;
+}
+
+#endif

source/dnode/vnode/tsdb/src/tsdbReadImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbint.h"
+#include "tsdbDef.h"
 
 #define TSDB_KEY_COL_OFFSET 0
 
@@ -25,8 +25,9 @@ static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int3
 static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
                                       int numOfColIds);
 static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
+static STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) { return NULL; }
 
-int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) {
+int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
   ASSERT(pReadh != NULL && pRepo != NULL);
 
   STsdbCfg *pCfg = REPO_CFG(pRepo);
@@ -259,7 +260,9 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
-    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
+                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+      return -1;
   }
 
   ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
@@ -286,7 +289,9 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
-    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
+                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+      return -1;
   }
 
   ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
@@ -624,7 +629,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
   ASSERT(pDataCol->colId == pBlockCol->colId);
 
-  STsdbRepo *pRepo = TSDB_READ_REPO(pReadh);
+  STsdb *   pRepo = TSDB_READ_REPO(pReadh);
   STsdbCfg *pCfg = REPO_CFG(pRepo);
   int       tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
 
@@ -662,3 +667,4 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
 
   return 0;
 }
+

source/dnode/vnode/tsdb/src/tsdbWrite.c

@@ -15,11 +15,11 @@
 
 #include "tsdbDef.h"
 
-int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg) {
+int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp) {
   // Check if mem is there. If not, create one.
-  pTsdb->mem = tsdbNewMemTable(pTsdb->pmaf);
+  pTsdb->mem = tsdbNewMemTable(pTsdb);
   if (pTsdb->mem == NULL) {
     return -1;
   }
-  return tsdbInsertDataToMemTable(pTsdb->mem, pMsg);
+  return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
 }

source/dnode/vnode/tsdb2/CMakeLists.txt

@@ -0,0 +1,9 @@
+aux_source_directory(src TSDB_SRC)
+add_library(tsdb STATIC ${TSDB_SRC})
+
+target_include_directories(
+    tsdb
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb2"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+target_link_libraries(tsdb os util common tfs)

source/dnode/vnode/tsdb2/inc/tsdbBuffer.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// #ifndef _TD_TSDB_BUFFER_H_
+// #define _TD_TSDB_BUFFER_H_
+
+// typedef struct {
+//   int64_t blockId;
+//   int     offset;
+//   int     remain;
+//   char    data[];
+// } STsdbBufBlock;
+
+// typedef struct {
+//   pthread_cond_t poolNotEmpty;
+//   int            bufBlockSize;
+//   int            tBufBlocks;
+//   int            nBufBlocks;
+//   int            nRecycleBlocks;
+//   int            nElasticBlocks;
+//   int64_t        index;
+//   SList*         bufBlockList;  
+// } STsdbBufPool;
+
+// #define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
+
+// STsdbBufPool* tsdbNewBufPool();
+// void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
+// int           tsdbOpenBufPool(STsdb* pRepo);
+// void          tsdbCloseBufPool(STsdb* pRepo);
+// SListNode*    tsdbAllocBufBlockFromPool(STsdb* pRepo);
+// int           tsdbExpandPool(STsdb* pRepo, int32_t oldTotalBlocks);
+// void          tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic);
+
+// // health cite
+// STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize);
+// void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock);
+
+// #endif /* _TD_TSDB_BUFFER_H_ */

source/dnode/vnode/tsdb2/inc/tsdbCommit.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_COMMIT_H_
+#define _TD_TSDB_COMMIT_H_
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
+
+void  tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
+int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
+void *tsdbCommitData(STsdb *pRepo);
+int   tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
+int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx);
+int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
+int   tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
+                         SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf);
+int   tsdbApplyRtn(STsdb *pRepo);
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
+}
+
+#endif /* _TD_TSDB_COMMIT_H_ */

source/dnode/vnode/tsdb2/inc/tsdbCompact.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+// #ifndef _TD_TSDB_COMPACT_H_
+// #define _TD_TSDB_COMPACT_H_
+
+// #ifdef __cplusplus
+// extern "C" {
+// #endif
+
+// void *tsdbCompactImpl(STsdb *pRepo);
+
+// #ifdef __cplusplus
+// }
+// #endif
+
+// #endif /* _TD_TSDB_COMPACT_H_ */

source/dnode/vnode/tsdb2/inc/tsdbFS.h

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_FS_H_
+#define _TD_TSDB_FS_H_
+
+/**
+ * 1. The fileset .head/.data/.last use the same fver 0 before 2021.10.10.
+ * 2. .head fver is 1 when extract aggregate block data from .data/.last file and save to separate .smad/.smal file
+ * since 2021.10.10
+ * // TODO update date and add release version.
+ */
+typedef enum {
+  TSDB_FS_VER_0 = 0,
+  TSDB_FS_VER_1,
+} ETsdbFsVer;
+
+#define TSDB_FVER_TYPE uint32_t
+#define TSDB_LATEST_FVER TSDB_FS_VER_1     // latest version for DFile
+#define TSDB_LATEST_SFS_VER TSDB_FS_VER_1  // latest version for 'current' file
+
+static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest version for DFile
+  switch (fType) {
+    case TSDB_FILE_HEAD:
+      return TSDB_FS_VER_1;
+    default:
+      return TSDB_FS_VER_0;
+  }
+}
+
+// ================== TSDB global config
+extern bool tsdbForceKeepFile;
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SMFile*     pmf;   // meta file pointer
+  SMFile      mf;    // meta file
+  SArray*     df;    // data file array
+} SFSStatus;
+
+typedef struct {
+  pthread_rwlock_t lock;
+
+  SFSStatus* cstatus;         // current status
+  SHashObj*  metaCache;       // meta cache
+  SHashObj*  metaCacheComp;   // meta cache for compact
+  bool       intxn;
+  SFSStatus* nstatus;  // new status
+} STsdbFS;
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS*   pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet* pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdb *pRepo);
+void     tsdbCloseFS(STsdb *pRepo);
+void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdb *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_rdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_wrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_unlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+#endif /* _TD_TSDB_FS_H_ */

source/dnode/vnode/tsdb2/inc/tsdbFile.h

@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TS_TSDB_FILE_H_
+#define _TS_TSDB_FILE_H_
+
+#include "os.h"
+
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+#define TSDB_FILE_STATE_OK 0
+#define TSDB_FILE_STATE_BAD 1
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
+#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
+#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
+#define TSDB_FILE_STATE(tf) ((tf)->state)
+#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
+#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
+#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
+#define ASSERT_TSDB_FSET_NFILES_VALID(s)                              \
+  do {                                                                \
+    uint8_t nDFiles = tsdbGetNFiles(s);                               \
+    ASSERT((nDFiles >= TSDB_FILE_MIN) && (nDFiles <= TSDB_FILE_MAX)); \
+  } while (0)
+typedef enum {
+  TSDB_FILE_HEAD = 0,
+  TSDB_FILE_DATA,
+  TSDB_FILE_LAST,
+  TSDB_FILE_SMAD,  // sma for .data
+  TSDB_FILE_SMAL,  // sma for .last
+  TSDB_FILE_MAX,
+  TSDB_FILE_META
+} TSDB_FILE_T;
+
+#define TSDB_FILE_MIN 3U  // min valid number of files in one DFileSet(.head/.data/.last)
+
+// =============== SMFile
+typedef struct {
+  int64_t  size;
+  int64_t  tombSize;
+  int64_t  nRecords;
+  int64_t  nDels;
+  uint32_t magic;
+} SMFInfo;
+
+typedef struct {
+  SMFInfo info;
+  TFILE   f;
+  int     fd;
+  uint8_t state;
+} SMFile;
+
+void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
+void  tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
+int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
+int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
+int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
+int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
+int   tsdbUpdateMFileHeader(SMFile* pMFile);
+int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
+int   tsdbScanAndTryFixMFile(STsdb* pRepo);
+int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
+void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
+
+static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
+  ASSERT(TSDB_FILE_CLOSED(pMFile));
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags | O_BINARY);
+  if (pMFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
+  if (TSDB_FILE_OPENED(pMFile)) {
+    close(pMFile->fd);
+    TSDB_FILE_SET_CLOSED(pMFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pMFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nwrite = taosWriteFile(pMFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pMFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pMFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
+
+static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nread = taosReadFile(pMFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+// =============== SDFile
+typedef struct {
+  uint32_t magic;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+  uint32_t fver;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo info;
+  TFILE   f;
+  int     fd;
+  uint8_t state;
+} SDFile;
+
+void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(void* buf, SDFile* pDFile, uint32_t sfver);
+int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader, TSDB_FILE_T ftype);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags | O_BINARY);
+  if (pDFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    close(pDFile->fd);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+typedef struct {
+  int      fid;
+  int      state;
+  uint16_t ver;  // fset version
+  SDFile   files[TSDB_FILE_MAX];
+} SDFileSet;
+
+typedef enum {
+  TSDB_FSET_VER_0 = 0,  // .head/.data/.last
+  TSDB_FSET_VER_1,      // .head/.data/.last/.smad/.smal
+} ETsdbFSetVer;
+
+#define TSDB_LATEST_FSET_VER TSDB_FSET_VER_1
+
+// get nDFiles in SDFileSet
+static FORCE_INLINE uint8_t tsdbGetNFiles(SDFileSet* pSet) {
+  switch (pSet->ver) {
+    case TSDB_FSET_VER_0:
+      return TSDB_FILE_MIN;
+    case TSDB_FSET_VER_1:
+    default:
+      return TSDB_FILE_MAX;
+  }
+}
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                        \
+  do {                                                                            \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < tsdbGetNFiles(s); ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                               \
+    }                                                                             \
+  } while (0);
+
+void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver, uint16_t fsetVer);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet, uint32_t sfver);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSrc);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSrc); ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsTickPerDay[precision];
+  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
+}
+
+static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+#endif /* _TS_TSDB_FILE_H_ */

source/dnode/vnode/tsdb2/inc/tsdbLog.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */

source/dnode/vnode/tsdb2/inc/tsdbMemTable.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMTABLE_H_
+#define _TD_TSDB_MEMTABLE_H_
+
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
+
+typedef struct {
+  STable *           pTable;
+  SSkipListIterator *pIter;
+} SCommitIter;
+
+struct STableData {
+  uint64_t   uid;
+  TSKEY      keyFirst;
+  TSKEY      keyLast;
+  int64_t    numOfRows;
+  SSkipList* pData;
+  T_REF_DECLARE()
+};
+
+enum { TSDB_UPDATE_META, TSDB_DROP_META };
+
+#ifdef WINDOWS
+#pragma pack(push ,1) 
+typedef struct {
+#else
+typedef struct __attribute__((packed)){
+#endif
+  char     act;
+  uint64_t uid;
+} SActObj;
+#ifdef WINDOWS
+#pragma pack(pop) 
+#endif
+
+typedef struct {
+  int  len;
+  char cont[];
+} SActCont;
+
+int   tsdbRefMemTable(STsdb* pRepo, SMemTable* pMemTable);
+int   tsdbUnRefMemTable(STsdb* pRepo, SMemTable* pMemTable);
+int   tsdbTakeMemSnapshot(STsdb* pRepo, SMemSnapshot* pSnapshot, SArray* pATable);
+void  tsdbUnTakeMemSnapShot(STsdb* pRepo, SMemSnapshot* pSnapshot);
+void* tsdbAllocBytes(STsdb* pRepo, int bytes);
+int   tsdbAsyncCommit(STsdb* pRepo);
+int   tsdbSyncCommitConfig(STsdb* pRepo);
+int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
+                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdb* pRepo);
+
+static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) {
+  if (pIter == NULL) return NULL;
+
+  SSkipListNode* node = tSkipListIterGet(pIter);
+  if (node == NULL) return NULL;
+
+  return (SMemRow)SL_GET_NODE_DATA(node);
+}
+
+static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
+  SMemRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
+
+  return memRowKey(row);
+}
+
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+  SMemRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TKEY_NULL;
+
+  return memRowTKey(row);
+}
+
+#endif /* _TD_TSDB_MEMTABLE_H_ */

source/dnode/vnode/tsdb2/inc/tsdbMemory.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMORY_H_
+#define _TD_TSDB_MEMORY_H_
+
+static void * taosTMalloc(size_t size);
+static void * taosTCalloc(size_t nmemb, size_t size);
+static void * taosTRealloc(void *ptr, size_t size);
+static void * taosTZfree(void *ptr);
+static size_t taosTSizeof(void *ptr);
+static void   taosTMemset(void *ptr, int c);
+
+static FORCE_INLINE void *taosTMalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = malloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = taosTMalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  taosTMemset(ret, 0);
+  return ret;
+}
+
+static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
+
+static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return taosTMalloc(size);
+
+  if (size <= taosTSizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  void* tptr1 = realloc(tptr, tsize);
+  if (tptr1 == NULL) return NULL;
+  tptr = tptr1;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+static FORCE_INLINE void* taosTZfree(void* ptr) {
+  if (ptr) {
+    free((void*)((char*)ptr - sizeof(size_t)));
+  }
+  return NULL;
+}
+
+
+#endif /* _TD_TSDB_MEMORY_H_ */

source/dnode/vnode/tsdb2/inc/tsdbMeta.h

@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_META_H_
+#define _TD_TSDB_META_H_
+
+#include "tskiplist.h"
+
+#define TSDB_MAX_TABLE_SCHEMAS 16
+
+#pragma pack(push, 1)
+typedef struct jsonMapValue {
+  void*   table;  // STable *
+  int16_t colId;  // the json col ID.
+} JsonMapValue;
+
+#pragma pack(pop)
+
+typedef struct STable {
+  STableId       tableId;
+  ETableType     type;
+  tstr*          name;  // NOTE: there a flexible string here
+  uint64_t       suid;
+  struct STable* pSuper;  // super table pointer
+  SArray*        schema;
+  STSchema*      tagSchema;
+  SKVRow         tagVal;
+  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
+  SHashObj*      jsonKeyMap;     // For json tag key  {"key":[t1, t2, t3]}
+  void*          eventHandler;   // TODO
+  void*          streamHandler;  // TODO
+  TSKEY          lastKey;
+  SMemRow        lastRow;
+  char*          sql;
+  void*          cqhandle;
+  SRWLatch       latch;  // TODO: implementa latch functions
+  SDataCol*      lastCols;
+  int16_t        maxColNum;
+  int16_t        restoreColumnNum;
+  bool           hasRestoreLastColumn;
+  int            lastColSVersion;
+  int16_t        cacheLastConfigVersion;
+  T_REF_DECLARE()
+} STable;
+
+typedef struct {
+  pthread_rwlock_t rwLock;
+
+  int32_t   nTables;
+  int32_t   maxTables;
+  STable**  tables;
+  SList*    superList;
+  SHashObj* uidMap;
+  int       maxRowBytes;
+  int       maxCols;
+} STsdbMeta;
+
+#define TSDB_INIT_NTABLES 1024
+#define TABLE_TYPE(t) (t)->type
+#define TABLE_NAME(t) (t)->name
+#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
+#define TABLE_UID(t) (t)->tableId.uid
+#define TABLE_TID(t) (t)->tableId.tid
+#define TABLE_SUID(t) (t)->suid
+// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
+#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
+#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
+#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
+#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
+
+STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
+void       tsdbFreeMeta(STsdbMeta* pMeta);
+int        tsdbOpenMeta(STsdb* pRepo);
+int        tsdbCloseMeta(STsdb* pRepo);
+STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
+STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t _version, int8_t rowType);
+int        tsdbWLockRepoMeta(STsdb* pRepo);
+int        tsdbRLockRepoMeta(STsdb* pRepo);
+int        tsdbUnlockRepoMeta(STsdb* pRepo);
+void       tsdbRefTable(STable* pTable);
+void       tsdbUnRefTable(STable* pTable);
+void       tsdbUpdateTableSchema(STsdb* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
+int        tsdbRestoreTable(STsdb* pRepo, void* cont, int contLen);
+void       tsdbOrgMeta(STsdb* pRepo);
+int        tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema);
+int16_t    tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId);
+int        tsdbUpdateLastColSchema(STable* pTable, STSchema* pNewSchema);
+STSchema*  tsdbGetTableLatestSchema(STable* pTable);
+void       tsdbFreeLastColumns(STable* pTable);
+int        tsdbCompareJsonMapValue(const void* a, const void* b);
+void*      tsdbGetJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId);
+
+static FORCE_INLINE int tsdbCompareSchemaVersion(const void* key1, const void* key2) {
+  if (*(int16_t*)key1 < schemaVersion(*(STSchema**)key2)) {
+    return -1;
+  } else if (*(int16_t*)key1 > schemaVersion(*(STSchema**)key2)) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t _version,
+                                                     int8_t rowType) {
+  STable*   pDTable = (pTable->pSuper != NULL) ? pTable->pSuper : pTable;  // for performance purpose
+  STSchema* pSchema = NULL;
+  STSchema* pTSchema = NULL;
+
+  if (lock) TSDB_RLOCK_TABLE(pDTable);
+  if (_version < 0) {  // get the latest version of schema
+    pTSchema = *(STSchema**)taosArrayGetLast(pDTable->schema);
+  } else {  // get the schema with version
+    void* ptr = taosArraySearch(pDTable->schema, &_version, tsdbCompareSchemaVersion, TD_EQ);
+    if (ptr == NULL) {
+      if (rowType == SMEM_ROW_KV) {
+        ptr = taosArrayGetLast(pDTable->schema);
+      } else {
+        terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+        goto _exit;
+      }
+    }
+    pTSchema = *(STSchema**)ptr;
+  }
+
+  ASSERT(pTSchema != NULL);
+
+  if (copy) {
+    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  } else {
+    pSchema = pTSchema;
+  }
+
+_exit:
+  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
+  return pSchema;
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
+  return tsdbGetTableSchemaImpl(pTable, false, false, -1, -1);
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableTagSchema(STable* pTable) {
+  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
+    STable* pSuper = pTable->pSuper;
+    if (pSuper == NULL) return NULL;
+    return pSuper->tagSchema;
+  } else if (pTable->type == TSDB_SUPER_TABLE) {
+    return pTable->tagSchema;
+  } else {
+    return NULL;
+  }
+}
+
+static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
+  ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow)));
+  return pTable->lastKey;
+}
+
+#endif /* _TD_TSDB_META_H_ */

source/dnode/vnode/tsdb2/inc/tsdbReadImpl.h

@@ -0,0 +1,273 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_READ_IMPL_H_
+#define _TD_TSDB_READ_IMPL_H_
+
+#include "os.h"
+#include "tfs.h"
+#include "tsdb.h"
+#include "tsdbFile.h"
+#include "tsdbMemory.h"
+#include "tsdbMeta.h"
+#include "tskiplist.h"
+
+typedef struct SReadH SReadH;
+
+typedef struct {
+  int32_t  tid;
+  uint32_t len;
+  uint32_t offset;
+  uint32_t hasLast : 2;
+  uint32_t numOfBlocks : 30;
+  uint64_t uid;
+  TSKEY    maxKey;
+} SBlockIdx;
+
+#if 0
+typedef struct {
+  int64_t last : 1;
+  int64_t offset : 63;
+  int32_t algorithm : 8;
+  int32_t numOfRows : 24;
+  int32_t len;
+  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  int16_t numOfSubBlocks;
+  int16_t numOfCols;  // not including timestamp column
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  } SBlock;
+#endif
+
+/**
+ * keyLen;     // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+ * numOfCols;  // not including timestamp column
+ */
+#define SBlockFieldsP0    \
+  int64_t last : 1;       \
+  int64_t offset : 63;    \
+  int32_t algorithm : 8;  \
+  int32_t numOfRows : 24; \
+  int32_t len;            \
+  int32_t keyLen;         \
+  int16_t numOfSubBlocks; \
+  int16_t numOfCols;      \
+  TSKEY   keyFirst;       \
+  TSKEY   keyLast
+
+/**
+ * aggrStat;   // only valid when blkVer > 0. 0 - no aggr part in .data/.last/.smad/.smal, 1 - has aggr in .smad/.smal
+ * blkVer;     // 0 - original block, 1 - block since importing .smad/.smal
+ * aggrOffset; // only valid when blkVer > 0 and aggrStat > 0
+ */
+#define SBlockFieldsP1   \
+  uint64_t aggrStat : 1; \
+  uint64_t blkVer : 7;   \
+  uint64_t aggrOffset : 56
+
+typedef struct {
+  SBlockFieldsP0;
+} SBlockV0;
+
+typedef struct {
+  SBlockFieldsP0;
+  SBlockFieldsP1;
+} SBlockV1;
+
+typedef enum {
+  TSDB_SBLK_VER_0 = 0,
+  TSDB_SBLK_VER_1,
+} ESBlockVer;
+
+#define SBlockVerLatest TSDB_SBLK_VER_1
+
+#define SBlock SBlockV1  // latest SBlock definition
+
+// lastest SBlockInfo definition
+typedef struct {
+  int32_t  delimiter;  // For recovery usage
+  int32_t  tid;
+  uint64_t uid;
+  SBlock   blocks[];
+} SBlockInfo;
+
+typedef struct {
+  int16_t  colId;
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+  int64_t  sum;
+  int64_t  max;
+  int64_t  min;
+  int16_t  maxIndex;
+  int16_t  minIndex;
+  int16_t  numOfNull;
+  uint8_t  offsetH;
+  char     padding[1];
+} SBlockColV0;
+
+typedef struct {
+  int16_t  colId;
+  uint8_t  offsetH;
+  uint8_t  reserved;  // reserved field, not used
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+} SBlockColV1;
+
+#define SBlockCol SBlockColV1  // latest SBlockCol definition
+
+typedef struct {
+  int16_t colId;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+} SAggrBlkColV1;
+
+#define SAggrBlkCol SAggrBlkColV1  // latest SAggrBlkCol definition
+
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+}
+
+typedef struct {
+  int32_t   delimiter;  // For recovery usage
+  int32_t   numOfCols;  // For recovery usage
+  uint64_t  uid;        // For recovery usage
+  SBlockCol cols[];
+} SBlockData;
+
+typedef void SAggrBlkData;  // SBlockCol cols[];
+
+struct SReadH {
+  STsdb *   pRepo;
+  SDFileSet     rSet;     // FSET to read
+  SArray *      aBlkIdx;  // SBlockIdx array
+  STable *      pTable;   // table to read
+  SBlockIdx *   pBlkIdx;  // current reading table SBlockIdx
+  int           cidx;
+  SBlockInfo *  pBlkInfo;      // SBlockInfoV#
+  SBlockData *  pBlkData;      // Block info
+  SAggrBlkData *pAggrBlkData;  // Aggregate Block info
+  SDataCols *   pDCols[2];
+  void *        pBuf;    // buffer
+  void *        pCBuf;   // compression buffer
+  void *        pExBuf;  // extra buffer
+};
+
+#define TSDB_READ_REPO(rh) ((rh)->pRepo)
+#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
+#define TSDB_READ_FSET(rh) (&((rh)->rSet))
+#define TSDB_READ_TABLE(rh) ((rh)->pTable)
+#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
+#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
+#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
+#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
+#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
+#define TSDB_READ_BUF(rh) ((rh)->pBuf)
+#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
+#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf)
+
+#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
+  (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+      return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
+    case TSDB_SBLK_VER_1:
+    default:
+      return TSDB_BLOCK_STATIS_SIZE(nCols, 1);
+  }
+}
+
+#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+      ASSERT(false);
+      return 0;
+    case TSDB_SBLK_VER_1:
+    default:
+      return TSDB_BLOCK_AGGR_SIZE(nCols, 1);
+  }
+}
+
+int   tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
+void  tsdbDestroyReadH(SReadH *pReadh);
+int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
+void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
+int   tsdbLoadBlockIdx(SReadH *pReadh);
+int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen);
+int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
+int   tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
+int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
+void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
+
+static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
+  void * pBuf = *ppBuf;
+  size_t tsize = taosTSizeof(pBuf);
+
+  if (tsize < size) {
+    if (tsize == 0) tsize = 1024;
+
+    while (tsize < size) {
+      tsize *= 2;
+    }
+
+    *ppBuf = taosTRealloc(pBuf, tsize);
+    if (*ppBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE SBlockCol *tsdbGetSBlockCol(SBlock *pBlock, SBlockCol **pDestBlkCol, SBlockCol *pBlkCols,
+                                                int colIdx) {
+  if (pBlock->blkVer == SBlockVerLatest) {
+    *pDestBlkCol = pBlkCols + colIdx;
+    return *pDestBlkCol;
+  }
+  if (pBlock->blkVer == TSDB_SBLK_VER_0) {
+    SBlockColV0 *pBlkCol = (SBlockColV0 *)pBlkCols + colIdx;
+    (*pDestBlkCol)->colId = pBlkCol->colId;
+    (*pDestBlkCol)->len = pBlkCol->len;
+    (*pDestBlkCol)->type = pBlkCol->type;
+    (*pDestBlkCol)->offset = pBlkCol->offset;
+    (*pDestBlkCol)->offsetH = pBlkCol->offsetH;
+  }
+  return *pDestBlkCol;
+}
+
+#endif /*_TD_TSDB_READ_IMPL_H_*/

source/dnode/vnode/tsdb2/inc/tsdbRowMergeBuf.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TSDB_ROW_MERGE_BUF_H
+#define TSDB_ROW_MERGE_BUF_H 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tsdb.h"
+#include "tchecksum.h"
+#include "tsdbReadImpl.h"
+
+typedef void* SMergeBuf;
+
+SDataRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
+
+static FORCE_INLINE int tsdbMergeBufMakeSureRoom(SMergeBuf *pBuf, STSchema* pSchema1, STSchema* pSchema2) {
+  size_t len1 = dataRowMaxBytesFromSchema(pSchema1);
+  size_t len2 = dataRowMaxBytesFromSchema(pSchema2);
+  return tsdbMakeRoom(pBuf, MAX(len1, len2));
+}
+
+static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) {
+  taosTZfree(buf);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef TSDB_ROW_MERGE_BUF_H */

source/dnode/vnode/tsdb2/inc/tsdbint.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_INT_H_
+#define _TD_TSDB_INT_H_
+
+#include "os.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tarray.h"
+#include "tchecksum.h"
+#include "tcoding.h"
+#include "tcompression.h"
+#include "tdataformat.h"
+#include "tfs.h"
+#include "thash.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tsdbMemory.h"
+#include "tskiplist.h"
+
+#include "tsdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Log
+#include "tsdbLog.h"
+// Meta
+#include "tsdbMeta.h"
+// // Buffer
+// #include "tsdbBuffer.h"
+// MemTable
+#include "tsdbMemTable.h"
+// File
+#include "tsdbFile.h"
+// FS
+#include "tsdbFS.h"
+// ReadImpl
+#include "tsdbReadImpl.h"
+// Commit
+#include "tsdbCommit.h"
+// Compact
+#include "tsdbCompact.h"
+
+#include "tsdbRowMergeBuf.h"
+// Main definitions
+struct STsdb {
+  uint8_t    state;
+  STsdbCfg   config;
+  STsdbStat  stat;
+  STsdbMeta* tsdbMeta;
+  SMemTable* mem;
+  SMemTable* imem;
+  STsdbFS*   fs;
+  SRtn       rtn;
+  SMergeBuf  mergeBuf;  // used when update=2
+};
+
+#define REPO_ID(r) (r)->config.tsdbId
+#define REPO_CFG(r) (&((r)->config))
+#define REPO_FS(r) ((r)->fs)
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+int             tsdbLockRepo(STsdb* pRepo);
+int             tsdbUnlockRepo(STsdb* pRepo);
+STsdbMeta*      tsdbGetMeta(STsdb* pRepo);
+int             tsdbCheckCommit(STsdb* pRepo);
+int             tsdbRestoreInfo(STsdb* pRepo);
+UNUSED_FUNC int tsdbCacheLastData(STsdb* pRepo, STsdbCfg* oldCfg);
+int32_t         tsdbLoadLastCache(STsdb* pRepo, STable* pTable);
+void            tsdbGetRootDir(int repoid, char dirName[]);
+void            tsdbGetDataDir(int repoid, char dirName[]);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TD_TSDB_INT_H_ */

source/dnode/vnode/tsdb2/src/tsdbBuffer.c

@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if 0
+
+#include "tsdbHealth.h"
+#include "tsdbint.h"
+
+#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
+
+// ---------------- INTERNAL FUNCTIONS ----------------
+STsdbBufPool *tsdbNewBufPool() {
+  STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool));
+  if (pBufPool == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL);
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    goto _err;
+  }
+
+  pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *));
+  if (pBufPool->bufBlockList == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  return pBufPool;
+
+_err:
+  tsdbFreeBufPool(pBufPool);
+  return NULL;
+}
+
+void tsdbFreeBufPool(STsdbBufPool *pBufPool) {
+  if (pBufPool) {
+    if (pBufPool->bufBlockList) {
+      ASSERT(listNEles(pBufPool->bufBlockList) == 0);
+      tdListFree(pBufPool->bufBlockList);
+    }
+
+    pthread_cond_destroy(&pBufPool->poolNotEmpty);
+
+    free(pBufPool);
+  }
+}
+
+int tsdbOpenBufPool(STsdb *pRepo) {
+  STsdbCfg *    pCfg = &(pRepo->config);
+  STsdbBufPool *pPool = pRepo->pPool;
+
+  ASSERT(pPool != NULL);
+  pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024;  // MB
+  pPool->tBufBlocks = pCfg->totalBlocks;
+  pPool->nBufBlocks = 0;
+  pPool->nElasticBlocks = 0;
+  pPool->index = 0;
+  pPool->nRecycleBlocks = 0;
+
+  for (int i = 0; i < pCfg->totalBlocks; i++) {
+    STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
+    if (pBufBlock == NULL) goto _err;
+
+    if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
+      tsdbFreeBufBlock(pBufBlock);
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      goto _err;
+    }
+
+    pPool->nBufBlocks++;
+  }
+
+  tsdbDebug("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo),
+            pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks);
+
+  return 0;
+
+_err:
+  tsdbCloseBufPool(pRepo);
+  return -1;
+}
+
+void tsdbCloseBufPool(STsdb *pRepo) {
+  if (pRepo == NULL) return;
+
+  STsdbBufPool * pBufPool = pRepo->pPool;
+  STsdbBufBlock *pBufBlock = NULL;
+
+  if (pBufPool) {
+    SListNode *pNode = NULL;
+    while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) {
+      tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+      tsdbFreeBufBlock(pBufBlock);
+      free(pNode);
+    }
+  }
+
+  tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
+}
+
+SListNode *tsdbAllocBufBlockFromPool(STsdb *pRepo) {
+  ASSERT(pRepo != NULL && pRepo->pPool != NULL);
+  ASSERT(IS_REPO_LOCKED(pRepo));
+
+  STsdbBufPool *pBufPool = pRepo->pPool;
+
+  while (POOL_IS_EMPTY(pBufPool)) {
+    if (tsDeadLockKillQuery) {
+      // supply new Block
+      if (tsdbInsertNewBlock(pRepo) > 0) {
+        tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo),
+                 pBufPool->nElasticBlocks, TD_DLIST_NELES(pBufPool->bufBlockList));
+        break;
+      } else {
+        // no newBlock, kill query free
+        if (!tsdbUrgeQueryFree(pRepo)) tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo));
+      }
+    }
+
+    pRepo->repoLocked = false;
+    pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
+    pRepo->repoLocked = true;
+  }
+
+  SListNode *pNode = tdListPopHead(pBufPool->bufBlockList);
+  ASSERT(pNode != NULL);
+  STsdbBufBlock *pBufBlock = NULL;
+  tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
+
+  pBufBlock->blockId = pBufPool->index++;
+  pBufBlock->offset = 0;
+  pBufBlock->remain = pBufPool->bufBlockSize;
+
+  tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
+  return pNode;
+}
+
+// ---------------- LOCAL FUNCTIONS ----------------
+STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) {
+  STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize);
+  if (pBufBlock == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pBufBlock->blockId = 0;
+  pBufBlock->offset = 0;
+  pBufBlock->remain = bufBlockSize;
+
+  return pBufBlock;
+}
+
+void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
+
+int tsdbExpandPool(STsdb *pRepo, int32_t oldTotalBlocks) {
+  if (oldTotalBlocks == pRepo->config.totalBlocks) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int err = TSDB_CODE_SUCCESS;
+
+  if (tsdbLockRepo(pRepo) < 0) return terrno;
+  STsdbBufPool *pPool = pRepo->pPool;
+
+  if (pRepo->config.totalBlocks > oldTotalBlocks) {
+    for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) {
+      STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
+      if (pBufBlock == NULL) goto err;
+
+      if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
+        tsdbFreeBufBlock(pBufBlock);
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        err = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        goto err;
+      }
+
+      pPool->nBufBlocks++;
+    }
+    pthread_cond_signal(&pPool->poolNotEmpty);
+  } else {
+    pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks;
+  }
+
+err:
+  tsdbUnlockRepo(pRepo);
+  return err;
+}
+
+void tsdbRecycleBufferBlock(STsdbBufPool *pPool, SListNode *pNode, bool bELastic) {
+  STsdbBufBlock *pBufBlock = NULL;
+  tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock));
+  tsdbFreeBufBlock(pBufBlock);
+  free(pNode);
+  if (bELastic) {
+    pPool->nElasticBlocks--;
+    tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks,
+             TD_DLIST_NELES(pPool->bufBlockList));
+  } else
+    pPool->nBufBlocks--;
+}
+
+#endif

source/dnode/vnode/tsdb2/src/tsdbCompact.c

@@ -0,0 +1,543 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if 0
+#include "tsdbint.h"
+
+typedef struct {
+  STable *    pTable;
+  SBlockIdx * pBlkIdx;
+  SBlockIdx   bindex;
+  SBlockInfo *pInfo;
+} STableCompactH;
+
+typedef struct {
+  SRtn       rtn;
+  SFSIter    fsIter;
+  SArray *   tbArray;  // table array to cache table obj and block indexes
+  SReadH     readh;
+  SDFileSet  wSet;
+  SArray *   aBlkIdx;
+  SArray *   aSupBlk;
+  SDataCols *pDataCols;
+} SCompactH;
+
+#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet))
+#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh))
+#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
+#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
+#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
+#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
+#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
+#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh))
+#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh))
+#define TSDB_COMPACT_EXBUF(pComph) TSDB_READ_EXBUF(&((pComph)->readh))
+
+// static int  tsdbAsyncCompact(STsdb *pRepo);
+static void tsdbStartCompact(STsdb *pRepo);
+static void tsdbEndCompact(STsdb *pRepo, int eno);
+static int  tsdbCompactMeta(STsdb *pRepo);
+static int  tsdbCompactTSData(STsdb *pRepo);
+static int  tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet);
+static bool tsdbShouldCompact(SCompactH *pComph);
+static int  tsdbInitCompactH(SCompactH *pComph, STsdb *pRepo);
+static void tsdbDestroyCompactH(SCompactH *pComph);
+static int  tsdbInitCompTbArray(SCompactH *pComph);
+static void tsdbDestroyCompTbArray(SCompactH *pComph);
+static int  tsdbCacheFSetIndex(SCompactH *pComph);
+static int  tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet);
+static void tsdbCompactFSetEnd(SCompactH *pComph);
+static int  tsdbCompactFSetImpl(SCompactH *pComph);
+static int  tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
+                                      void **ppCBuf, void **ppExBuf);
+
+enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
+// int tsdbCompact(STsdb *pRepo) { return tsdbAsyncCompact(pRepo); }
+
+void *tsdbCompactImpl(STsdb *pRepo) {
+  // Check if there are files in TSDB FS to compact
+  if (REPO_FS(pRepo)->cstatus->pmf == NULL) {
+    pRepo->compactState = TSDB_NO_COMPACT;
+    tsem_post(&(pRepo->readyToCommit));
+    tsdbInfo("vgId:%d compact over, no file to compact in FS", REPO_ID(pRepo));
+    return NULL;
+  }
+
+  tsdbStartCompact(pRepo);
+
+  if (tsdbCompactMeta(pRepo) < 0) {
+    tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbCompactTSData(pRepo) < 0) {
+    tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS);
+  return NULL;
+
+_err:
+  pRepo->code = terrno;
+  tsdbEndCompact(pRepo, terrno);
+  return NULL;
+}
+
+// static int tsdbAsyncCompact(STsdb *pRepo) {
+//   if (pRepo->compactState != TSDB_NO_COMPACT) {
+//     tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
+//     return 0; 
+//   } 
+//   pRepo->compactState = TSDB_WAITING_COMPACT;   
+//   tsem_wait(&(pRepo->readyToCommit));
+//   return tsdbScheduleCommit(pRepo, COMPACT_REQ);
+// }
+
+static void tsdbStartCompact(STsdb *pRepo) {
+  assert(pRepo->compactState != TSDB_IN_COMPACT);
+  tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
+  tsdbStartFSTxn(pRepo, 0, 0);
+  pRepo->code = TSDB_CODE_SUCCESS;
+  pRepo->compactState = TSDB_IN_COMPACT;
+}
+
+static void tsdbEndCompact(STsdb *pRepo, int eno) {
+  if (eno != TSDB_CODE_SUCCESS) {
+    tsdbEndFSTxnWithError(REPO_FS(pRepo));
+  } else {
+    tsdbEndFSTxn(pRepo);
+  }
+  pRepo->compactState = TSDB_NO_COMPACT;
+  tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
+  tsem_post(&(pRepo->readyToCommit));
+}
+
+static int tsdbCompactMeta(STsdb *pRepo) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  tsdbUpdateMFile(pfs, pfs->cstatus->pmf);
+  return 0;
+}
+
+  static int tsdbCompactTSData(STsdb *pRepo) {
+    SCompactH  compactH;
+    SDFileSet *pSet = NULL;
+
+    tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo));
+
+    // If no file, just return 0;
+    if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) {
+      tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo));
+      return 0;
+    }
+
+    if (tsdbInitCompactH(&compactH, pRepo) < 0) {
+      return -1;
+    }
+
+    while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) {
+      // Remove those expired files
+      if (pSet->fid < compactH.rtn.minFid) {
+        tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
+                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
+        continue;
+      }
+
+      if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) {
+        tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL);
+        tsdbUpdateDFileSet(REPO_FS(pRepo), pSet);
+        continue;
+      }
+
+      if (tsdbCompactFSet(&compactH, pSet) < 0) {
+        tsdbDestroyCompactH(&compactH);
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        return -1;
+      }
+    }
+
+    tsdbDestroyCompactH(&compactH);
+    tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo));
+    return 0;
+  }
+
+  static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) {
+    STsdb *pRepo = TSDB_COMPACT_REPO(pComph);
+    SDiskID    did;
+
+    tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet),
+              TSDB_FSET_ID(pSet));
+
+    if (tsdbCompactFSetInit(pComph, pSet) < 0) {
+      return -1;
+    }
+
+    if (!tsdbShouldCompact(pComph)) {
+      tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid);
+      if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) {
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+    } else {
+      // Create new fset as compacted fset
+      tfsAllocDisk(tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &(did.level), &(did.id));
+      if (did.level == TFS_UNDECIDED_LEVEL) {
+        terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet),
+                      FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_LATEST_FSET_VER);
+      if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) {
+        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      if (tsdbCompactFSetImpl(pComph) < 0) {
+        tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+        tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph));
+        tsdbCompactFSetEnd(pComph);
+        return -1;
+      }
+
+      tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+      tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph));
+      tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid);
+    }
+
+    tsdbCompactFSetEnd(pComph);
+    return 0;
+  }
+
+  static bool tsdbShouldCompact(SCompactH *pComph) {
+    // if (tsdbForceCompactFile) {
+    //   return true;
+    // }
+    STsdb *     pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg *      pCfg = REPO_CFG(pRepo);
+    SReadH *        pReadh = &(pComph->readh);
+    STableCompactH *pTh;
+    SBlock *        pBlock;
+    int             defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
+    SDFile *        pDataF = TSDB_READ_DATA_FILE(pReadh);
+    SDFile *        pLastF = TSDB_READ_LAST_FILE(pReadh);
+
+    int     tblocks = 0;       // total blocks
+    int     nSubBlocks = 0;    // # of blocks with sub-blocks
+    int     nSmallBlocks = 0;  // # of blocks with rows < defaultRows
+    int64_t tsize = 0;
+
+    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
+      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
+
+      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
+
+      for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) {
+        tblocks++;
+        pBlock = pTh->pInfo->blocks + bidx;
+
+        if (pBlock->numOfRows < defaultRows) {
+          nSmallBlocks++;
+        }
+
+        if (pBlock->numOfSubBlocks > 1) {
+          nSubBlocks++;
+          for (int k = 0; k < pBlock->numOfSubBlocks; k++) {
+            SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k;
+            tsize = tsize + iBlock->len;
+          }
+        } else if (pBlock->numOfSubBlocks == 1) {
+          tsize += pBlock->len;
+        } else {
+          ASSERT(0);
+        }
+      }
+    }
+
+    return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) ||
+            (tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85));
+  }
+
+  static int tsdbInitCompactH(SCompactH *pComph, STsdb *pRepo) {
+    STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+    memset(pComph, 0, sizeof(*pComph));
+
+    TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph));
+
+    tsdbGetRtnSnap(pRepo, &(pComph->rtn));
+    tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
+
+    if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) {
+      return -1;
+    }
+
+    if (tsdbInitCompTbArray(pComph) < 0) {
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+    if (pComph->aBlkIdx == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
+    if (pComph->aSupBlk == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+    if (pComph->pDataCols == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbDestroyCompactH(pComph);
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static void tsdbDestroyCompactH(SCompactH *pComph) {
+    pComph->pDataCols = tdFreeDataCols(pComph->pDataCols);
+    pComph->aSupBlk = taosArrayDestroy(pComph->aSupBlk);
+    pComph->aBlkIdx = taosArrayDestroy(pComph->aBlkIdx);
+    tsdbDestroyCompTbArray(pComph);
+    tsdbDestroyReadH(&(pComph->readh));
+    tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
+  }
+
+  static int tsdbInitCompTbArray(SCompactH *pComph) {  // Init pComp->tbArray
+    STsdb *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+    if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
+
+    pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH));
+    if (pComph->tbArray == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbUnlockRepoMeta(pRepo);
+      return -1;
+    }
+
+    // Note here must start from 0
+    for (int i = 0; i < pMeta->maxTables; i++) {
+      STableCompactH ch = {0};
+      if (pMeta->tables[i] != NULL) {
+        tsdbRefTable(pMeta->tables[i]);
+        ch.pTable = pMeta->tables[i];
+      }
+
+      if (taosArrayPush(pComph->tbArray, &ch) == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tsdbUnlockRepoMeta(pRepo);
+        return -1;
+      }
+    }
+
+    if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
+    return 0;
+  }
+
+  static void tsdbDestroyCompTbArray(SCompactH *pComph) {
+    STableCompactH *pTh;
+
+    if (pComph->tbArray == NULL) return;
+
+    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
+      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
+      if (pTh->pTable) {
+        tsdbUnRefTable(pTh->pTable);
+      }
+
+      // pTh->pInfo = taosTZfree(pTh->pInfo);
+      tfree(pTh->pInfo);
+    }
+
+    pComph->tbArray = taosArrayDestroy(pComph->tbArray);
+  }
+
+  static int tsdbCacheFSetIndex(SCompactH *pComph) {
+    SReadH *pReadH = &(pComph->readh);
+
+    if (tsdbLoadBlockIdx(pReadH) < 0) {
+      return -1;
+    }
+
+    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
+      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
+      pTh->pBlkIdx = NULL;
+
+      if (pTh->pTable == NULL) continue;
+      if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) {
+        return -1;
+      }
+
+      if (pReadH->pBlkIdx == NULL) continue;
+      pTh->bindex = *(pReadH->pBlkIdx);
+      pTh->pBlkIdx = &(pTh->bindex);
+
+      uint32_t originLen = 0;
+      if (tsdbLoadBlockInfo(pReadH, (void **)(&(pTh->pInfo)), &originLen) < 0) {
+        return -1;
+      }
+    }
+
+    return 0;
+  }
+
+  static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) {
+    taosArrayClear(pComph->aBlkIdx);
+    taosArrayClear(pComph->aSupBlk);
+
+    if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) {
+      return -1;
+    }
+
+    if (tsdbCacheFSetIndex(pComph) < 0) {
+      tsdbCloseAndUnsetFSet(&(pComph->readh));
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); }
+
+  static int tsdbCompactFSetImpl(SCompactH *pComph) {
+    STsdb *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg * pCfg = REPO_CFG(pRepo);
+    SReadH *   pReadh = &(pComph->readh);
+    SBlockIdx  blkIdx;
+    void **    ppBuf = &(TSDB_COMPACT_BUF(pComph));
+    void **    ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph));
+    void **    ppExBuf = &(TSDB_COMPACT_EXBUF(pComph));
+    int        defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
+
+    taosArrayClear(pComph->aBlkIdx);
+
+    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
+      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
+      STSchema *      pSchema;
+
+      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
+
+      pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1, -1);
+      taosArrayClear(pComph->aSupBlk);
+      if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) ||
+          (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tdFreeSchema(pSchema);
+        return -1;
+      }
+      tdFreeSchema(pSchema);
+
+      // Loop to compact each block data
+      for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) {
+        SBlock *pBlock = pTh->pInfo->blocks + i;
+
+        // Load the block data
+        if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) {
+          return -1;
+        }
+
+        // Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file
+        if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) {
+          if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf, ppExBuf) < 0) {
+            return -1;
+          }
+        } else {
+          int ridx = 0;
+
+          while (true) {
+            if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
+            int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
+
+            tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);
+
+            if (pComph->pDataCols->numOfRows < defaultRows) {
+              break;
+            }
+
+            if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf, ppExBuf) < 0) {
+              return -1;
+            }
+            tdResetDataCols(pComph->pDataCols);
+          }
+        }
+      }
+
+      if (pComph->pDataCols->numOfRows > 0 &&
+          tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf, ppExBuf) < 0) {
+        return -1;
+      }
+
+      if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) <
+          0) {
+        return -1;
+      }
+
+      if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        return -1;
+      }
+    }
+
+    if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) {
+      return -1;
+    }
+
+    return 0;
+  }
+
+  static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
+                                       void **ppCBuf, void **ppExBuf) {
+    STsdb *pRepo = TSDB_COMPACT_REPO(pComph);
+    STsdbCfg * pCfg = REPO_CFG(pRepo);
+    SDFile *   pDFile;
+    bool       isLast;
+    SBlock     block;
+
+    ASSERT(pDataCols->numOfRows > 0);
+
+    if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
+      pDFile = TSDB_COMPACT_LAST_FILE(pComph);
+      isLast = true;
+    } else {
+      pDFile = TSDB_COMPACT_DATA_FILE(pComph);
+      isLast = false;
+    }
+
+    if (tsdbWriteBlockImpl(pRepo, pTable, pDFile,
+                           isLast ? TSDB_COMPACT_SMAL_FILE(pComph) : TSDB_COMPACT_SMAD_FILE(pComph), pDataCols, &block,
+                           isLast, true, ppBuf, ppCBuf, ppExBuf) < 0) {
+      return -1;
+    }
+
+    if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    return 0;
+  }
+
+#endif

source/dnode/vnode/tsdb2/src/tsdbFile.c

@@ -0,0 +1,719 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+
+static const char *TSDB_FNAME_SUFFIX[] = {
+    "head",  // TSDB_FILE_HEAD
+    "data",  // TSDB_FILE_DATA
+    "last",  // TSDB_FILE_LAST
+    "smad",  // TSDB_FILE_SMA_DATA(Small Materialized Aggregate for .data File)
+    "smal",  // TSDB_FILE_SMA_LAST(Small Materialized Aggregate for .last File)
+    "",      // TSDB_FILE_MAX
+    "meta",  // TSDB_FILE_META
+};
+
+static void  tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
+static int   tsdbRollBackMFile(SMFile *pMFile);
+static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
+static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo, TSDB_FVER_TYPE sfver);
+static int   tsdbRollBackDFile(SDFile *pDFile);
+
+// ============== SMFile
+void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
+  char fname[TSDB_FILENAME_LEN];
+
+  TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
+
+  memset(&(pMFile->info), 0, sizeof(pMFile->info));
+  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
+
+  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
+  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
+}
+
+void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
+  *pMFile = *pOMFile;
+  TSDB_FILE_SET_CLOSED(pMFile);
+}
+
+int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
+  int tlen = 0;
+
+  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
+  tlen += tfsEncodeFile(buf, &(pMFile->f));
+
+  return tlen;
+}
+
+void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
+  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
+  buf = tfsDecodeFile(buf, &(pMFile->f));
+  TSDB_FILE_SET_CLOSED(pMFile);
+
+  return buf;
+}
+
+int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
+  int tlen = 0;
+
+  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
+  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));
+
+  return tlen;
+}
+
+void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
+  char *aname;
+  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
+  buf = taosDecodeString(buf, &aname);
+  tstrncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
+  TSDB_FILE_SET_CLOSED(pMFile);
+
+  tfree(aname);
+
+  return buf;
+}
+
+int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
+  if (from == NULL && to == NULL) return 0;
+
+  if (from != NULL) {
+    if (to == NULL) {
+      return tsdbRemoveMFile(from);
+    } else {
+      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
+        if (from->info.size > to->info.size) {
+          tsdbRollBackMFile(to);
+        }
+      } else {
+        return tsdbRemoveMFile(from);
+      }
+    }
+  }
+
+  return 0;
+}
+
+int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
+  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  if (pMFile->fd < 0) {
+    if (errno == ENOENT) {
+      // Try to create directory recursively
+      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
+      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
+
+      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+      if (pMFile->fd < 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+    } else {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
+  }
+
+  if (!updateHeader) {
+    return 0;
+  }
+
+  pMFile->info.size += TSDB_FILE_HEAD_SIZE;
+
+  if (tsdbUpdateMFileHeader(pMFile) < 0) {
+    tsdbCloseMFile(pMFile);
+    tsdbRemoveMFile(pMFile);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbUpdateMFileHeader(SMFile *pMFile) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+
+  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  void *ptr = buf;
+  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
+
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  tsdbDecodeMFInfo(buf, pInfo);
+  return 0;
+}
+
+int tsdbScanAndTryFixMFile(STsdb *pRepo) {
+  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
+  struct stat mfstat;
+  SMFile      mf;
+
+  if (pMFile == NULL) {
+    // No meta file, no need to scan
+    return 0;
+  }
+
+  tsdbInitMFileEx(&mf, pMFile);
+
+  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
+    tsdbError("vgId:%d meta file %s not exist, report to upper layer to fix it", REPO_ID(pRepo),
+              TSDB_FILE_FULL_NAME(pMFile));
+    pRepo->state |= TSDB_STATE_BAD_META;
+    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
+    return 0;
+  }
+
+  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (pMFile->info.size < mfstat.st_size) {
+    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+      return -1;
+    }
+
+    if (taosFtruncateFile(mf.fd, mf.info.size) < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbCloseMFile(&mf);
+      return -1;
+    }
+
+    if (tsdbUpdateMFileHeader(&mf) < 0) {
+      tsdbCloseMFile(&mf);
+      return -1;
+    }
+
+    tsdbCloseMFile(&mf);
+    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+             mfstat.st_size, pMFile->info.size);
+  } else if (pMFile->info.size > mfstat.st_size) {
+    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
+              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
+    pRepo->state |= TSDB_STATE_BAD_META;
+    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return 0;
+  } else {
+    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
+  }
+
+  return 0;
+}
+
+int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI64(buf, pInfo->size);
+  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
+  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
+  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
+  tlen += taosEncodeFixedU32(buf, pInfo->magic);
+
+  return tlen;
+}
+
+void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
+  buf = taosDecodeVariantI64(buf, &(pInfo->size));
+  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
+  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
+  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
+  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
+
+  return buf;
+}
+
+static int tsdbRollBackMFile(SMFile *pMFile) {
+  SMFile mf;
+
+  tsdbInitMFileEx(&mf, pMFile);
+
+  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+    return -1;
+  }
+
+  if (taosFtruncateFile(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbCloseMFile(&mf);
+    return -1;
+  }
+
+  if (tsdbUpdateMFileHeader(&mf) < 0) {
+    tsdbCloseMFile(&mf);
+    return -1;
+  }
+
+  TSDB_FILE_FSYNC(&mf);
+
+  tsdbCloseMFile(&mf);
+  return 0;
+}
+
+// ============== Operations on SDFile
+void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) {
+  char fname[TSDB_FILENAME_LEN];
+
+  TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK);
+
+  TSDB_FILE_SET_CLOSED(pDFile);
+
+  memset(&(pDFile->info), 0, sizeof(pDFile->info));
+  pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
+  pDFile->info.fver = tsdbGetDFSVersion(ftype);
+
+  tsdbGetFilename(vid, fid, ver, ftype, fname);
+  tfsInitFile(&(pDFile->f), did.level, did.id, fname);
+}
+
+void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
+  *pDFile = *pODFile;
+  TSDB_FILE_SET_CLOSED(pDFile);
+}
+
+int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
+  int tlen = 0;
+
+  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
+  tlen += tfsEncodeFile(buf, &(pDFile->f));
+
+  return tlen;
+}
+
+void *tsdbDecodeSDFile(void *buf, SDFile *pDFile, uint32_t sfver) {
+  buf = tsdbDecodeDFInfo(buf, &(pDFile->info), sfver);
+  buf = tfsDecodeFile(buf, &(pDFile->f));
+  TSDB_FILE_SET_CLOSED(pDFile);
+
+  return buf;
+}
+
+static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) {
+  int tlen = 0;
+
+  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
+  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile));
+
+  return tlen;
+}
+
+static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
+  char *aname;
+  // The sync module would send DFileSet with latest verion.
+  buf = tsdbDecodeDFInfo(buf, &(pDFile->info), TSDB_LATEST_SFS_VER);
+  buf = taosDecodeString(buf, &aname);
+  tstrncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN);
+  TSDB_FILE_SET_CLOSED(pDFile);
+  tfree(aname);
+
+  return buf;
+}
+
+int tsdbCreateDFile(SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) {
+  ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  if (pDFile->fd < 0) {
+    if (errno == ENOENT) {
+      // Try to create directory recursively
+      char *s = strdup(TFILE_REL_NAME(&(pDFile->f)));
+      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
+
+      pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+      if (pDFile->fd < 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+    } else {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
+  }
+
+  if (!updateHeader) {
+    return 0;
+  }
+
+  pDFile->info.size += TSDB_FILE_HEAD_SIZE;
+  pDFile->info.fver = tsdbGetDFSVersion(fType);
+
+  if (tsdbUpdateDFileHeader(pDFile) < 0) {
+    tsdbCloseDFile(pDFile);
+    tsdbRemoveDFile(pDFile);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbUpdateDFileHeader(SDFile *pDFile) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+
+  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  void *ptr = buf;
+  tsdbEncodeDFInfo(&ptr, &(pDFile->info));
+
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+  if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
+  char     buf[TSDB_FILE_HEAD_SIZE] = "\0";
+  // uint32_t _version;
+
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  void *pBuf = buf;
+  pBuf = tsdbDecodeDFInfo(pBuf, pInfo, TSDB_LATEST_FVER);  // only make sure the parameter sfver > 0
+  return 0;
+}
+
+static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
+  struct stat dfstat;
+  SDFile      df;
+
+  tsdbInitDFileEx(&df, pDFile);
+
+  if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
+    tsdbError("vgId:%d data file %s not exist, report to upper layer to fix it", REPO_ID(pRepo),
+              TSDB_FILE_FULL_NAME(pDFile));
+    pRepo->state |= TSDB_STATE_BAD_DATA;
+    TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
+    return 0;
+  }
+
+  if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (pDFile->info.size < dfstat.st_size) {
+    if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
+      return -1;
+    }
+
+    if (taosFtruncateFile(df.fd, df.info.size) < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbCloseDFile(&df);
+      return -1;
+    }
+
+    if (tsdbUpdateDFileHeader(&df) < 0) {
+      tsdbCloseDFile(&df);
+      return -1;
+    }
+
+    tsdbCloseDFile(&df);
+    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+             dfstat.st_size, pDFile->info.size);
+  } else if (pDFile->info.size > dfstat.st_size) {
+    tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
+              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
+    pRepo->state |= TSDB_STATE_BAD_DATA;
+    TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return 0;
+  } else {
+    tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile));
+  }
+
+  return 0;
+}
+
+static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) {
+  int tlen = 0;
+  tlen += taosEncodeFixedU32(buf, pInfo->fver);
+  tlen += taosEncodeFixedU32(buf, pInfo->magic);
+  tlen += taosEncodeFixedU32(buf, pInfo->len);
+  tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
+  tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
+  tlen += taosEncodeFixedU32(buf, pInfo->offset);
+  tlen += taosEncodeFixedU64(buf, pInfo->size);
+  tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
+
+  return tlen;
+}
+
+static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo, TSDB_FVER_TYPE sfver) {
+  if (sfver > TSDB_FS_VER_0) {
+    buf = taosDecodeFixedU32(buf, &(pInfo->fver));
+  } else {
+    pInfo->fver = TSDB_FS_VER_0;  // default value
+  }
+  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
+  buf = taosDecodeFixedU32(buf, &(pInfo->len));
+  buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
+  buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
+  buf = taosDecodeFixedU32(buf, &(pInfo->offset));
+  buf = taosDecodeFixedU64(buf, &(pInfo->size));
+  buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
+
+  return buf;
+}
+
+static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
+  ASSERT(from != NULL || to != NULL);
+
+  if (from != NULL) {
+    if (to == NULL) {
+      tsdbRemoveDFile(from);
+    } else {
+      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
+        if (from->info.size > to->info.size) {
+          tsdbRollBackDFile(to);
+        }
+      } else {
+        (void)tsdbRemoveDFile(from);
+      }
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbRollBackDFile(SDFile *pDFile) {
+  SDFile df = *pDFile;
+
+  if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
+    return -1;
+  }
+
+  if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbCloseDFile(&df);
+    return -1;
+  }
+
+  if (tsdbUpdateDFileHeader(&df) < 0) {
+    tsdbCloseDFile(&df);
+    return -1;
+  }
+
+  TSDB_FILE_FSYNC(&df);
+
+  tsdbCloseDFile(&df);
+  return 0;
+}
+
+// ============== Operations on SDFileSet
+void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver, uint16_t fsetVer) {
+  pSet->fid = fid;
+  pSet->state = 0;
+  pSet->ver = fsetVer;
+
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
+    tsdbInitDFile(pDFile, did, vid, fid, ver, ftype);
+  }
+}
+
+void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pOSet);
+  pSet->fid = pOSet->fid;
+  pSet->ver = pOSet->ver;
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype));
+  }
+}
+
+int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
+  int tlen = 0;
+
+  tlen += taosEncodeFixedI32(buf, pSet->fid);
+  tlen += taosEncodeFixedU16(buf, pSet->ver);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+
+  return tlen;
+}
+
+void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet, uint32_t sfver) {
+  int32_t fid;
+
+  buf = taosDecodeFixedI32(buf, &(fid));
+  pSet->state = 0;
+  pSet->fid = fid;
+
+  if (sfver > TSDB_FS_VER_0) {
+    buf = taosDecodeFixedU16(buf, &(pSet->ver));
+  }
+
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype), sfver);
+  }
+  return buf;
+}
+
+int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) {
+  int tlen = 0;
+
+  tlen += taosEncodeFixedI32(buf, pSet->fid);
+  tlen += taosEncodeFixedU16(buf, pSet->ver);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+
+  return tlen;
+}
+
+void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) {
+  int32_t fid;
+
+  buf = taosDecodeFixedI32(buf, &(fid));
+  buf = taosDecodeFixedU16(buf, &(pSet->ver));
+  pSet->fid = fid;
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+  return buf;
+}
+
+int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
+  uint8_t nFilesFrom = from ? tsdbGetNFiles(from) : 0;
+  uint8_t nFilesTo = to ? tsdbGetNFiles(to) : 0;
+  for (TSDB_FILE_T ftype = 0; ftype < MAX(nFilesFrom, nFilesTo); ftype++) {
+    SDFile *pDFileFrom = ftype < nFilesFrom ? TSDB_DFILE_IN_SET(from, ftype) : NULL;
+    SDFile *pDFileTo = ftype < nFilesTo ? TSDB_DFILE_IN_SET(to, ftype) : NULL;
+    if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) {
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader, ftype) < 0) {
+      tsdbCloseDFileSet(pSet);
+      tsdbRemoveDFileSet(pSet);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet) {
+  ASSERT_TSDB_FSET_NFILES_VALID(pSet);
+  for (TSDB_FILE_T ftype = 0; ftype < tsdbGetNFiles(pSet); ftype++) {
+    if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *_version) {
+  char *p = NULL;
+  *_version = 0;
+  *ftype = TSDB_FILE_MAX;
+
+  sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, _version);
+  for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) {
+    if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) {
+      *ftype = i;
+      break;
+    }
+  }
+
+  tfree(p);
+  return 0;
+}
+
+static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) {
+  ASSERT(ftype != TSDB_FILE_MAX);
+
+  if (ftype < TSDB_FILE_MAX) {
+    if (ver == 0) {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]);
+    } else {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid,
+               TSDB_FNAME_SUFFIX[ftype], ver);
+    }
+  } else {
+    if (ver == 0) {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]);
+    } else {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
+    }
+  }
+}

source/dnode/vnode/tsdb2/src/tsdbReadImpl.c

@@ -0,0 +1,878 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+
+#define TSDB_KEY_COL_OFFSET 0
+
+static void tsdbResetReadTable(SReadH *pReadh);
+static void tsdbResetReadFile(SReadH *pReadh);
+static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
+static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                         int maxPoints, char *buffer, int bufferSize);
+static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                      int numOfColIds);
+static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
+static int  tsdbLoadBlockStatisFromDFile(SReadH *pReadh, SBlock *pBlock);
+static int  tsdbLoadBlockStatisFromAggr(SReadH *pReadh, SBlock *pBlock);
+
+int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
+  ASSERT(pReadh != NULL && pRepo != NULL);
+
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+  memset((void *)pReadh, 0, sizeof(*pReadh));
+  pReadh->pRepo = pRepo;
+
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+
+  pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+  if (pReadh->aBlkIdx == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[0] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[1] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbDestroyReadH(SReadH *pReadh) {
+  if (pReadh == NULL) return;
+  pReadh->pExBuf = taosTZfree(pReadh->pExBuf);
+  pReadh->pCBuf = taosTZfree(pReadh->pCBuf);
+  pReadh->pBuf = taosTZfree(pReadh->pBuf);
+  pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]);
+  pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]);
+  pReadh->pAggrBlkData = taosTZfree(pReadh->pAggrBlkData);
+  pReadh->pBlkData = taosTZfree(pReadh->pBlkData);
+  pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+  pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+  pReadh->pRepo = NULL;
+}
+
+int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
+  ASSERT(pSet != NULL);
+  tsdbResetReadFile(pReadh);
+
+  pReadh->rSet = *pSet;
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+  if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) {
+    tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet),
+              tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
+
+int tsdbLoadBlockIdx(SReadH *pReadh) {
+  SDFile *  pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx blkIdx;
+
+  ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
+
+  // No data at all, just return
+  if (pHeadf->info.offset <= 0) return 0;
+
+  if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (nread < pHeadf->info.len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len);
+    return -1;
+  }
+
+  void *ptr = TSDB_READ_BUF(pReadh);
+  int   tsize = 0;
+  while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) {
+    ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx);
+    ASSERT(ptr != NULL);
+
+    if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    tsize++;
+    ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
+                             ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
+  }
+
+  return 0;
+}
+
+int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1, -1);
+
+  pReadh->pTable = pTable;
+
+  if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  size_t size = taosArrayGetSize(pReadh->aBlkIdx);
+  if (size > 0) {
+    while (true) {
+      if (pReadh->cidx >= size) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      }
+
+      SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
+      if (pBlkIdx->tid == TABLE_TID(pTable)) {
+        if (pBlkIdx->uid == TABLE_UID(pTable)) {
+          pReadh->pBlkIdx = pBlkIdx;
+        } else {
+          pReadh->pBlkIdx = NULL;
+        }
+        pReadh->cidx++;
+        break;
+      } else if (pBlkIdx->tid > TABLE_TID(pTable)) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      } else {
+        pReadh->cidx++;
+      }
+    }
+  } else {
+    pReadh->pBlkIdx = NULL;
+  }
+
+  return 0;
+}
+
+#if 0
+int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
+  ASSERT(pReadh->pBlkIdx != NULL);
+
+  SDFile *   pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
+
+  if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (nread < pBlkIdx->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
+
+  if (pTarget) {
+    memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  }
+
+  return 0;
+}
+#endif
+
+static FORCE_INLINE int32_t tsdbGetSBlockVer(int32_t fver) {
+  switch (fver) {
+    case TSDB_FS_VER_0:
+      return TSDB_SBLK_VER_0;
+    case TSDB_FS_VER_1:
+      return TSDB_SBLK_VER_1;
+    default:
+      return SBlockVerLatest;
+  }
+}
+
+static FORCE_INLINE size_t tsdbSizeOfSBlock(int32_t sBlkVer) {
+  switch (sBlkVer) {
+    case TSDB_SBLK_VER_0:
+      return sizeof(SBlockV0);
+    case TSDB_SBLK_VER_1:
+      return sizeof(SBlockV1);
+    default:
+      return sizeof(SBlock);
+  }
+}
+
+static int tsdbSBlkInfoRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, SBlockIdx *pBlkIdx, uint32_t *dstBlkInfoLen) {
+  int sBlkVer = tsdbGetSBlockVer(pHeadf->info.fver);
+  if (sBlkVer > TSDB_SBLK_VER_0) {
+    *dstBlkInfoLen = pBlkIdx->len;
+    return TSDB_CODE_SUCCESS;
+  }
+  size_t originBlkSize = tsdbSizeOfSBlock(sBlkVer);
+  size_t nBlks = (pBlkIdx->len - sizeof(SBlockInfo)) / originBlkSize;
+
+  *dstBlkInfoLen = (uint32_t)(sizeof(SBlockInfo) + nBlks * sizeof(SBlock));
+
+  if (pBlkIdx->len == *dstBlkInfoLen) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  ASSERT(*dstBlkInfoLen >= pBlkIdx->len);
+
+  SBlockInfo *tmpBlkInfo = NULL;
+  if (tsdbMakeRoom((void **)(&tmpBlkInfo), *dstBlkInfoLen) < 0) return -1;
+  memset(tmpBlkInfo, 0, *dstBlkInfoLen);                 // the blkVer is set to 0
+  memcpy(tmpBlkInfo, *pDstBlkInfo, sizeof(SBlockInfo));  // copy header
+  uint32_t nSubBlks = 0;
+  for (int i = 0; i < nBlks; ++i) {
+    SBlock *tmpBlk = tmpBlkInfo->blocks + i;
+    memcpy(tmpBlk, POINTER_SHIFT((*pDstBlkInfo)->blocks, i * originBlkSize), originBlkSize);
+    if (i < pBlkIdx->numOfBlocks) {      // super blocks
+      if (tmpBlk->numOfSubBlocks > 1) {  // has sub blocks
+        tmpBlk->offset = sizeof(SBlockInfo) + (pBlkIdx->numOfBlocks + nSubBlks) * sizeof(SBlock);
+        nSubBlks += tmpBlk->numOfSubBlocks;
+      }
+    }
+    // TODO: update the fields if the SBlock definition change later
+  }
+
+  taosTZfree(*pDstBlkInfo);
+  *pDstBlkInfo = tmpBlkInfo;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen) {
+  ASSERT(pReadh->pBlkIdx != NULL);
+
+  SDFile *    pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx * pBlkIdx = pReadh->pBlkIdx;
+
+  if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&pReadh->pBlkInfo), pBlkIdx->len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (nread < pBlkIdx->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
+
+  uint32_t dstBlkInfoLen = 0;
+  if (tsdbSBlkInfoRefactor(pHeadf, &(pReadh->pBlkInfo), pBlkIdx, &dstBlkInfoLen) < 0) {
+    return -1;
+  }
+
+  if (extendedLen != NULL) {
+    if (pTarget != NULL) {
+      if (*extendedLen < dstBlkInfoLen) {
+        char *t = realloc(*pTarget, dstBlkInfoLen);
+        if (t == NULL) {
+          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+          return -1;
+        }
+        *pTarget = t;
+      }
+      memcpy(*pTarget, (void *)(pReadh->pBlkInfo), dstBlkInfoLen);
+    }
+    *extendedLen = dstBlkInfoLen;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+  int8_t update = pReadh->pRepo->config.update;
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+  int8_t update = pReadh->pRepo->config.update;
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, update != TD_ROW_PARTIAL_UPDATE) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+static int tsdbLoadBlockStatisFromDFile(SReadH *pReadh, SBlock *pBlock) {
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+
+  if (nread < size) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+  return 0;
+}
+
+static int tsdbLoadBlockStatisFromAggr(SReadH *pReadh, SBlock *pBlock) {
+  ASSERT((pBlock->blkVer > TSDB_SBLK_VER_0) && (pBlock->aggrStat));  // TODO: remove after pass all the test
+  SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh);
+
+  if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset,
+              tstrerror(terrno));
+    return -1;
+  }
+
+  size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
+  if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1;
+
+  int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);
+  if (nreadAggr < 0) {
+    tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno),
+              (uint64_t)pBlock->aggrOffset, sizeAggr);
+    return -1;
+  }
+
+  if (nreadAggr < sizeAggr) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr,
+              nreadAggr);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr);
+    return -1;
+  }
+  return 0;
+}
+
+int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
+  ASSERT(pBlock->numOfSubBlocks <= 1);
+
+  if (pBlock->blkVer > TSDB_SBLK_VER_0) {
+    if (pBlock->aggrStat) {
+      return tsdbLoadBlockStatisFromAggr(pReadh, pBlock);
+    }
+    return 1;
+  }
+  return tsdbLoadBlockStatisFromDFile(pReadh, pBlock);
+}
+
+int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
+  ASSERT(pBlock->numOfSubBlocks <= 1);
+  return tsdbLoadBlockStatisFromDFile(pReadh, pBlock);
+}
+
+int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pIdx->tid);
+  tlen += taosEncodeVariantU32(buf, pIdx->len);
+  tlen += taosEncodeVariantU32(buf, pIdx->offset);
+  tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
+  tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
+  tlen += taosEncodeFixedU64(buf, pIdx->uid);
+  tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
+
+  return tlen;
+}
+
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
+  uint8_t  hasLast = 0;
+  uint32_t numOfBlocks = 0;
+  uint64_t value = 0;
+
+  if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
+  if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
+  pIdx->hasLast = hasLast;
+  if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
+  pIdx->numOfBlocks = numOfBlocks;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->uid = (int64_t)value;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->maxKey = (TSKEY)value;
+
+  return buf;
+}
+
+void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock) {
+  if (pBlock->blkVer == TSDB_SBLK_VER_0) {
+    SBlockData *pBlockData = pReadh->pBlkData;
+
+    for (int i = 0, j = 0; i < numOfCols;) {
+      if (j >= pBlockData->numOfCols) {
+        pStatis[i].numOfNull = -1;
+        i++;
+        continue;
+      }
+      SBlockColV0 *pSBlkCol = ((SBlockColV0 *)(pBlockData->cols)) + j;
+      if (pStatis[i].colId == pSBlkCol->colId) {
+        pStatis[i].sum = pSBlkCol->sum;
+        pStatis[i].max = pSBlkCol->max;
+        pStatis[i].min = pSBlkCol->min;
+        pStatis[i].maxIndex = pSBlkCol->maxIndex;
+        pStatis[i].minIndex = pSBlkCol->minIndex;
+        pStatis[i].numOfNull = pSBlkCol->numOfNull;
+        i++;
+        j++;
+      } else if (pStatis[i].colId < pSBlkCol->colId) {
+        pStatis[i].numOfNull = -1;
+        i++;
+      } else {
+        j++;
+      }
+    }
+  } else if (pBlock->aggrStat) {
+    SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData;
+
+    for (int i = 0, j = 0; i < numOfCols;) {
+      if (j >= pBlock->numOfCols) {
+        pStatis[i].numOfNull = -1;
+        i++;
+        continue;
+      }
+      SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j;
+      if (pStatis[i].colId == pAggrBlkCol->colId) {
+        pStatis[i].sum = pAggrBlkCol->sum;
+        pStatis[i].max = pAggrBlkCol->max;
+        pStatis[i].min = pAggrBlkCol->min;
+        pStatis[i].maxIndex = pAggrBlkCol->maxIndex;
+        pStatis[i].minIndex = pAggrBlkCol->minIndex;
+        pStatis[i].numOfNull = pAggrBlkCol->numOfNull;
+        i++;
+        j++;
+      } else if (pStatis[i].colId < pAggrBlkCol->colId) {
+        pStatis[i].numOfNull = -1;
+        i++;
+      } else {
+        j++;
+      }
+    }
+  }
+}
+
+static void tsdbResetReadTable(SReadH *pReadh) {
+  tdResetDataCols(pReadh->pDCols[0]);
+  tdResetDataCols(pReadh->pDCols[1]);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+}
+
+static void tsdbResetReadFile(SReadH *pReadh) {
+  tsdbResetReadTable(pReadh);
+  taosArrayClear(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+}
+
+static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+
+  tdResetDataCols(pDataCols);
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1;
+
+  SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh);
+
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset,
+              pBlock->len);
+    return -1;
+  }
+
+  if (nread < pBlock->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64
+              " expected bytes:%d read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread);
+    return -1;
+  }
+
+  int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
+    return -1;
+  }
+
+  ASSERT(tsize < pBlock->len);
+  ASSERT(pBlockData->numOfCols == pBlock->numOfCols);
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  // Recover the data
+  int ccol = 0;  // loop iter for SBlockCol object
+  int dcol = 0;  // loop iter for SDataCols object
+  SBlockCol blockCol = {0};
+  SBlockCol *pBlockCol = &blockCol;
+  while (dcol < pDataCols->numOfCols) {
+    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+    if (dcol != 0 && ccol >= pBlockData->numOfCols) {
+      // Set current column as NULL and forward
+      dataColReset(pDataCol);
+      dcol++;
+      continue;
+    }
+
+    int16_t  tcolId = 0;
+    uint32_t toffset = TSDB_KEY_COL_OFFSET;
+    int32_t  tlen = pBlock->keyLen;
+
+
+    if (dcol != 0) {
+      tsdbGetSBlockCol(pBlock, &pBlockCol, pBlockData->cols, ccol);
+      tcolId = pBlockCol->colId;
+      toffset = tsdbGetBlockColOffset(pBlockCol);
+      tlen = pBlockCol->len;
+    } else {
+      ASSERT(pDataCol->colId == tcolId);
+    }
+
+    if (tcolId == pDataCol->colId) {
+      if (pBlock->algorithm == TWO_STAGE_COMP) {
+        int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+        if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
+      }
+
+      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
+                                       pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
+                  TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
+        return -1;
+      }
+
+      if (dcol != 0) {
+        ccol++;
+      }
+      dcol++;
+    } else if (tcolId < pDataCol->colId) {
+      ccol++;
+    } else {
+      // Set current column as NULL and forward
+      dataColReset(pDataCol);
+      dcol++;
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                        int maxPoints, char *buffer, int bufferSize) {
+  if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  tdAllocMemForCol(pDataCol, maxPoints);
+
+  // Decode the data
+  if (comp) {
+    // Need to decompress
+    int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData,
+                                                             pDataCol->spaceSize, comp, buffer, bufferSize);
+    if (tlen <= 0) {
+      tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
+                len, comp, numOfRows, maxPoints, bufferSize);
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      return -1;
+    }
+    pDataCol->len = tlen;
+  } else {
+    // No need to decompress, just memcpy it
+    pDataCol->len = len - sizeof(TSCKSUM);
+    memcpy(pDataCol->pData, content, pDataCol->len);
+  }
+
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    dataColSetOffset(pDataCol, numOfRows);
+  }
+  return 0;
+}
+
+static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                     int numOfColIds) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+  ASSERT(colIds[0] == 0);
+
+  SDFile *  pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+  SBlockCol blockCol = {0};
+
+  tdResetDataCols(pDataCols);
+
+  // If only load timestamp column, no need to load SBlockData part
+  if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1;
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  int dcol = 0;
+  int ccol = 0;
+  for (int i = 0; i < numOfColIds; i++) {
+    int16_t    colId = colIds[i];
+    SDataCol * pDataCol = NULL;
+    SBlockCol *pBlockCol = NULL;
+
+    while (true) {
+      if (dcol >= pDataCols->numOfCols) {
+        pDataCol = NULL;
+        break;
+      }
+      pDataCol = &pDataCols->cols[dcol];
+      if (pDataCol->colId > colId) {
+        pDataCol = NULL;
+        break;
+      } else {
+        dcol++;
+        if (pDataCol->colId == colId) break;
+      }
+    }
+
+    if (pDataCol == NULL) continue;
+    ASSERT(pDataCol->colId == colId);
+
+    if (colId == 0) {  // load the key row
+      blockCol.colId = colId;
+      blockCol.len = pBlock->keyLen;
+      blockCol.type = pDataCol->type;
+      blockCol.offset = TSDB_KEY_COL_OFFSET;
+      pBlockCol = &blockCol;
+    } else {  // load non-key rows
+      while (true) {
+        if (ccol >= pBlock->numOfCols) {
+          pBlockCol = NULL;
+          break;
+        }
+
+        pBlockCol = &blockCol;
+        tsdbGetSBlockCol(pBlock, &pBlockCol, pReadh->pBlkData->cols, ccol);
+
+        if (pBlockCol->colId > colId) {
+          pBlockCol = NULL;
+          break;
+        } else {
+          ccol++;
+          if (pBlockCol->colId == colId) break;
+        }
+      }
+
+      if (pBlockCol == NULL) {
+        dataColReset(pDataCol);
+        continue;
+      }
+
+      ASSERT(pBlockCol->colId == pDataCol->colId);
+    }
+
+    if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
+  }
+
+  return 0;
+}
+
+static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
+  ASSERT(pDataCol->colId == pBlockCol->colId);
+
+  STsdb *pRepo = TSDB_READ_REPO(pReadh);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
+  if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
+
+  int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) +
+                   tsdbGetBlockColOffset(pBlockCol);
+  if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len);
+    return -1;
+  }
+
+  if (nread < pBlockCol->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread);
+    return -1;
+  }
+
+  if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
+                                   pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
+    tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+              pBlockCol->colId, offset);
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/tsdb2/src/tsdbRecover.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/dnode/vnode/tsdb2/src/tsdbRowMergeBuf.c

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbRowMergeBuf.h"
+#include "tdataformat.h"
+
+// row1 has higher priority
+SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
+  if(row2 == NULL) return row1;
+  if(row1 == NULL) return row2;
+  ASSERT(pSchema1->version == memRowVersion(row1)); 
+  ASSERT(pSchema2->version == memRowVersion(row2));
+
+  if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) {
+    return NULL;
+  }
+  return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2);
+}

source/dnode/vnode/tsdb2/tests/CMakeLists.txt

@@ -0,0 +1,6 @@
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+add_executable(tsdbTests ${SOURCE_LIST})
+target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc)
+
+add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)

source/dnode/vnode/tsdb2/tests/tsdbTests.cpp

@@ -0,0 +1,163 @@
+#include <gtest/gtest.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+#include "tsdb.h"
+#include "tsdbMain.h"
+
+static double getCurTime() {
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  return tv.tv_sec + tv.tv_usec * 1E-6;
+}
+
+typedef struct {
+  STsdbRepo *pRepo;
+  bool       isAscend;
+  int        tid;
+  uint64_t   uid;
+  int        sversion;
+  TSKEY      startTime;
+  TSKEY      interval;
+  int        totalRows;
+  int        rowsPerSubmit;
+  STSchema * pSchema;
+} SInsertInfo;
+
+static int insertData(SInsertInfo *pInfo) {
+  SSubmitMsg *pMsg =
+      (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
+  if (pMsg == NULL) return -1;
+  TSKEY start_time = pInfo->startTime;
+
+  // Loop to write data
+  double stime = getCurTime();
+
+  for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) {
+    memset((void *)pMsg, 0, sizeof(SSubmitMsg));
+    SSubmitBlk *pBlock = (SSubmitBlk *)pMsg->blocks;
+    pBlock->uid = pInfo->uid;
+    pBlock->tid = pInfo->tid;
+    pBlock->sversion = pInfo->sversion;
+    pBlock->dataLen = 0;
+    pBlock->schemaLen = 0;
+    pBlock->numOfRows = 0;
+    for (int i = 0; i < pInfo->rowsPerSubmit; i++) {
+      // start_time += 1000;
+      if (pInfo->isAscend) {
+        start_time += pInfo->interval;
+      } else {
+        start_time -= pInfo->interval;
+      }
+      SDataRow row = (SDataRow)(pBlock->data + pBlock->dataLen);
+      tdInitDataRow(row, pInfo->pSchema);
+
+      for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
+        STColumn *pTCol = schemaColAt(pInfo->pSchema, j);
+        if (j == 0) {  // Just for timestamp
+          tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset);
+        } else {  // For int
+          int val = 10;
+          tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset);
+        }
+      }
+      pBlock->dataLen += dataRowLen(row);
+      pBlock->numOfRows++;
+    }
+    pMsg->length = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + pBlock->dataLen;
+    pMsg->numOfBlocks = 1;
+
+    pBlock->dataLen = htonl(pBlock->dataLen);
+    pBlock->numOfRows = htonl(pBlock->numOfRows);
+    pBlock->schemaLen = htonl(pBlock->schemaLen);
+    pBlock->uid = htobe64(pBlock->uid);
+    pBlock->tid = htonl(pBlock->tid);
+
+    pBlock->sversion = htonl(pBlock->sversion);
+    pBlock->padding = htonl(pBlock->padding);
+
+    pMsg->length = htonl(pMsg->length);
+    pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+
+    if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
+      tfree(pMsg);
+      return -1;
+    }
+  }
+
+  double etime = getCurTime();
+
+  printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
+  tfree(pMsg);
+  return 0;
+}
+
+static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables,
+                       int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision,
+                       int8_t compression) {
+  pCfg->tsdbId = tsdbId;
+  pCfg->cacheBlockSize = cacheBlockSize;
+  pCfg->totalBlocks = totalBlocks;
+  // pCfg->maxTables = maxTables;
+  pCfg->daysPerFile = daysPerFile;
+  pCfg->keep = keep;
+  pCfg->minRowsPerFileBlock = minRows;
+  pCfg->maxRowsPerFileBlock = maxRows;
+  pCfg->precision = precision;
+  pCfg->compression = compression;
+}
+
+static void tsdbSetTableCfg(STableCfg *pCfg) {
+  STSchemaBuilder schemaBuilder = {0};
+
+  pCfg->type = TSDB_NORMAL_TABLE;
+  pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID;
+  pCfg->tableId.tid = 1;
+  pCfg->tableId.uid = 5849583783847394;
+  tdInitTSchemaBuilder(&schemaBuilder, 0);
+
+  int colId = 0;
+  for (int i = 0; i < 5; i++) {
+    tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0);
+    colId++;
+  }
+
+  pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder);
+  pCfg->name = strdup("t1");
+
+  tdDestroyTSchemaBuilder(&schemaBuilder);
+}
+
+TEST(TsdbTest, testInsertSpeed) {
+  int         vnode = 1;
+  int         ret = 0;
+  STsdbCfg    tsdbCfg;
+  STableCfg   tableCfg;
+  std::string testDir = "./test";
+  char *      rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str());
+
+  tsdbDebugFlag = 131; //NOTE: you must set the flag
+
+  taosRemoveDir(rootDir);
+
+  // Create and open repository
+  tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
+  tsdbCreateRepo(rootDir, &tsdbCfg);
+  STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL);
+  ASSERT_NE(repo, nullptr);
+
+  // Create table
+  tsdbSetTableCfg(&tableCfg);
+  tsdbCreateTable(repo, &tableCfg);
+
+  // Insert data
+  SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 100, tableCfg.schema};
+
+  insertData(&iInfo);
+
+  tsdbCloseRepo(repo, 1);
+}
+
+static char *getTKey(const void *data) {
+  return (char *)data;
+}

source/libs/CMakeLists.txt

@@ -12,3 +12,4 @@ add_subdirectory(planner)
 add_subdirectory(function)
 add_subdirectory(qcom)
 add_subdirectory(qworker)
+add_subdirectory(tfs)

source/libs/catalog/inc/catalogInt.h

@@ -22,20 +22,31 @@ extern "C" {
 
 #include "catalog.h"
 #include "common.h"
-#include "tlog.h"
+#include "query.h"
 
 #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
 #define CTG_DEFAULT_CACHE_DB_NUMBER 20
 #define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000
+#define CTG_DEFAULT_RENT_SECOND 10
+#define CTG_DEFAULT_RENT_SLOT_SIZE 10
+
+#define CTG_RENT_SLOT_SECOND 2
 
 #define CTG_DEFAULT_INVALID_VERSION (-1)
 
+#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_TDB_INVALID_TABLE_ID
+
 enum {
   CTG_READ = 1,
   CTG_WRITE,
 };
 
+enum {
+  CTG_RENT_DB = 1,
+  CTG_RENT_STABLE,
+};
+
 typedef struct SVgroupListCache {
   int32_t vgroupVersion;
   SHashObj *cache;        // key:vgId, value:SVgroupInfo
@@ -51,30 +62,76 @@ typedef struct STableMetaCache {
   SHashObj *stableCache;     //key:suid, value:STableMeta*
 } STableMetaCache;
 
+typedef struct SRentSlotInfo {
+  SRWLatch lock;
+  bool     needSort;
+  SArray  *meta;  // element is SDbVgVersion or SSTableMetaVersion
+} SRentSlotInfo;
+
+typedef struct SMetaRentMgmt {
+  int8_t         type;
+  uint16_t       slotNum;
+  uint16_t       slotRIdx;
+  int64_t        lastReadMsec;
+  SRentSlotInfo *slots;
+} SMetaRentMgmt;
+
 typedef struct SCatalog {
+  uint64_t         clusterId;
   SDBVgroupCache   dbCache;
   STableMetaCache  tableCache;
+  SMetaRentMgmt    dbRent;
+  SMetaRentMgmt    stableRent;
 } SCatalog;
 
+typedef struct SCtgApiStat {
+
+} SCtgApiStat;
+
+typedef struct SCtgResourceStat {
+
+} SCtgResourceStat;
+
+typedef struct SCtgCacheStat {
+
+} SCtgCacheStat;
+
+typedef struct SCatalogStat {
+  SCtgApiStat      api;
+  SCtgResourceStat resource;
+  SCtgCacheStat    cache;
+} SCatalogStat;
+
 typedef struct SCatalogMgmt {
-  void       *pMsgSender;   // used to send messsage to mnode to fetch necessary metadata
+  SHashObj             *pCluster;     //key: clusterId, value: SCatalog*
-  SHashObj   *pCluster;     // items cached for each cluster, the hash key is the cluster-id got from mgmt node
+  SCatalogStat          stat;
   SCatalogCfg           cfg;
 } SCatalogMgmt;
 
 typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 
-#define ctgFatal(...)  do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+#define CTG_IS_META_NONE(type) ((type) == META_TYPE_NON_TABLE)
-#define ctgError(...)  do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+#define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
-#define ctgWarn(...)   do { if (ctgDebugFlag & DEBUG_WARN)  { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }}  while(0)
+#define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE)
-#define ctgInfo(...)   do { if (ctgDebugFlag & DEBUG_INFO)  { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+#define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE)
-#define ctgDebug(...)  do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+
-#define ctgTrace(...)  do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+#define CTG_IS_STABLE(isSTable) (1 == (isSTable))
-#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
+#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable))
+#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0)
+#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0)
+#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE))
+
+#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) 
+
+#define ctgFatal(param, ...)  qFatal("CTG:%p " param, pCatalog, __VA_ARGS__)
+#define ctgError(param, ...)  qError("CTG:%p " param, pCatalog, __VA_ARGS__)
+#define ctgWarn(param, ...)   qWarn("CTG:%p " param, pCatalog, __VA_ARGS__)
+#define ctgInfo(param, ...)   qInfo("CTG:%p " param, pCatalog, __VA_ARGS__)
+#define ctgDebug(param, ...)  qDebug("CTG:%p " param, pCatalog, __VA_ARGS__)
+#define ctgTrace(param, ...)  qTrace("CTG:%p " param, pCatalog, __VA_ARGS__)
 
 #define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
 #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
-#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
 #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
 #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
@@ -82,15 +139,15 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 #define CTG_LOCK(type, _lock) do {   \
   if (CTG_READ == (type)) {          \
     assert(atomic_load_32((_lock)) >= 0);  \
-    ctgDebug("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosRLockLatch(_lock);           \
-    ctgDebug("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     assert(atomic_load_32((_lock)) > 0);  \
   } else {                                                \
     assert(atomic_load_32((_lock)) >= 0);  \
-    ctgDebug("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
+    qDebug("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
     taosWLockLatch(_lock);                                \
-    ctgDebug("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
+    qDebug("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
     assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
   }                                                       \
 } while (0)
@@ -98,15 +155,15 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 #define CTG_UNLOCK(type, _lock) do {                       \
   if (CTG_READ == (type)) {                                \
     assert(atomic_load_32((_lock)) > 0);  \
-    ctgDebug("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosRUnLockLatch(_lock);                              \
-    ctgDebug("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     assert(atomic_load_32((_lock)) >= 0);  \
   } else {                                                \
     assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
-    ctgDebug("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosWUnLockLatch(_lock);                              \
-    ctgDebug("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    qDebug("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     assert(atomic_load_32((_lock)) >= 0);  \
   }                                                       \
 } while (0)

source/libs/catalog/test/CMakeLists.txt

@@ -16,3 +16,8 @@ TARGET_INCLUDE_DIRECTORIES(
         PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/"
         PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc"
 )
+
+add_test(
+    NAME catalogTest
+    COMMAND catalogTest
+)

source/libs/catalog/test/catalogTests.cpp

@@ -42,10 +42,13 @@ extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMeta
 void ctgTestSetPrepareTableMeta();
 void ctgTestSetPrepareCTableMeta();
 void ctgTestSetPrepareSTableMeta();
+void ctgTestSetPrepareMultiSTableMeta();
 
 bool ctgTestStop = false;
 bool ctgTestEnableSleep = false;
-bool ctgTestDeadLoop = true;
+bool ctgTestDeadLoop = false;
+int32_t ctgTestPrintNum = 200000;
+int32_t ctgTestMTRunSec = 30;
 
 int32_t ctgTestCurrentVgVersion = 0;
 int32_t ctgTestVgVersion = 1;
@@ -54,6 +57,8 @@ int32_t ctgTestColNum = 2;
 int32_t ctgTestTagNum = 1;
 int32_t ctgTestSVersion = 1;
 int32_t ctgTestTVersion = 1;
+int32_t ctgTestSuid = 2;
+int64_t ctgTestDbId = 33;
 
 uint64_t ctgTestClusterId = 0x1;
 char *ctgTestDbname = "1.db1";
@@ -101,7 +106,6 @@ void ctgTestInitLogFile() {
   const char    *defaultLogFileNamePrefix = "taoslog";
   const int32_t  maxLogFileNum = 10;
 
-  ctgDebugFlag = 159;
   tsAsyncLog = 0;
 
   char temp[128] = {0};
@@ -128,7 +132,7 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
   char tbFullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(&cn, tbFullName);
 
-  output->metaNum = 2;
+  SET_META_TYPE_BOTH_TABLE(output->metaType);
 
   strcpy(output->ctbFname, tbFullName);
 
@@ -183,6 +187,7 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) {
   ctgTestCurrentVgVersion = dbVgroup->vgVersion;
   
   dbVgroup->hashMethod = 0;
+  dbVgroup->dbId = ctgTestDbId;
   dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
 
   vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion);
@@ -216,6 +221,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
   ctgTestCurrentVgVersion = ctgTestVgVersion;
   rspMsg->vgNum = htonl(ctgTestVgNum);
   rspMsg->hashMethod = 0;
+  rspMsg->uid = htobe64(ctgTestDbId);
 
   SVgroupInfo *vg = NULL;
   uint32_t hashUnit = UINT32_MAX / ctgTestVgNum;
@@ -338,8 +344,8 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
   rspMsg->update = 1;
   rspMsg->sversion = htonl(ctgTestSVersion);
   rspMsg->tversion = htonl(ctgTestTVersion);
-  rspMsg->suid = htobe64(0x0000000000000002);
+  rspMsg->suid = htobe64(ctgTestSuid);
-  rspMsg->tuid = htobe64(0x0000000000000003);
+  rspMsg->tuid = htobe64(ctgTestSuid);
   rspMsg->vgId = 0;
 
   SSchema *s = NULL;
@@ -365,6 +371,53 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
   return;
 }
 
+void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
+  STableMetaMsg *rspMsg = NULL; //todo
+  static int32_t idx = 1;
+
+  pRsp->code =0;
+  pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema);
+  pRsp->pCont = calloc(1, pRsp->contLen);
+  rspMsg = (STableMetaMsg *)pRsp->pCont;
+  sprintf(rspMsg->tbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx);
+  sprintf(rspMsg->stbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx);
+  rspMsg->numOfTags = htonl(ctgTestTagNum);
+  rspMsg->numOfColumns = htonl(ctgTestColNum);
+  rspMsg->precision = 1;
+  rspMsg->tableType = TSDB_SUPER_TABLE;
+  rspMsg->update = 1;
+  rspMsg->sversion = htonl(ctgTestSVersion);
+  rspMsg->tversion = htonl(ctgTestTVersion);
+  rspMsg->suid = htobe64(ctgTestSuid + idx);
+  rspMsg->tuid = htobe64(ctgTestSuid + idx);
+  rspMsg->vgId = 0;
+
+  SSchema *s = NULL;
+  s = &rspMsg->pSchema[0];
+  s->type = TSDB_DATA_TYPE_TIMESTAMP;
+  s->colId = htonl(1);
+  s->bytes = htonl(8);
+  strcpy(s->name, "ts");
+
+  s = &rspMsg->pSchema[1];
+  s->type = TSDB_DATA_TYPE_INT;
+  s->colId = htonl(2);
+  s->bytes = htonl(4);
+  strcpy(s->name, "col1s");
+
+  s = &rspMsg->pSchema[2];
+  s->type = TSDB_DATA_TYPE_BINARY;
+  s->colId = htonl(3);
+  s->bytes = htonl(12);
+  strcpy(s->name, "tag1s");
+
+  ++idx;
+  
+  return;
+}
+
+
+
 void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
   
@@ -390,6 +443,14 @@ void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg
   return;
 }
 
+void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
+  ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp);
+  
+  ctgTestSetPrepareMultiSTableMeta();
+  
+  return;
+}
+
 
 
 void ctgTestSetPrepareDbVgroups() {
@@ -444,6 +505,20 @@ void ctgTestSetPrepareSTableMeta() {
   }
 }
 
+void ctgTestSetPrepareMultiSTableMeta() {
+  static Stub stub;
+  stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta);
+  {
+    AddrAny any("libtransport.so");
+    std::map<std::string,void*> result;
+    any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
+    for (const auto& f : result) {
+      stub.set(f.second, ctgTestPrepareMultiSTableMeta);
+    }
+  }
+}
+
+
 void ctgTestSetPrepareDbVgroupsAndNormalMeta() {
   static Stub stub;
   stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta);
@@ -484,6 +559,19 @@ void ctgTestSetPrepareDbVgroupsAndSuperMeta() {
   }
 }
 
+void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() {
+  static Stub stub;
+  stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta);
+  {
+    AddrAny any("libtransport.so");
+    std::map<std::string,void*> result;
+    any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
+    for (const auto& f : result) {
+      stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta);
+    }
+  }
+}
+
 
 }
 
@@ -507,7 +595,7 @@ void *ctgTestGetDbVgroupThread(void *param) {
     if (ctgTestEnableSleep) {
       usleep(rand()%5);
     }
-    if (++n % 50000 == 0) {
+    if (++n % ctgTestPrintNum == 0) {
       printf("Get:%d\n", n);
     }
   }
@@ -531,7 +619,7 @@ void *ctgTestSetDbVgroupThread(void *param) {
     if (ctgTestEnableSleep) {    
       usleep(rand()%5);
     }
-    if (++n % 50000 == 0) {
+    if (++n % ctgTestPrintNum == 0) {
       printf("Set:%d\n", n);
     }
   }
@@ -563,7 +651,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
       usleep(rand()%5);
     }
     
-    if (++n % 50000 == 0) {
+    if (++n % ctgTestPrintNum == 0) {
       printf("Get:%d\n", n);
     }
   }
@@ -589,7 +677,7 @@ void *ctgTestSetCtableMetaThread(void *param) {
     if (ctgTestEnableSleep) {    
       usleep(rand()%5);
     }
-    if (++n % 50000 == 0) {
+    if (++n % ctgTestPrintNum == 0) {
       printf("Set:%d\n", n);
     }
   }
@@ -600,7 +688,6 @@ void *ctgTestSetCtableMetaThread(void *param) {
 
 }
 
-#if 0
 
 TEST(tableMeta, normalTable) {
   struct SCatalog* pCtg = NULL;
@@ -628,6 +715,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.numOfEps, 3);
 
+
   ctgTestSetPrepareTableMeta();
 
   STableMeta *tableMeta = NULL;
@@ -654,6 +742,41 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(tableMeta->tableInfo.precision, 1);
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
+  SDbVgVersion *dbs = NULL;
+  SSTableMetaVersion *stb = NULL;
+  uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
+  int32_t i = 0;
+  while (i < 5) {
+    ++i;
+    code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
+    ASSERT_EQ(code, 0);
+    code = catalogGetExpiredSTables(pCtg, &stb, &stbNum);
+    ASSERT_EQ(code, 0);
+    
+    if (dbNum) {
+      printf("got expired db,dbId:%"PRId64"\n", dbs->dbId);
+      free(dbs);
+      dbs = NULL;
+    } else {
+      printf("no expired db\n");
+    }
+
+    if (stbNum) {
+      printf("got expired stb,suid:%"PRId64"\n", stb->suid);
+      free(stb);
+      stb = NULL;
+    } else {
+      printf("no expired stb\n");
+    }
+
+    allDbNum += dbNum;
+    allStbNum += stbNum;
+    sleep(2);
+  }
+  
+  ASSERT_EQ(allDbNum, 1);
+  ASSERT_EQ(allStbNum, 0);
+
   catalogDestroy();
 }
 
@@ -715,6 +838,42 @@ TEST(tableMeta, childTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.precision, 1);
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
+  SDbVgVersion *dbs = NULL;
+  SSTableMetaVersion *stb = NULL;
+  uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
+  int32_t i = 0;
+  while (i < 5) {
+    ++i;
+    code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
+    ASSERT_EQ(code, 0);
+    code = catalogGetExpiredSTables(pCtg, &stb, &stbNum);
+    ASSERT_EQ(code, 0);
+    
+    if (dbNum) {
+      printf("got expired db,dbId:%"PRId64"\n", dbs->dbId);
+      free(dbs);
+      dbs = NULL;
+    } else {
+      printf("no expired db\n");
+    }
+
+    if (stbNum) {
+      printf("got expired stb,suid:%"PRId64"\n", stb->suid);
+      free(stb);
+      stb = NULL;
+    } else {
+      printf("no expired stb\n");
+    }
+
+    allDbNum += dbNum;
+    allStbNum += stbNum;
+    sleep(2);
+  }
+  
+  ASSERT_EQ(allDbNum, 1);
+  ASSERT_EQ(allStbNum, 1);
+
+
   catalogDestroy();
 }
 
@@ -745,6 +904,8 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
   ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
   ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
+  ASSERT_EQ(tableMeta->uid, ctgTestSuid);
+  ASSERT_EQ(tableMeta->suid, ctgTestSuid);
   ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
   ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
   ASSERT_EQ(tableMeta->tableInfo.precision, 1);
@@ -768,7 +929,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   tableMeta = NULL;
-  code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
+  code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(tableMeta->vgId, 9);
   ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@@ -779,6 +940,40 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.precision, 1);
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
+  SDbVgVersion *dbs = NULL;
+  SSTableMetaVersion *stb = NULL;
+  uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
+  int32_t i = 0;
+  while (i < 5) {
+    ++i;
+    code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
+    ASSERT_EQ(code, 0);
+    code = catalogGetExpiredSTables(pCtg, &stb, &stbNum);
+    ASSERT_EQ(code, 0);
+    
+    if (dbNum) {
+      printf("got expired db,dbId:%"PRId64"\n", dbs->dbId);
+      free(dbs);
+      dbs = NULL;
+    } else {
+      printf("no expired db\n");
+    }
+
+    if (stbNum) {
+      printf("got expired stb,suid:%"PRId64"\n", stb->suid);
+      free(stb);
+      stb = NULL;
+    } else {
+      printf("no expired stb\n");
+    }
+
+    allDbNum += dbNum;
+    allStbNum += stbNum;
+    sleep(2);
+  }
+  
+  ASSERT_EQ(allDbNum, 1);
+  ASSERT_EQ(allStbNum, 1);
 
 
   catalogDestroy();
@@ -948,7 +1143,6 @@ TEST(dbVgroup, getSetDbVgroupCase) {
   catalogDestroy();
 }
 
-
 TEST(multiThread, getSetDbVgroupCase) {
   struct SCatalog* pCtg = NULL;
   void *mockPointer = (void *)0x1;
@@ -956,6 +1150,7 @@ TEST(multiThread, getSetDbVgroupCase) {
   SVgroupInfo *pvgInfo = NULL;
   SDBVgroupInfo dbVgroup = {0};
   SArray *vgList = NULL;
+  ctgTestStop = false;
 
   ctgTestInitLogFile();
   
@@ -988,7 +1183,7 @@ TEST(multiThread, getSetDbVgroupCase) {
     if (ctgTestDeadLoop) {
       sleep(1);
     } else {
-      sleep(600);
+      sleep(ctgTestMTRunSec);
       break;
     }
   }
@@ -999,9 +1194,6 @@ TEST(multiThread, getSetDbVgroupCase) {
   catalogDestroy();
 }
 
-#endif
-
-
 TEST(multiThread, ctableMeta) {
   struct SCatalog* pCtg = NULL;
   void *mockPointer = (void *)0x1;
@@ -1009,6 +1201,7 @@ TEST(multiThread, ctableMeta) {
   SVgroupInfo *pvgInfo = NULL;
   SDBVgroupInfo dbVgroup = {0};
   SArray *vgList = NULL;
+  ctgTestStop = false;
 
   ctgTestSetPrepareDbVgroupsAndChildMeta();
 
@@ -1038,7 +1231,7 @@ TEST(multiThread, ctableMeta) {
     if (ctgTestDeadLoop) {
       sleep(1);
     } else {
-      sleep(600);
+      sleep(ctgTestMTRunSec);
       break;
     }
   }
@@ -1050,6 +1243,78 @@ TEST(multiThread, ctableMeta) {
 }
 
 
+TEST(rentTest, allRent) {
+  struct SCatalog* pCtg = NULL;
+  void *mockPointer = (void *)0x1;
+  SVgroupInfo vgInfo = {0};
+  SVgroupInfo *pvgInfo = NULL;
+  SDBVgroupInfo dbVgroup = {0};
+  SArray *vgList = NULL;
+  ctgTestStop = false;
+  SDbVgVersion *dbs = NULL;
+  SSTableMetaVersion *stable = NULL;
+  uint32_t num = 0;
+
+  ctgTestSetPrepareDbVgroupsAndMultiSuperMeta();
+
+  initQueryModuleMsgHandle();
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+
+  for (int32_t i = 1; i <= 10; ++i) {
+    sprintf(n.tname, "%s_%d", ctgTestSTablename, i);
+
+    STableMeta *tableMeta = NULL;
+    code = catalogGetSTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
+    ASSERT_EQ(code, 0);
+    ASSERT_EQ(tableMeta->vgId, 0);
+    ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
+    ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
+    ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
+    ASSERT_EQ(tableMeta->uid, ctgTestSuid + i);
+    ASSERT_EQ(tableMeta->suid, ctgTestSuid + i);
+    ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
+    ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
+    ASSERT_EQ(tableMeta->tableInfo.precision, 1);
+    ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
+    
+    code = catalogGetExpiredDBs(pCtg, &dbs, &num);
+    ASSERT_EQ(code, 0);
+    printf("%d - expired dbNum:%d\n", i, num);
+    if (dbs) {
+      printf("%d - expired dbId:%"PRId64", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion);
+      free(dbs);
+      dbs = NULL;
+    }
+    
+    code = catalogGetExpiredSTables(pCtg, &stable, &num);
+    ASSERT_EQ(code, 0);
+    printf("%d - expired stableNum:%d\n", i, num);
+    if (stable) {
+      for (int32_t n = 0; n < num; ++n) {
+        printf("suid:%"PRId64", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion);
+      }
+      free(stable);
+      stable = NULL;
+    }
+    printf("*************************************************\n");
+    
+    sleep(2);
+  }
+  
+  catalogDestroy();
+}
+
+
+
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();

source/libs/parser/inc/astGenerator.h

@@ -123,7 +123,7 @@ typedef struct SCreatedTableInfo {
   SToken             name;        // table name token
   SToken             stbName;     // super table name token , for using clause
   SArray            *pTagNames;   // create by using super table, tag name
-  SArray            *pTagVals;    // create by using super table, tag value
+  SArray            *pTagVals;    // create by using super table, tag value. SArray<SToken>
   char              *fullname;    // table full name
   int8_t             igExist;     // ignore if exists
 } SCreatedTableInfo;

source/libs/parser/inc/parserInt.h

@@ -44,14 +44,14 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t
 
 /**
  * Validate the sql info, according to the corresponding metadata info from catalog.
- * @param pCatalog
+ * @param pCtx
- * @param pSqlInfo
+ * @param pInfo
- * @param pQueryInfo a bounded AST with essential meta data from local buffer or mgmt node
+ * @param pQueryInfo
- * @param id
+ * @param msgBuf
- * @param msg
+ * @param msgBufLen
  * @return
  */
-int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen);
+int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen);
 
 /**
  * validate the ddl ast, and convert the ast to the corresponding message format
@@ -62,6 +62,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ
  */
 SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
 
+/**
+ *
+ * @param pInfo
+ * @param pCtx
+ * @param msgBuf
+ * @param msgBufLen
+ * @return
+ */
 SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
 
 /**

source/libs/parser/src/astGenerator.c

@@ -686,7 +686,7 @@ void destroySqlNode(SSqlNode *pSqlNode) {
 void freeCreateTableInfo(void* p) {
   SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p;
   taosArrayDestroy(pInfo->pTagNames);
-  taosArrayDestroyEx(pInfo->pTagVals, freeItem);
+  taosArrayDestroy(pInfo->pTagVals);
   tfree(pInfo->fullname);
 }
 

source/libs/parser/src/astValidate.c

@@ -3630,8 +3630,8 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) {
+int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) {
-  assert(pCatalog != NULL && pInfo != NULL);
+  assert(pCtx != NULL && pInfo != NULL);
   int32_t code = 0;
 
   SMsgBuf  m = {.buf = msgBuf, .len = msgBufLen};
@@ -3682,22 +3682,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       break;
     }
 
-    case TSDB_SQL_USE_DB: {
-      const char* msg = "invalid db name";
-      SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
-
-      if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg);
-      }
-
-      int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken);
-      if (ret != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg);
-      }
-
-      break;
-    }
-
     case TSDB_SQL_RESET_CACHE: {
       return TSDB_CODE_SUCCESS;
     }
@@ -3712,55 +3696,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       break;
     }
 
-    case TSDB_SQL_CREATE_DNODE: {
-      const char* msg = "invalid host name (ip address)";
-
-      if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) {
-        return buildInvalidOperationMsg(pMsgBuf, msg);
-      }
-
-      SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
-      if (id->type == TK_STRING) {
-        id->n = strdequote(id->z);
-      }
-      break;
-    }
-
-    case TSDB_SQL_CREATE_ACCT:
-    case TSDB_SQL_ALTER_ACCT: {
-      const char* msg1 = "invalid state option, available options[no, r, w, all]";
-      const char* msg2 = "invalid user/account name";
-      const char* msg3 = "name too long";
-
-      SToken* pName = &pInfo->pMiscInfo->user.user;
-      SToken* pPwd = &pInfo->pMiscInfo->user.passwd;
-
-      if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      }
-
-      if (pName->n >= TSDB_USER_LEN) {
-        return buildInvalidOperationMsg(pMsgBuf, msg3);
-      }
-
-      if (tscValidateName(pName) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
-      }
-
-      SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt;
-      if (pAcctOpt->stat.n > 0) {
-        if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) {
-        } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) {
-        } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) {
-        } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) {
-        } else {
-          return buildInvalidOperationMsg(pMsgBuf, msg1);
-        }
-      }
-
-      break;
-    }
-
     case TSDB_SQL_DESCRIBE_TABLE: {
       const char* msg1 = "invalid table name";
 
@@ -3865,29 +3800,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       return TSDB_CODE_SUCCESS;
     }
 
-    case TSDB_SQL_CREATE_TABLE: {
-      SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
-
-      if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) {
-        if ((code = doCheckForCreateTable(pSql, 0, pInfo)) != TSDB_CODE_SUCCESS) {
-          return code;
-        }
-
-      } else if (pCreateTable->type == TSQL_CREATE_TABLE_FROM_STABLE) {
-        assert(pCmd->numOfCols == 0);
-        if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
-          return code;
-        }
-
-      } else if (pCreateTable->type == TSQL_CREATE_STREAM) {
-        if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
-          return code;
-        }
-      }
-
-      break;
-    }
-
     case TSDB_SQL_SELECT: {
       const char * msg1 = "no nested query supported in union clause";
       code = loadAllTableMeta(pSql, pInfo);
@@ -3981,10 +3893,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       }
       break;
     }
-    #endif
     default:
       return buildInvalidOperationMsg(pMsgBuf, "not support sql expression");
   }
+#endif
+  }
 
   SCatalogReq req  = {0};
   SMetaData   data = {0};
@@ -3997,7 +3910,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
   }
 
   // load the meta data from catalog
-  code = catalogGetAllMeta(pCatalog, NULL, NULL, &req, &data);
+  code = catalogGetAllMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &req, &data);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }

source/libs/parser/src/dCDAstProcess.c

@@ -1,12 +1,11 @@
-#include <astGenerator.h>
+#include "tmsg.h"
-#include <tmsg.h>
+#include "tglobal.h"
-#include "astToMsg.h"
 #include "parserInt.h"
+#include "ttime.h"
+#include "astToMsg.h"
+#include "astGenerator.h"
 #include "parserUtil.h"
 #include "queryInfoUtil.h"
-#include "tglobal.h"
-#include "tmsg.h"
-#include "ttime.h"
 
 /* is contained in pFieldList or not */
 static bool has(SArray* pFieldList, int32_t startIndex, const char* name) {
@@ -332,7 +331,6 @@ static int32_t doParseSerializeTagValue(SSchema* pTagSchema, int32_t numOfInputT
 
     char* endPtr = NULL;
     char  tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0};
-
     SKvParam param = {.builder = pKvRowBuilder, .schema = pSchema};
 
     SToken* pItem = taosArrayGet(pTagValList, i);

source/libs/parser/src/insertParser.c

@@ -629,7 +629,6 @@ int32_t parseInsertSql(SParseContext* pContext, SVnodeModifOpStmtInfo** pInfo) {
 
   *pInfo = context.pOutput;
   context.pOutput->nodeType = TSDB_SQL_INSERT;
-  context.pOutput->schemaAttache = pContext->schemaAttached;
   context.pOutput->payloadType = PAYLOAD_TYPE_KV;
 
   int32_t code = skipInsertInto(&context);

source/libs/parser/src/parser.c

@@ -31,8 +31,8 @@ bool isInsertSql(const char* pStr, size_t length) {
   } while (1);
 }
 
-bool qIsDdlQuery(const SQueryNode* pQuery) {
+bool qIsDdlQuery(const SQueryNode* pQueryNode) {
-  return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type && TSDB_SQL_CREATE_TABLE != pQuery->type;
+  return TSDB_SQL_INSERT != pQueryNode->type && TSDB_SQL_SELECT != pQueryNode->type && TSDB_SQL_CREATE_TABLE != pQueryNode->type;
 }
 
 int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
@@ -67,7 +67,7 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
       return terrno;
     }
 
-    int32_t code = qParserValidateSqlNode(pCxt->ctx.pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen);
+    int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
     if (code == TSDB_CODE_SUCCESS) {
       *pQuery = (SQueryNode*)pQueryInfo;
     }

source/libs/qcom/src/querymsg.c

@@ -97,6 +97,7 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
   
   pRsp->vgVersion = ntohl(pRsp->vgVersion);
   pRsp->vgNum = ntohl(pRsp->vgNum);
+  pRsp->uid = be64toh(pRsp->uid);
 
   if (pRsp->vgNum < 0) {
     qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum);
@@ -111,6 +112,7 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
 
   pOut->dbVgroup.vgVersion = pRsp->vgVersion;
   pOut->dbVgroup.hashMethod = pRsp->hashMethod;
+  pOut->dbVgroup.dbId = pRsp->uid;
   pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (NULL == pOut->dbVgroup.vgInfo) {
     qError("hash init[%d] failed", pRsp->vgNum);
@@ -149,8 +151,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) {
   pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns);
   pMetaMsg->sversion = ntohl(pMetaMsg->sversion);
   pMetaMsg->tversion = ntohl(pMetaMsg->tversion);
-  pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
+  pMetaMsg->tuid = be64toh(pMetaMsg->tuid);
-  pMetaMsg->suid = htobe64(pMetaMsg->suid);
+  pMetaMsg->suid = be64toh(pMetaMsg->suid);
   pMetaMsg->vgId = ntohl(pMetaMsg->vgId);
 
   if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
@@ -208,7 +210,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STabl
 
   pTableMeta->vgId = isSuperTable ? 0 : msg->vgId;
   pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType;
-  pTableMeta->uid  = msg->tuid;
+  pTableMeta->uid  = isSuperTable ? msg->suid : msg->tuid;
   pTableMeta->suid = msg->suid;
   pTableMeta->sversion = msg->sversion;
   pTableMeta->tversion = msg->tversion;
@@ -244,7 +246,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
   }
 
   if (pMetaMsg->tableType == TSDB_CHILD_TABLE) {
-    pOut->metaNum = 2;
+    SET_META_TYPE_BOTH_TABLE(pOut->metaType);
 
     if (pMetaMsg->dbFname[0]) {
       snprintf(pOut->ctbFname, sizeof(pOut->ctbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname);
@@ -261,7 +263,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
 
     code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta);
   } else {
-    pOut->metaNum = 1;
+    SET_META_TYPE_TABLE(pOut->metaType);
     
     if (pMetaMsg->dbFname[0]) {
       snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname);

source/libs/scheduler/src/scheduler.c

@@ -505,7 +505,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms
       break;
     }
     case TDMT_VND_SUBMIT_RSP: {
-        if (rspCode != TSDB_CODE_SUCCESS) {
+        if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) {
           SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode));
         } else {
           SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg;
@@ -521,7 +521,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms
     case TDMT_VND_QUERY_RSP: {
         SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
         
-        if (rsp->code != TSDB_CODE_SUCCESS) {
+        if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) {
           SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code));
         } else {
           code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY);
@@ -534,7 +534,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms
     case TDMT_VND_RES_READY_RSP: {
         SResReadyRsp *rsp = (SResReadyRsp *)msg;
         
-        if (rsp->code != TSDB_CODE_SUCCESS) {
+        if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) {
           SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code));
         } else {
           code = schProcessOnTaskSuccess(job, task);
@@ -549,7 +549,9 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms
         SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
 
         job->res = rsp;
+        if (rsp) {
           job->resNumOfRows = rsp->numOfRows;
+        }
         
         SCH_ERR_JRET(schProcessOnDataFetched(job));
         break;
@@ -1100,6 +1102,7 @@ void scheduleFreeJob(void *pJob) {
   taosHashCleanup(job->failTasks);
   taosHashCleanup(job->succTasks);
   taosArrayDestroy(job->levels);
+  
   tfree(job);
 }
 

source/libs/tfs/CMakeLists.txt

@@ -0,0 +1,9 @@
+aux_source_directory(src TFS_SRC)
+add_library(tfs STATIC ${TFS_SRC})
+target_include_directories(
+  tfs
+  PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tfs"
+  PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(tfs os util common)

source/libs/tfs/src/tfs.c

@@ -15,11 +15,11 @@
 
 #include "os.h"
 
-#include "hash.h"
 #include "taosdef.h"
 #include "taoserror.h"
 #include "tfs.h"
 #include "tfsint.h"
+#include "thash.h"
 
 #define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
 
@@ -270,7 +270,8 @@ int tfsMkdirRecurAt(const char *rname, int level, int id) {
       // Some platform may modify the contents of the string passed into dirname(). Others may return a pointer to
       // internal static storage space that will be overwritten by next call. For case like that, we should not use
       // the pointer directly in this recursion.
-      // See https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html
+      // See
+      // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html
       char *dir = strdup(dirname(s));
 
       if (tfsMkdirRecurAt(dir, level, id) < 0) {
@@ -334,7 +335,7 @@ int tfsRename(char *orname, char *nrname) {
       snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
       snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname);
 
-      taosRename(oaname, naname);
+      taosRenameFile(oaname, naname);
     }
   }
 
@@ -504,7 +505,6 @@ static int tfsFormatDir(char *idir, char *odir) {
 
   wordfree(&wep);
   return 0;
-
 }
 
 static int tfsCheck() {
@@ -599,12 +599,10 @@ void taosGetDisk() {
   SysDiskSize  diskSize;
   SFSMeta      fsMeta;
 
-  if (tscEmbedded) {
   tfsUpdateInfo(&fsMeta, NULL, 0);
   tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
   tsUsedDataDirGB = (float)(fsMeta.used / unit);
   tsAvailDataDirGB = (float)(fsMeta.avail / unit);
-  }
 
   if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
     tsTotalLogDirGB = (float)(diskSize.tsize / unit);

source/util/src/thash.c

@@ -215,7 +215,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) {
   return taosHashGetSize(pHashObj) == 0;
 }
 
-int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
+int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) {
   uint32_t   hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
   SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal);
   if (pNewNode == NULL) {
@@ -274,6 +274,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
     __rd_unlock((void*) &pHashObj->lock, pHashObj->type);
     atomic_add_fetch_32(&pHashObj->size, 1);
 
+    if (newAdded) {
+      *newAdded = true;
+    }
+    
     return 0;
   } else {
     // not support the update operation, return error
@@ -290,10 +294,23 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
     // enable resize
     __rd_unlock((void*) &pHashObj->lock, pHashObj->type);
 
+    if (newAdded) {
+      *newAdded = false;
+    }
+
     return pHashObj->enableUpdate ? 0 : -2;
   }
 }
 
+int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
+  return taosHashPutImpl(pHashObj, key, keyLen, data, size, NULL);
+}
+
+int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) {
+  return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded);
+}
+
+
 void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
   return taosHashGetClone(pHashObj, key, keyLen, NULL);
 }

source/util/src/tlog.c

@@ -95,7 +95,6 @@ int32_t tsdbDebugFlag = 131;
 int32_t tqDebugFlag = 131;
 int32_t cqDebugFlag = 131;
 int32_t fsDebugFlag = 135;
-int32_t ctgDebugFlag = 131;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;

src/tfs/CMakeLists.txt

@@ -1,12 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-INCLUDE_DIRECTORIES(inc)
-AUX_SOURCE_DIRECTORY(src SRC)
-ADD_LIBRARY(tfs ${SRC})
-TARGET_LINK_LIBRARIES(tfs tutil)
-
-IF (TD_LINUX)
-  # Someone has no gtest directory, so comment it
-  # ADD_SUBDIRECTORY(tests)
-ENDIF ()