Merge branch 'develop' into feature/sangshuduo/TD-2771-python-taosdemo

CMakeLists.txt

@@ -15,6 +15,7 @@ SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
+SET(TD_STORAGE FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

Jenkinsfile

@@ -45,6 +45,7 @@ def pre_test(){
     git pull
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
+    git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop
@@ -79,13 +80,14 @@ pipeline {
               changeRequest()
           }
       parallel {
-        stage('python_1') {
+        stage('python_1_s1') {
           agent{label 'p1'}
           steps {
             
             pre_test()
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
+              date
               cd ${WKC}/tests
               find pytest -name '*'sql|xargs rm -rf
               ./test-all.sh p1
@@ -94,23 +96,35 @@ pipeline {
             
           }
         }
-        stage('python_2') {
+        stage('python_2_s5') {
           agent{label 'p2'}
           steps {
             
             pre_test()
+            timeout(time: 45, unit: 'MINUTES'){
             sh '''
+            date
             cd ${WKC}/tests
             find pytest -name '*'sql|xargs rm -rf
             ./test-all.sh p2
             date'''
-            sh '''
+            }
-            cd ${WKC}/tests
-            ./test-all.sh b4fq
-            '''
           }
         }
-        stage('test_b1') {
+        stage('python_3_s6') {
+          agent{label 'p3'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh p3
+              date'''
+            }
+          }
+        }
+        stage('test_b1_s2') {
           agent{label 'b1'}
           steps {     
             timeout(time: 90, unit: 'MINUTES'){       
@@ -123,7 +137,7 @@ pipeline {
           }
         }
 
-        stage('test_crash_gen') {
+        stage('test_crash_gen_s3') {
           agent{label "b2"}
           steps {
             pre_test()
@@ -139,7 +153,7 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
               date
               cd ${WKC}/tests
@@ -150,7 +164,7 @@ pipeline {
           }
         }
 
-        stage('test_valgrind') {
+        stage('test_valgrind_s4') {
           agent{label "b3"}
 
           steps {
@@ -162,7 +176,7 @@ pipeline {
                 ./handle_val_log.sh
                 '''
             }     
-            timeout(time: 90, unit: 'MINUTES'){      
+            timeout(time: 45, unit: 'MINUTES'){      
               sh '''
               date
               cd ${WKC}/tests
@@ -171,8 +185,58 @@ pipeline {
             }
           }
         }
-   
+        stage('test_b4_s7') {
-        
+          agent{label 'b4'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b4fq
+              date'''
+            }
+          }
+        }
+        stage('test_b5_s8') {
+          agent{label 'b5'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b5fq
+              date'''
+            }
+          }
+        }
+        stage('test_b6_s9') {
+          agent{label 'b6'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b6fq
+              date'''
+            }
+          }
+        }
+        stage('test_b7_s10') {
+          agent{label 'b7'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b7fq
+              date'''
+            }
+          }
+        }        
     }
   }
   }

cmake/define.inc

@@ -21,6 +21,10 @@ IF (TD_TSDB_PLUGINS)
   ADD_DEFINITIONS(-D_TSDB_PLUGINS)
 ENDIF ()
 
+IF (TD_STORAGE)
+  ADD_DEFINITIONS(-D_STORAGE)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

src/CMakeLists.txt

@@ -5,6 +5,7 @@ PROJECT(TDengine)
 ADD_SUBDIRECTORY(os)
 ADD_SUBDIRECTORY(common)
 ADD_SUBDIRECTORY(util)
+ADD_SUBDIRECTORY(tfs)
 ADD_SUBDIRECTORY(rpc)
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)

src/client/src/tscSQLParser.c

@@ -3440,6 +3440,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) {
 
 static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableCond, SStringBuilder* sb) {
   const char* msg0 = "invalid table name list";
+  const char* msg1 = "not string following like";
 
   if (pTableCond == NULL) {
     return TSDB_CODE_SUCCESS;
@@ -3457,6 +3458,10 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   if (pTableCond->nSQLOptr == TK_IN) {
     ret = tablenameListToString(pRight, sb);
   } else if (pTableCond->nSQLOptr == TK_LIKE) {
+    if (pRight->nSQLOptr != TK_STRING) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
+    
     ret = tablenameCondToString(pRight, sb);
   }
 

src/client/src/tscSub.c

@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p == NULL) {
     return dflt;
   }
@@ -76,7 +76,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p != NULL) {
     p->key = ts;
     tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
@@ -270,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);

src/client/src/tscSubquery.c

@@ -2410,7 +2410,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
 
   // record the total inserted rows
   if (numOfRows > 0) {
-    pParentObj->res.numOfRows += numOfRows;
+    atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows);
   }
 
   if (taos_errno(tres) != TSDB_CODE_SUCCESS) {

src/client/src/tscSystem.c

@@ -18,7 +18,6 @@
 #include "tref.h"
 #include "trpc.h"
 #include "tnote.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "tsched.h"
@@ -49,7 +48,7 @@ int32_t   tscNumOfThreads = 1;     // num of rpc threads
 static    pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 
-void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
+void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) {
   taosGetDisk();
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
 }
@@ -65,7 +64,7 @@ void tscReleaseRpc(void *param)  {
     return;
   }
   pthread_mutex_lock(&rpcObjMutex);
-  taosCacheRelease(tscRpcCache, (void *)&param, false); 
+  taosCacheRelease(tscRpcCache, (void *)&param, true); 
   pthread_mutex_unlock(&rpcObjMutex);
 } 
 
@@ -88,7 +87,7 @@ int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncry
   rpcInit.sessions = tsMaxConnections;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
-  rpcInit.idleTime = 2000; 
+  rpcInit.idleTime = tsShellActivityTimer * 1000; 
   rpcInit.ckey = "key"; 
   rpcInit.spi = 1; 
   rpcInit.secret = (char *)secretEncrypt;
@@ -216,7 +215,6 @@ void taos_cleanup(void) {
   taosCloseRef(id);
 
   taosCleanupKeywordsTable();
-  taosCloseLog();
 
   p = tscRpcCache; 
   tscRpcCache = NULL;
@@ -226,7 +224,10 @@ void taos_cleanup(void) {
     pthread_mutex_destroy(&rpcObjMutex);
   }
 
-  if (tscEmbedded == 0) rpcCleanup();
+  if (tscEmbedded == 0) {
+    rpcCleanup();
+    taosCloseLog();
+  };
 
   p = tscTmr;
   tscTmr = NULL;

src/common/inc/tdataformat.h

@@ -68,9 +68,9 @@ typedef struct {
 typedef struct {
   int      version;    // version
   int      numOfCols;  // Number of columns appended
-  int      tlen;       // maximum length of a SDataRow without the header part
+  int      tlen;       // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
   uint16_t flen;       // First part length in a SDataRow after the header part
-  uint16_t vlen;       // pure value part length, excluded the overhead
+  uint16_t vlen;       // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
 } STSchema;
 
@@ -278,7 +278,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
-void       tdFreeDataCols(SDataCols *pCols);
+SDataCols *tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
 

src/common/inc/tglobal.h

@@ -16,6 +16,8 @@
 #ifndef TDENGINE_COMMON_GLOBAL_H
 #define TDENGINE_COMMON_GLOBAL_H
 
+#include "taosdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -147,7 +149,6 @@ extern char    tsDataDir[];
 extern char    tsLogDir[];
 extern char    tsScriptDir[];
 extern int64_t tsMsPerDay[3];
-extern char    tsVnodeBakDir[];
 
 // system info
 extern char    tsOsName[];
@@ -196,6 +197,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+typedef struct {
+  char dir[TSDB_FILENAME_LEN];
+  int  level;
+  int  primary;
+} SDiskCfg;
+extern int32_t  tsDiskCfgNum;
+extern SDiskCfg tsDiskCfg[];
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void    taosInitGlobalCfg();
@@ -204,6 +213,9 @@ void    taosSetAllDebugFlag();
 bool    taosCfgDynamicOptions(char *msg);
 int     taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+void    taosAddDataDir(int index, char *v1, int level, int primary);
+void    taosReadDataDirCfg(char *v1, char *v2, char *v3);
+void    taosPrintDataDirCfg();
 
 #ifdef __cplusplus
 }

src/common/src/tdataformat.c

@@ -289,23 +289,31 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
     return NULL;
   }
 
-  pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
+  pCols->maxPoints = maxRows;
-  if (pCols->cols == NULL) {
+
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
+  if (maxCols > 0) {
-    tdFreeDataCols(pCols);
+    pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
-    return NULL;
+    if (pCols->cols == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
+
+    pCols->maxCols = maxCols;
   }
 
   pCols->maxRowSize = maxRowSize;
-  pCols->maxCols = maxCols;
-  pCols->maxPoints = maxRows;
   pCols->bufSize = maxRowSize * maxRows;
 
-  pCols->buf = malloc(pCols->bufSize);
+  if (pCols->bufSize > 0) {
-  if (pCols->buf == NULL) {
+    pCols->buf = malloc(pCols->bufSize);
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
+    if (pCols->buf == NULL) {
-    tdFreeDataCols(pCols);
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
-    return NULL;
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
   }
 
   return pCols;
@@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   return 0;
 }
 
-void tdFreeDataCols(SDataCols *pCols) {
+SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     tfree(pCols->buf);
     tfree(pCols->cols);
     free(pCols);
   }
+  return NULL;
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {

src/common/src/tglobal.c

@@ -182,7 +182,14 @@ char   tsDnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsDataDir[TSDB_FILENAME_LEN] = {0};
 char   tsScriptDir[TSDB_FILENAME_LEN] = {0};
-char   tsVnodeBakDir[TSDB_FILENAME_LEN] = {0};
+
+int32_t  tsDiskCfgNum = 0;
+
+#ifndef _STORAGE
+SDiskCfg tsDiskCfg[1];
+#else
+SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+#endif
 
 /*
  * minimum scale for whole system, millisecond by default
@@ -227,6 +234,7 @@ int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
+int32_t fsDebugFlag = 135;
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
@@ -334,6 +342,39 @@ bool taosCfgDynamicOptions(char *msg) {
   return false;
 }
 
+void taosAddDataDir(int index, char *v1, int level, int primary) {
+  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
+  tsDiskCfg[index].level = level;
+  tsDiskCfg[index].primary = primary;
+  uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
+}
+
+#ifndef _STORAGE
+void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
+  if (tsDiskCfgNum == 1) {
+    SDiskCfg *cfg = &tsDiskCfg[0];
+    uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
+  }
+  taosAddDataDir(0, v1, 0, 1);
+  tsDiskCfgNum = 1;
+}
+
+void taosPrintDataDirCfg() {
+  for (int i = 0; i < tsDiskCfgNum; ++i) {
+    SDiskCfg *cfg = &tsDiskCfg[i];
+    uInfo(" dataDir: %s", cfg->dir);
+  }
+}
+#endif
+
+static void taosCheckDataDirCfg() {
+  if (tsDiskCfgNum <= 0) {
+    taosAddDataDir(0, tsDataDir, 0, 1);
+    tsDiskCfgNum = 1;
+    uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
+  }
+}
+
 static void doInitGlobalConfig(void) {
   osInit();
   srand(taosSafeRand());
@@ -415,7 +456,7 @@ static void doInitGlobalConfig(void) {
 
   cfg.option = "dataDir";
   cfg.ptr = tsDataDir;
-  cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
+  cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -1448,6 +1489,7 @@ int32_t taosCheckGlobalCfg() {
     snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
   }
 
+  taosCheckDataDirCfg();
   taosGetSystemInfo();
 
   tsSetLocale();

src/cq/src/cqMain.c

@@ -125,8 +125,6 @@ void cqFree(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 
   if (delete) {
-    pthread_mutex_unlock(&pContext->mutex);
-    
     pthread_mutex_destroy(&pContext->mutex);
     
     taosTmrCleanUp(pContext->tmrCtrl);
@@ -186,6 +184,18 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   return pContext;
 }
 
+static void freeSCqContext(void *handle) {
+  if (handle == NULL) {
+    return;
+  }
+  SCqContext *pContext = handle;
+  pthread_mutex_destroy(&pContext->mutex);
+    
+  taosTmrCleanUp(pContext->tmrCtrl);
+  pContext->tmrCtrl = NULL;
+  cDebug("vgId:%d, CQ is closed", pContext->vgId);
+  free(pContext);
+}
 void cqClose(void *handle) {
   if (tsEnableStream == 0) {
     return;
@@ -217,6 +227,8 @@ void cqClose(void *handle) {
     
     taosRemoveRef(cqObjRef, rid);
   }
+
+  freeSCqContext(pContext);
 }
 
 void cqStart(void *handle) {

src/dnode/CMakeLists.txt

@@ -27,7 +27,7 @@ IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(taosd grant)
 ENDIF ()
 
-IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT)
+IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
@@ -43,5 +43,6 @@ ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
     COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
+    COMMAND ${CMAKE_COMMAND} -E echo monitor 0  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMENT "prepare taosd environment")
 ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})

src/dnode/inc/dnodeMgmt.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_DNODE_MGMT_H
-#define TDENGINE_DNODE_MGMT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "trpc.h"
-
-int32_t dnodeInitMgmt();
-void    dnodeCleanupMgmt();
-int32_t dnodeInitMgmtTimer();
-void    dnodeCleanupMgmtTimer();
-void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
-
-void*   dnodeGetVnode(int32_t vgId);
-int32_t dnodeGetVnodeStatus(void *pVnode);
-void*   dnodeGetVnodeRworker(void *pVnode);
-void*   dnodeGetVnodeWworker(void *pVnode);
-void*   dnodeGetVnodeWal(void *pVnode);
-void*   dnodeGetVnodeTsdb(void *pVnode);
-void    dnodeReleaseVnode(void *pVnode);
-
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/dnode/src/dnodeMain.c

@@ -21,7 +21,7 @@
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "tsync.h"
 #include "dnodeStep.h"
 #include "dnodePeer.h"
@@ -189,32 +189,35 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  if (dnodeCreateDir(tsDataDir) < 0) {
+  if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
+    dError("failed to init TFS since %s", tstrerror(terrno));
-   return -1;
+    return -1;
   }
+  strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
-  sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
+  // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
 
   //TODO(dengyihao): no need to init here 
   if (dnodeCreateDir(tsMnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
    return -1;
   } 
-  //TODO(dengyihao): no need to init here
+
-  if (dnodeCreateDir(tsVnodeDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno));
-   return -1;
-  }
   if (dnodeCreateDir(tsDnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno));
    return -1;
-  } 
+  }
-  if (dnodeCreateDir(tsVnodeBakDir) < 0) {
+
-   dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno));
+  if (tfsMkdir("vnode") < 0) {
-   return -1;
+    dError("failed to create vnode dir since %s", tstrerror(terrno));
+    return -1;
+  }
+
+  if (tfsMkdir("vnode_bak") < 0) {
+    dError("failed to create vnode_bak dir since %s", tstrerror(terrno));
+    return -1;
   }
 
   dnodeCheckDataDirOpenned(tsDnodeDir);
@@ -223,7 +226,7 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() {}
+static void dnodeCleanupStorage() { tfsDestroy(); }
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;

src/dnode/src/dnodeVMgmt.c

@@ -174,7 +174,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
     vnodeRelease(pVnode);
     return code;
   } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
   }
 }

src/inc/taoserror.h

@@ -242,6 +242,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "No table d
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "File already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "Need to reconfigure table")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "Invalid information to create table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            0, 0x0613, "No available disk")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               0, 0x0614, "TSDB messed message")
 
 // query
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "Invalid handle")

src/inc/tfs.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFS_H
+#define TD_TFS_H
+
+#include "tglobal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int level;
+  int id;
+} SDiskID;
+
+#define TFS_UNDECIDED_LEVEL -1
+#define TFS_UNDECIDED_ID -1
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+
+// FS APIs ====================================
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SFSMeta;
+
+int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+void tfsDestroy();
+void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsGetMeta(SFSMeta *pMeta);
+void tfsAllocDisk(int expLevel, int *level, int *id);
+
+const char *TFS_PRIMARY_PATH();
+const char *TFS_DISK_PATH(int level, int id);
+
+// TFILE APIs ====================================
+typedef struct {
+  int  level;
+  int  id;
+  char rname[TSDB_FILENAME_LEN];  // REL name
+  char aname[TSDB_FILENAME_LEN];  // ABS name
+} TFILE;
+
+#define TFILE_LEVEL(pf) ((pf)->level)
+#define TFILE_ID(pf) ((pf)->id)
+#define TFILE_NAME(pf) ((pf)->aname)
+#define TFILE_REL_NAME(pf) ((pf)->rname)
+
+#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
+#define tfsclose(fd) close(fd)
+#define tfsremove(pf) remove(TFILE_NAME(pf))
+#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) rename(TFILE_NAME(sf), TFILE_NAME(df))
+
+void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
+int   tfsEncodeFile(void **buf, TFILE *pf);
+void *tfsDecodeFile(void *buf, TFILE *pf);
+void  tfsbasename(const TFILE *pf, char *dest);
+void  tfsdirname(const TFILE *pf, char *dest);
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id);
+int tfsMkdirRecurAt(const char *rname, int level, int id);
+int tfsMkdir(const char *rname);
+int tfsRmdir(const char *rname);
+int tfsRename(char *orname, char *nrname);
+
+typedef struct TDIR TDIR;
+
+TDIR *       tfsOpendir(const char *rname);
+const TFILE *tfsReaddir(TDIR *tdir);
+void         tfsClosedir(TDIR *tdir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/tsdb.h

@@ -40,7 +40,8 @@ extern "C" {
 
 // TSDB STATE DEFINITION
 #define TSDB_STATE_OK 0x0
-#define TSDB_STATE_BAD_FILE 0x1
+#define TSDB_STATE_BAD_META 0x1
+#define TSDB_STATE_BAD_DATA 0x2
 
 // --------- TSDB APPLICATION HANDLE DEFINITION
 typedef struct {
@@ -48,7 +49,7 @@ typedef struct {
   void *cqH;
   int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
-  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
+  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
 } STsdbAppH;
 
@@ -76,17 +77,17 @@ typedef struct {
   int64_t pointsWritten;  // total data points written
 } STsdbStat;
 
-typedef void TSDB_REPO_T;  // use void to hide implementation details from outside
+typedef struct STsdbRepo STsdbRepo;
 
-STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
+STsdbCfg *tsdbGetCfg(const STsdbRepo *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
+int32_t    tsdbCreateRepo(int repoid);
-int32_t      tsdbDropRepo(char *rootDir);
+int32_t    tsdbDropRepo(int repoid);
-TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
+STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
-int          tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
+int        tsdbCloseRepo(STsdbRepo *repo, int toCommit);
-int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
+int32_t    tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg);
-int          tsdbGetState(TSDB_REPO_T *repo);
+int        tsdbGetState(STsdbRepo *repo);
 
 // --------- TSDB TABLE DEFINITION
 typedef struct {
@@ -110,8 +111,8 @@ typedef struct {
 
 void tsdbClearTableCfg(STableCfg *config);
 
-void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
+void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes);
-char* tsdbGetTableName(void *pTable);
+char *tsdbGetTableName(void *pTable);
 
 #define TSDB_TABLEID(_table) ((STableId*) (_table))
 #define TSDB_PREV_ROW  0x1
@@ -119,12 +120,11 @@ char* tsdbGetTableName(void *pTable);
 
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
 
-int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg);
-int   tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
+int tsdbDropTable(STsdbRepo *pRepo, STableId tableId);
-int   tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg);
-// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
 
-uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
+uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
@@ -134,7 +134,7 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
+STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo);
 
 // the meter information report structure
 typedef struct {
@@ -152,7 +152,7 @@ typedef struct {
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -168,9 +168,9 @@ typedef struct STsdbQueryCond {
 } STsdbQueryCond;
 
 typedef struct SMemRef {
-  int32_t  ref;
+  int32_t ref;
-  void    *mem;
+  void *  mem;
-  void    *imem;
+  void *  imem;
 } SMemRef;
 
 typedef struct SDataBlockInfo {
@@ -182,14 +182,14 @@ typedef struct SDataBlockInfo {
 } SDataBlockInfo;
 
 typedef struct {
-  void  *pTable;
+  void *pTable;
-  TSKEY  lastKey;
+  TSKEY lastKey;
 } STableKeyInfo;
 
 typedef struct {
   size_t    numOfTables;
-  SArray   *pGroupList;
+  SArray *  pGroupList;
-  SHashObj *map;         // speedup acquire the tableQueryInfo by table uid
+  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 /**
@@ -202,7 +202,8 @@ typedef struct {
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -214,14 +215,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tableInfo  table list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * get the queried table object list
  * @param pHandle
  * @return
  */
-SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
+SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 
 /**
  * get the group list according to table id from client
@@ -231,8 +233,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
  * @param qinfo
  * @return
  */
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo, SMemRef* pRef);
+                                               void *qinfo, SMemRef *pRef);
 
 
 /**
@@ -268,7 +270,7 @@ SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t
  * @param pBlockInfo
  * @return
  */
-void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
+void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo);
 
 /**
  *
@@ -299,7 +301,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL
  * @param stableid. super table sid
  * @param pTagCond. tag query condition
  */
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
                                  int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                  SColIndex *pColIndex, int32_t numOfCols);
 
@@ -317,7 +319,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
  * @param pGroupInfo  the generated result
  * @return
  */
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 
 /**
  *
@@ -326,7 +328,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST
  * @param pGroupInfo
  * @return
  */
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
 
 /**
  * clean up the query handle
@@ -345,10 +347,14 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int
 
 int  tsdbInitCommitQueue();
 void tsdbDestroyCommitQueue();
-int  tsdbSyncCommit(TSDB_REPO_T *repo);
+int  tsdbSyncCommit(STsdbRepo *repo);
 void tsdbIncCommitRef(int vgId);
 void tsdbDecCommitRef(int vgId);
 
+// For TSDB file sync
+int tsdbSyncSend(void *pRepo, SOCKET socketFd);
+int tsdbSyncRecv(void *pRepo, SOCKET socketFd);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/tsync.h

@@ -56,16 +56,6 @@ typedef struct {
   int32_t  role[TAOS_SYNC_MAX_REPLICA];
 } SNodesRole;
 
-/* 
-  if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file
-  is found between index and eindex, index shall be updated, name shall be set, size shall be set to 
-  file size, and file magic number shall be returned. 
-
-  if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
-  zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
-*/
-typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
-
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
 typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
@@ -83,24 +73,31 @@ typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
 typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 
 // when data file is synced successfully, notity app
-typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
+typedef void     (*FStartSyncFile)(int32_t vgId);
+typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
 
 // get file version
 typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
 
+typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
+typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
+
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
   SSyncCfg syncCfg;    // configuration from mgmt
   char     path[TSDB_FILENAME_LEN];  // path to the file
-  FGetFileInfo      getFileInfo;
+  void *   pTsdb;
-  FGetWalInfo       getWalInfo;
+  FGetWalInfo       getWalInfoFp;
-  FWriteToCache     writeToCache;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
+  FNotifyRole       notifyRoleFp;
-  FNotifyFlowCtrl   notifyFlowCtrl;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
-  FNotifyFileSynced notifyFileSynced;
+  FStartSyncFile    startSyncFileFp;
-  FGetVersion       getVersion;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/kit/shell/src/shellEngine.c

@@ -470,7 +470,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
 
   wordexp_t full_path;
 
-  if (wordexp(fname, &full_path, 0) != 0) {
+  if (wordexp((char *)fname, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
     return -1;
   }

src/mnode/src/mnodePeer.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tsched.h"
-#include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
 #include "tbn.h"

src/mnode/src/mnodeSdb.c

@@ -242,11 +242,6 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeEpSet(NULL);
 }
 
-static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
-  sdbUpdateMnodeRoles();
-  return 0;
-}
-
 static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
   return walGetWalFile(tsSdbMgmt.wal, fileName, fileId);
 }
@@ -262,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbUpdateMnodeRoles();
 }
 
-static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
+static void sdbStartFileSync(int32_t vgId) {}
+
+static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {}
 
 static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
 
@@ -396,14 +393,14 @@ int32_t sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.getFileInfo = sdbGetFileInfo;
+  syncInfo.getWalInfoFp = sdbGetWalInfo;
-  syncInfo.getWalInfo = sdbGetWalInfo;
+  syncInfo.writeToCacheFp = sdbWriteFwdToQueue;
-  syncInfo.writeToCache = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
-  syncInfo.notifyRole = sdbNotifyRole;
+  syncInfo.notifyRoleFp = sdbNotifyRole;
-  syncInfo.notifyFileSynced = sdbNotifyFileSynced;
+  syncInfo.startSyncFileFp = sdbStartFileSync;
-  syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
+  syncInfo.stopSyncFileFp = sdbStopFileSync;
-  syncInfo.getVersion = sdbGetSyncVersion;
+  syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl;
+  syncInfo.getVersionFp = sdbGetSyncVersion;
   tsSdbMgmt.cfg = syncCfg;
 
   if (tsSdbMgmt.sync) {

src/os/inc/osDarwin.h

@@ -105,6 +105,8 @@ typedef int(*__compar_fn_t)(const void *, const void *);
   #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
 #endif
 
+#define TAOS_OS_FUNC_PTHREAD_RWLOCK
+
 int64_t tsosStr2int64(char *str);
 
 #include "eok.h"

src/os/inc/osFile.h

@@ -26,6 +26,7 @@ int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
 int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
+int64_t taosCopy(char *from, char *to);
 
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)

src/os/inc/osMemory.h

@@ -35,7 +35,7 @@ void   taosDumpMemoryLeak();
 void * taosTMalloc(size_t size);
 void * taosTCalloc(size_t nmemb, size_t size);
 void * taosTRealloc(void *ptr, size_t size);
-void   taosTZfree(void *ptr);
+void * taosTZfree(void *ptr);
 size_t taosTSizeof(void *ptr);
 void   taosTMemset(void *ptr, int c);
 

src/os/inc/osSemphone.h

@@ -28,6 +28,21 @@ extern "C" {
   #define tsem_destroy sem_destroy
 #endif
 
+#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK
+  #define pthread_rwlock_t pthread_mutex_t
+  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+
+  #define pthread_spinlock_t pthread_mutex_t
+  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+#endif
+
 // TAOS_OS_FUNC_SEMPHONE_PTHREAD
 bool    taosCheckPthreadValid(pthread_t thread);
 int64_t taosGetSelfPthreadId();

src/os/inc/osSysinfo.h

@@ -21,10 +21,16 @@ extern "C" {
 #endif
 
 // TAOS_OS_FUNC_SYSINFO
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SysDiskSize;
+
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 void taosGetSystemInfo();
 bool taosGetProcIO(float *readKB, float *writeKB);
 bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetDisk();
+void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;
 bool taosGetProcMemory(float *memoryUsedMB) ;
 bool taosGetSysMemory(float *memoryUsedMB);

src/os/inc/osWindows.h

@@ -201,13 +201,15 @@ int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
 typedef struct {
   int    we_wordc;
-  char **we_wordv;
+  char  *we_wordv[1];
   int    we_offs;
-  char   wordPos[20];
+  char   wordPos[1025];
 } wordexp_t;
-int  wordexp(const char *words, wordexp_t *pwordexp, int flags);
+int  wordexp(char *words, wordexp_t *pwordexp, int flags);
 void wordfree(wordexp_t *pwordexp);
 
+char *realpath(char *path, char *resolved_path);
+
 #define openlog(a, b, c)
 #define closelog()
 #define LOG_ERR 0

src/os/src/darwin/darwinSysInfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 #include <errno.h>
 #include <libproc.h>
 
@@ -70,8 +71,6 @@ void taosGetSystemInfo() {
   taosGetSystemLocale();
 }
 
-bool taosGetDisk() { return true; }
-
 bool taosGetProcIO(float *readKB, float *writeKB) {
   *readKB = 0;
   *writeKB = 0;
@@ -106,6 +105,19 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+  struct statvfs info;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  } else {
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
+  }
+}
+
 char cmdline[1024];
 
 char *taosGetCmdlineByPID(int pid) {

src/os/src/detail/osFile.c

@@ -119,6 +119,40 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
   return (int64_t)lseek(fd, (long)offset, whence);
 }
 
+int64_t taosCopy(char *from, char *to) {
+  char    buffer[4096];
+  int     fidto = -1, fidfrom = -1;
+  int64_t size = 0;
+  int64_t bytes;
+
+  fidfrom = open(from, O_RDONLY | O_BINARY);
+  if (fidfrom < 0) goto _err;
+
+  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
+  if (fidto < 0) goto _err;
+
+  while (true) {
+    bytes = taosRead(fidfrom, buffer, sizeof(buffer));
+    if (bytes < 0) goto _err;
+    if (bytes == 0) break;
+
+    size += bytes;
+
+    if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err;
+    if (bytes < sizeof(buffer)) break;
+  }
+
+  close(fidfrom);
+  close(fidto);
+  return size;
+
+_err:
+  if (fidfrom >= 0) close(fidfrom);
+  if (fidto >= 0) close(fidto);
+  remove(to);
+  return -1;
+}
+
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {

src/os/src/detail/osMemory.c

@@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) {
   return (void *)((char *)tptr + sizeof(size_t));
 }
 
-void taosTZfree(void *ptr) {
+void* taosTZfree(void* ptr) {
   if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
+    free((void*)((char*)ptr - sizeof(size_t)));
   }
+  return NULL;
 }

src/os/src/detail/osSysinfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 
 #ifndef TAOS_OS_FUNC_SYSINFO
 
@@ -316,37 +317,17 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   struct statvfs info;
-  const double   unit = 1024 * 1024 * 1024;
+  if (statvfs(tsDataDir, &info)) {
-
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
-  if (tscEmbedded) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    if (statvfs(tsDataDir, &info)) {
+    return -1;
-      uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
-      return false;
-    } else {
-      tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-      tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-    }
-  }
-
-  if (statvfs(tsLogDir, &info)) {
-    uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
-    return false;
   } else {
-    tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
+    diskSize->tsize = info.f_blocks * info.f_frsize;
-    tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
   }
-
-  if (statvfs("/tmp", &info)) {
-    uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
-    return false;
-  } else {
-    tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-  }
-
-  return true;
 }
 
 static bool taosGetCardInfo(int64_t *bytes) {
@@ -510,7 +491,7 @@ void taosGetSystemInfo() {
   float tmp1, tmp2;
   taosGetSysMemory(&tmp1);
   taosGetProcMemory(&tmp2);
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
@@ -537,7 +518,6 @@ void taosPrintOsInfo() {
   uInfo(" os release:             %s", buf.release);
   uInfo(" os version:             %s", buf.version);
   uInfo(" os machine:             %s", buf.machine);
-  uInfo("==================================");
 }
 
 void taosKillSystem() {

src/os/src/windows/wSysinfo.c

@@ -21,6 +21,7 @@
 #include "ttimer.h"
 #include "tulog.h"
 #include "tutil.h"
+#include "taoserror.h"
 #if (_WIN64)
 #include <iphlpapi.h>
 #include <mswsock.h>
@@ -126,37 +127,22 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
-  const double    unit = 1024 * 1024 * 1024;
-  BOOL            fResult;
   unsigned _int64 i64FreeBytesToCaller;
   unsigned _int64 i64TotalBytes;
   unsigned _int64 i64FreeBytes;
 
-  if (tscEmbedded) {
+  BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-    fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
+                                     (PULARGE_INTEGER)&i64FreeBytes);
-                                  (PULARGE_INTEGER)&i64FreeBytes);
-    if (fResult) {
-      tsTotalDataDirGB = (float)(i64TotalBytes / unit);
-      tsAvailDataDirGB = (float)(i64FreeBytes / unit);
-    }
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
   if (fResult) {
-    tsTotalLogDirGB = (float)(i64TotalBytes / unit);
+    diskSize->tsize = (int64_t)(i64TotalBytes);
-    tsAvailLogDirGB = (float)(i64FreeBytes / unit);
+    diskSize->avail = (int64_t)(i64FreeBytes);
+    return 0;
+  } else {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   }
-
-  fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
-  if (fResult) {
-    tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
-    tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
-  }
-
-  return true;
 }
 
 bool taosGetBandSpeed(float *bandSpeedKb) {
@@ -207,7 +193,7 @@ void taosGetSystemInfo() {
   tsTotalMemoryMB = taosGetTotalMemory();
 
   float tmp1, tmp2;
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);

src/os/src/windows/wWordexp.c

@@ -21,13 +21,24 @@
 #include "tulog.h"
 #include "tutil.h"
 
-int wordexp(const char *words, wordexp_t *pwordexp, int flags) {
+int wordexp(char *words, wordexp_t *pwordexp, int flags) {
   pwordexp->we_offs = 0;
   pwordexp->we_wordc = 1;
-  pwordexp->we_wordv = (char **)(pwordexp->wordPos);
+  pwordexp->we_wordv[0] = pwordexp->wordPos;
-  pwordexp->we_wordv[0] = (char *)words;
+
+  memset(pwordexp->wordPos, 0, 1025);
+  if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) {
+    pwordexp->we_wordv[0] = words;
+    uError("failed to parse relative path:%s to abs path", words);
+    return -1;
+  }
+
+  uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos);
   return 0;
 }
 
 void wordfree(wordexp_t *pwordexp) {}
 
+char *realpath(char *path, char *resolved_path) {
+  return _fullpath(path, resolved_path, TSDB_FILENAME_LEN - 1);
+}

src/plugins/http/src/httpRestJson.c

@@ -35,7 +35,7 @@ void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t
   // data row array end
   httpJsonToken(jsonBuf, JsonArrEnd);
 
-  cmd->numOfRows = affect_rows;
+  cmd->numOfRows = 1;
 }
 
 void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) {

src/plugins/monitor/src/monMain.c

@@ -20,7 +20,6 @@
 #include "tlog.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "tsystem.h"
 #include "tscUtil.h"
 #include "tsclient.h"
 #include "dnode.h"

src/query/inc/queryLog.h

@@ -30,6 +30,7 @@ extern uint32_t qDebugFlag;
 #define qInfo(...)  do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", 255, __VA_ARGS__); }}        while(0)
 #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
 #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP)  { taosDumpData((unsigned char *)a, l); }} while(0)
 
 #ifdef __cplusplus
 }

src/query/src/qExecutor.c

@@ -7160,14 +7160,23 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
     if (f) {
       off_t s = lseek(fileno(f), 0, SEEK_END);
 
-      qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, s);
+      qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, (uint64_t)s);
       if (fseek(f, 0, SEEK_SET) >= 0) {
         size_t sz = fread(data, 1, s, f);
         if(sz < s) {  // todo handle error
+          qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s);
           assert(0);
         }
       } else {
         UNUSED(s);
+        qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno));
+        assert(0);
+      }
+
+      if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) {
+        qDump(data, s);
+        
+        assert(0);
       }
 
       fclose(f);

src/rpc/src/rpcTcp.c

@@ -364,10 +364,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
 }
 
 void taosStopTcpClient(void *chandle) {
-  SThreadObj *pThreadObj = chandle;
+  SClientObj *pClientObj = chandle;
-  if (pThreadObj == NULL) return;
 
-  tDebug ("%s TCP client is stopped", pThreadObj->label);
+  if (pClientObj == NULL) return;
+
+  tDebug ("%s TCP client is stopped", pClientObj->label);
 }
 
 void taosCleanUpTcpClient(void *chandle) {

src/rpc/src/rpcUdp.c

@@ -15,7 +15,6 @@
 
 #include "os.h"
 #include "tsocket.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "taosdef.h"

src/sync/inc/syncInt.h

@@ -108,14 +108,17 @@ typedef struct SSyncNode {
   SSyncFwds *  pSyncFwds;  // saved forward info if quorum >1
   void *       pFwdTimer;
   void *       pRoleTimer;
-  FGetFileInfo      getFileInfo;
+  void *       pTsdb;
-  FGetWalInfo       getWalInfo;
+  FGetWalInfo       getWalInfoFp;
-  FWriteToCache     writeToCache;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
+  FNotifyRole       notifyRoleFp;
-  FNotifyFlowCtrl   notifyFlowCtrl;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
-  FNotifyFileSynced notifyFileSynced;
+  FStartSyncFile    startSyncFileFp;
-  FGetVersion       getVersion;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
   pthread_mutex_t   mutex;
 } SSyncNode;
 

src/sync/inc/syncMsg.h

@@ -99,16 +99,12 @@ typedef struct {
 
 typedef struct {
   SSyncHead head;
-  char      name[TSDB_FILENAME_LEN];
-  uint32_t  magic;
-  uint32_t  index;
   uint64_t  fversion;
-  int64_t   size;
+} SFileVersion;
-} SFileInfo;
 
 typedef struct {
   SSyncHead head;
-  int8_t    sync;
+  int8_t    ack;
 } SFileAck;
 
 typedef struct {
@@ -136,7 +132,7 @@ void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
 void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
 
 void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId);
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
 
 #ifdef __cplusplus
 }

src/sync/src/syncMain.c

@@ -174,19 +174,22 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
   pthread_mutex_init(&pNode->mutex, NULL);
 
-  pNode->getFileInfo = pInfo->getFileInfo;
+  pNode->getWalInfoFp = pInfo->getWalInfoFp;
-  pNode->getWalInfo = pInfo->getWalInfo;
+  pNode->writeToCacheFp = pInfo->writeToCacheFp;
-  pNode->writeToCache = pInfo->writeToCache;
+  pNode->notifyRoleFp = pInfo->notifyRoleFp;
-  pNode->notifyRole = pInfo->notifyRole;
   pNode->confirmForward = pInfo->confirmForward;
-  pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
+  pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp;
-  pNode->notifyFileSynced = pInfo->notifyFileSynced;
+  pNode->startSyncFileFp = pInfo->startSyncFileFp;
-  pNode->getVersion = pInfo->getVersion;
+  pNode->stopSyncFileFp = pInfo->stopSyncFileFp;
+  pNode->getVersionFp = pInfo->getVersionFp;
+  pNode->sendFileFp = pInfo->sendFileFp;
+  pNode->recvFileFp = pInfo->recvFileFp;
 
   pNode->selfIndex = -1;
   pNode->vgId = pInfo->vgId;
   pNode->replica = pCfg->replica;
   pNode->quorum = pCfg->quorum;
+  pNode->pTsdb = pInfo->pTsdb;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
 
   pNode->refCount = 1;
@@ -248,8 +251,8 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   syncAddArbitrator(pNode);
   taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *));
 
-  if (pNode->notifyRole) {
+  if (pNode->notifyRoleFp) {
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -357,7 +360,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   if (pNewCfg->replica <= 1) {
     sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
     nodeRole = TAOS_SYNC_ROLE_MASTER;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -417,7 +420,7 @@ void syncRecover(int64_t rid) {
   // if take this node to unsync state, the whole system may not work
 
   nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   nodeVersion = 0;
 
   pthread_mutex_lock(&pNode->mutex);
@@ -625,8 +628,8 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
     pNode->peerInfo[index]->numOfRetrieves = 0;
   }
 
-  if (pNode->notifyFlowCtrl) {
+  if (pNode->notifyFlowCtrlFp) {
-    (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+    (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 }
 
@@ -694,7 +697,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
       taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER);
 
       syncResetFlowCtrl(pNode);
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     } else {
       pPeer = pNode->peerInfo[index];
       sInfo("%s, it shall work as master", pPeer->id);
@@ -730,7 +733,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
         sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
        }
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     }
   } else {
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -742,7 +745,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         if (masterIndex == pNode->selfIndex) {
           sError("%s, peer is master, work as slave instead", pTemp->id);
           nodeRole = TAOS_SYNC_ROLE_SLAVE;
-          (*pNode->notifyRole)(pNode->vgId, nodeRole);
+          (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
         }
       }
     }
@@ -759,7 +762,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) {
   if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
     sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version);
     nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     code = -1;
 
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -796,7 +799,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
       } else {
         sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
         nodeRole = TAOS_SYNC_ROLE_SLAVE;
-        (*pNode->notifyRole)(pNode->vgId, nodeRole);
+        (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
       }
     } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
       sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
@@ -989,7 +992,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
 
   if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
     // nodeVersion = pHead->version;
-    (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+    (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   } else {
     if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
       syncSaveIntoBuffer(pPeer, pHead);

src/sync/src/syncMsg.c

@@ -102,9 +102,9 @@ void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) {
   syncBuildHead(&pMsg->head);
 }
 
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId) {
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) {
   pMsg->head.type = TAOS_SMSG_SYNC_FILE;
   pMsg->head.vgId = vgId;
-  pMsg->head.len = sizeof(SFileInfo) - sizeof(SSyncHead);
+  pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead);
   syncBuildHead(&pMsg->head);
 }

src/sync/src/syncRestore.c

@@ -25,139 +25,44 @@
 #include "tsync.h"
 #include "syncInt.h"
 
-static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) {
+static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) {
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  char       fname[TSDB_FILENAME_LEN * 3] = {0};
-  uint32_t   magic;
-  uint64_t   fversion;
-  int64_t    size;
-  uint32_t   index = sindex;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (sindex < 0 || eindex < sindex) return;
+  SFileVersion fileVersion;
-
+  memset(&fileVersion, 0, sizeof(SFileVersion));
-  sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex);
+  int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
-
+  if (ret != sizeof(SFileVersion)) {
-  while (1) {
+    sError("%s, failed to read fver since %s", pPeer->id, strerror(errno));
-    name[0] = 0;
+    return -1;
-    magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion);
-    if (magic == 0) break;
-
-    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
-    (void)remove(fname);
-    sInfo("%s, %s is removed for its extra", pPeer->id, fname);
-
-    index++;
-    if (index > eindex) break;
   }
+
+  SFileAck fileVersionAck;
+  memset(&fileVersionAck, 0, sizeof(SFileAck));
+  syncBuildFileAck(&fileVersionAck, pNode->vgId);
+  ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  *fversion = htobe64(fileVersion.fversion);
+  return 0;
 }
 
 static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  uint32_t   pindex = 0;  // index in last restore
-  bool       fileChanged = false;
 
-  *fversion = 0;
+  if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
-  sinfo.index = -1;
+    sError("%s, failed to restore file", pPeer->id);
-  while (1) {
+    return -1;
-    // read file info
-    minfo.index = -1;
-    int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo) || minfo.index == -1) {
-      sError("%s, failed to read fileinfo while restore file since %s", pPeer->id, strerror(errno));
-      break;
-    }
-
-    assert(ret == sizeof(SFileInfo));
-    ret = syncCheckHead((SSyncHead *)(&minfo));
-    if (ret != 0) {
-      sError("%s, failed to check fileinfo while restore file since %s", pPeer->id, strerror(ret));
-      break;
-    }
-
-    // if no more file from master, break;
-    if (minfo.name[0] == 0 || minfo.magic == 0) {
-      sDebug("%s, no more files to restore", pPeer->id);
-
-      // remove extra files after the current index
-      if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX);
-      code = 0;
-      break;
-    }
-
-    sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id,
-           minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic);
-
-    // remove extra files on slave between the current and last index
-    syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
-    pindex = minfo.index;
-
-    // check the file info
-    sinfo = minfo;
-    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
-    sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, sinfo.name,
-           sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic);
-
-    // if file not there or magic is not the same, file shall be synced
-    memset(&fileAck, 0, sizeof(SFileAck));
-    syncBuildFileAck(&fileAck, pNode->vgId);
-    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.size != minfo.size || sinfo.name[0] == 0) ? 1 : 0;
-
-    // send file ack
-    ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      sDebug("%s, %s is the same", pPeer->id, minfo.name);
-      continue;
-    } else {
-      sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-    }
-
-    // if sync is required, open file, receive from master, and write to file
-    // get the full path to file
-    minfo.name[sizeof(minfo.name) - 1] = 0;
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
-
-    int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (dfd < 0) {
-      sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
-    fsync(dfd);
-    close(dfd);
-    if (ret < 0) {
-      sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    fileChanged = true;
-    sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
   }
 
-  if (code == 0 && fileChanged) {
+  if (syncRecvFileVersion(pPeer, fversion) < 0) {
-    // data file is changed, code shall be set to 1
+    return -1;
-    *fversion = minfo.fversion;
-    code = 1;
-    sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
   }
 
-  if (code < 0) {
+  sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion);
-    sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno));
+  return 0;
-  }
-
-  return code;
 }
 
 static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
@@ -195,7 +100,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
     }
     lastVer = pHead->version;
 
-    ret = (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
+    ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
     if (ret != 0) {
       sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version);
       break;
@@ -215,7 +120,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
   SSyncNode *pNode = pPeer->pSyncNode;
   SWalHead * pHead = (SWalHead *)offset;
 
-  (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+  (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   offset += pHead->len + sizeof(SWalHead);
 
   return offset;
@@ -315,20 +220,16 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
   sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId);
 
   sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+  (*pNode->startSyncFileFp)(pNode->vgId);
+
   int32_t code = syncRestoreFile(pPeer, &fversion);
   if (code < 0) {
-    sError("%s, failed to restore file", pPeer->id);
+    (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
+    sError("%s, failed to restore files", pPeer->id);
     return -1;
   }
 
-  // if code > 0, data file is changed, notify app, and pass the version
+  (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
-  if (code > 0 && pNode->notifyFileSynced) {
-    if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) {
-      sError("%s, app not in ready state", pPeer->id);
-      return -1;
-    }
-  }
-
   nodeVersion = fversion;
 
   sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
@@ -368,7 +269,7 @@ void *syncRestoreData(void *param) {
   atomic_add_fetch_32(&tsSyncNum, 1);
   sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
 
-  (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
+  (*pNode->notifyRoleFp)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
 
   if (syncOpenRecvBuffer(pNode) < 0) {
     sError("%s, failed to allocate recv buffer, restart connection", pPeer->id);
@@ -385,7 +286,7 @@ void *syncRestoreData(void *param) {
     }
   }
 
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
 
   nodeSStatus = TAOS_SYNC_STATUS_INIT;
   sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);

src/sync/src/syncRetrieve.c

@@ -27,7 +27,7 @@
 
 static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return -1;
@@ -39,7 +39,7 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return true;
@@ -55,7 +55,7 @@ static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     return -1;
@@ -67,7 +67,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     pPeer->fileChanged = 1;
@@ -84,104 +84,54 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   return false;
 }
 
+static int32_t syncSendFileVersion(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  SFileVersion fileVersion;
+  memset(&fileVersion, 0, sizeof(SFileVersion));
+  syncBuildFileVersion(&fileVersion, pNode->vgId);
+
+  uint64_t fver = pPeer->lastFileVer;
+  fileVersion.fversion = htobe64(fver);
+  int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
+  if (ret != sizeof(SFileVersion)) {
+    sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno));
+    return -1;
+  }
+
+  SFileAck fileAck;
+  memset(&fileAck, 0, sizeof(SFileAck));
+  ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  // set the peer sync version
+  pPeer->sversion = fver;
+
+  return 0;
+}
+
 static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
 
   if (syncGetFileVersion(pNode, pPeer) < 0) {
     pPeer->fileChanged = 1;
     return -1;
   }
 
-  while (1) {
+  if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
-    // retrieve file info
+    sError("%s, failed to retrieve file", pPeer->id);
-    fileInfo.name[0] = 0;
+    return -1;
-    fileInfo.size = 0;
-    fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
-                                           &fileInfo.size, &fileInfo.fversion);
-    syncBuildFileInfo(&fileInfo, pNode->vgId);
-    sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, fileInfo.name,
-           fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic);
-
-    // send the file info
-    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo)) {
-      code = -1;
-      sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    // if no file anymore, break
-    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
-      code = 0;
-      sDebug("%s, no more files to sync", pPeer->id);
-      break;
-    }
-
-    // wait for the ack from peer
-    ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      code = -1;
-      sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    ret = syncCheckHead((SSyncHead*)(&fileAck));
-    if (ret != 0) {
-      code = -1;
-      sError("%s, failed to check file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(ret));
-      break;
-    }
-
-    // set the peer sync version
-    pPeer->sversion = fileInfo.fversion;
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      fileInfo.index++;
-      sDebug("%s, %s is the same, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-      continue;
-    } else {
-      sDebug("%s, %s will be sent, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-    }
-
-    // get the full path to file
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
-
-    // send the file to peer
-    int32_t sfd = open(name, O_RDONLY | O_BINARY);
-    if (sfd < 0) {
-      code = -1;
-      sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    ret = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
-    close(sfd);
-    if (ret < 0) {
-      code = -1;
-      sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
-    fileInfo.index++;
-
-    // check if processed files are modified
-    if (syncAreFilesModified(pNode, pPeer)) {
-      code = -1;
-      break;
-    }
   }
 
-  if (code != TSDB_CODE_SUCCESS) {
+  if (syncSendFileVersion(pPeer) < 0) {
-    sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code);
+    return -1;
   }
 
-  return code;
+  sInfo("%s, all files are retrieved", pPeer->id);
+  return 0;
 }
 
 // if only a partial record is read out, upper layer will reload the file to get a complete record
@@ -345,7 +295,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   while (1) {
     // retrieve wal info
     wname[0] = 0;
-    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
+    code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index);
     if (code < 0) {
       sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code);
       break;
@@ -477,7 +427,7 @@ void *syncRetrieveData(void *param) {
   sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus],
         pPeer->numOfRetrieves);
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves);
 
   pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (pPeer->syncFd < 0) {
@@ -497,10 +447,10 @@ void *syncRetrieveData(void *param) {
     pPeer->numOfRetrieves++;
   } else {
     pPeer->numOfRetrieves = 0;
-    // if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+    // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
 
   pPeer->fileChanged = 0;
   taosCloseSocket(pPeer->syncFd);

src/sync/test/syncServer.c

@@ -296,11 +296,10 @@ void initSync() {
   pCfg->replica = 1;
   pCfg->quorum = 1;
   syncInfo.vgId = 1;
-  syncInfo.getFileInfo = getFileInfo;
+  syncInfo.getWalInfoFp = getWalInfo;
-  syncInfo.getWalInfo = getWalInfo;
+  syncInfo.writeToCacheFp = writeToCache;
-  syncInfo.writeToCache = writeToCache;
   syncInfo.confirmForward = confirmForward;
-  syncInfo.notifyRole = notifyRole;
+  syncInfo.notifyRoleFp = notifyRole;
 
   pCfg->nodeInfo[0].nodeId = 1;
   pCfg->nodeInfo[0].nodePort = 7010;

src/tfs/CMakeLists.txt

@@ -0,0 +1,12 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(tfs ${SRC})
+TARGET_LINK_LIBRARIES(tfs tutil)
+
+IF (TD_LINUX)
+  # Someone has no gtest directory, so comment it
+  # ADD_SUBDIRECTORY(tests)
+ENDIF ()

src/tfs/inc/tfsint.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFSINT_H
+#define TD_TFSINT_H
+
+#include "tlog.h"
+#include "tglobal.h"
+#include "tfs.h"
+#include "tcoding.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int fsDebugFlag;
+
+// For debug purpose
+#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }}
+#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }}
+#define fWarn(...)  { if (fsDebugFlag & DEBUG_WARN)  { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }}
+#define fInfo(...)  { if (fsDebugFlag & DEBUG_INFO)  { taosPrintLog("TFS ", 255, __VA_ARGS__); }}
+#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+
+// Global Definitions
+#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
+
+// tdisk.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+} SDiskMeta;
+
+typedef struct SDisk {
+  int       level;
+  int       id;
+  char      dir[TSDB_FILENAME_LEN];
+  SDiskMeta dmeta;
+} SDisk;
+
+#define DISK_LEVEL(pd) ((pd)->level)
+#define DISK_ID(pd) ((pd)->id)
+#define DISK_DIR(pd) ((pd)->dir)
+#define DISK_META(pd) ((pd)->dmeta)
+#define DISK_SIZE(pd) ((pd)->dmeta.size)
+#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free)
+
+SDisk *tfsNewDisk(int level, int id, const char *dir);
+SDisk *tfsFreeDisk(SDisk *pDisk);
+int    tfsUpdateDiskInfo(SDisk *pDisk);
+
+// ttier.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+typedef struct STier {
+  pthread_spinlock_t lock;
+  int                level;
+  int16_t            ndisk;   // # of disks mounted to this tier
+  int16_t            nextid;  // next disk id to allocate
+  STierMeta          tmeta;
+  SDisk *            disks[TSDB_MAX_DISKS_PER_TIER];
+} STier;
+
+#define TIER_LEVEL(pt) ((pt)->level)
+#define TIER_NDISKS(pt) ((pt)->ndisk)
+#define TIER_SIZE(pt) ((pt)->tmeta.size)
+#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free)
+#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks)
+#define DISK_AT_TIER(pt, id) ((pt)->disks[id])
+
+int    tfsInitTier(STier *pTier, int level);
+void   tfsDestroyTier(STier *pTier);
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
+void   tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta);
+int    tfsAllocDiskOnTier(STier *pTier);
+void   tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta);
+void   tfsPosNextId(STier *pTier);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/tfs/src/tdisk.c

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taoserror.h"
+#include "tfsint.h"
+
+// PROTECTED ====================================
+SDisk *tfsNewDisk(int level, int id, const char *dir) {
+  SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk));
+  if (pDisk == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pDisk->level = level;
+  pDisk->id = id;
+  strncpy(pDisk->dir, dir, TSDB_FILENAME_LEN);
+
+  return pDisk;
+}
+
+SDisk *tfsFreeDisk(SDisk *pDisk) {
+  if (pDisk) {
+    free(pDisk);
+  }
+  return NULL;
+}
+
+int tfsUpdateDiskInfo(SDisk *pDisk) {
+  ASSERT(pDisk != NULL);
+
+  SysDiskSize diskSize = {0};
+
+  int code = taosGetDiskSize(pDisk->dir, &diskSize);
+  if (code != 0) {
+    fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir,
+           strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+  }
+
+  pDisk->dmeta.size = diskSize.tsize;
+  pDisk->dmeta.free = diskSize.tsize - diskSize.avail;
+
+  return code;
+}

src/tfs/src/tfs.c

@@ -0,0 +1,600 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+
+#include "hash.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfs.h"
+#include "tfsint.h"
+
+#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
+
+typedef struct {
+  pthread_spinlock_t lock;
+  SFSMeta            meta;
+  int                nlevel;
+  STier              tiers[TSDB_MAX_TIERS];
+  SHashObj *         map;  // name to did map
+} SFS;
+
+typedef struct {
+  SDisk *pDisk;
+} SDiskIter;
+
+#define TFS_META() (pfs->meta)
+#define TFS_NLEVEL() (pfs->nlevel)
+#define TFS_TIERS() (pfs->tiers)
+#define TFS_TIER_AT(level) (TFS_TIERS() + (level))
+#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id)
+#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID)
+#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL()))
+#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level))))
+#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id))
+
+#define tfsLock() pthread_spin_lock(&(pfs->lock))
+#define tfsUnLock() pthread_spin_unlock(&(pfs->lock))
+
+static SFS  tfs = {0};
+static SFS *pfs = &tfs;
+
+// STATIC DECLARATION
+static int    tfsMount(SDiskCfg *pCfg);
+static int    tfsCheck();
+static int    tfsCheckAndFormatCfg(SDiskCfg *pCfg);
+static int    tfsFormatDir(char *idir, char *odir);
+static SDisk *tfsGetDiskByID(SDiskID did);
+static SDisk *tfsGetDiskByName(const char *dir);
+static int    tfsOpendirImpl(TDIR *tdir);
+static void   tfsInitDiskIter(SDiskIter *pIter);
+static SDisk *tfsNextDisk(SDiskIter *pIter);
+
+// FS APIs ====================================
+int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
+  ASSERT(ndisk > 0);
+
+  for (int level = 0; level < TSDB_MAX_TIERS; level++) {
+    if (tfsInitTier(TFS_TIER_AT(level), level) < 0) {
+      while (true) {
+        level--;
+        if (level < 0) break;
+
+        tfsDestroyTier(TFS_TIER_AT(level));
+      }
+
+      return -1;
+    }
+  }
+
+  pthread_spin_init(&(pfs->lock), 0);
+
+  pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
+                          taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (pfs->map == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    tfsDestroy();
+    return -1;
+  }
+
+  for (int idisk = 0; idisk < ndisk; idisk++) {
+    if (tfsMount(pDiskCfg + idisk) < 0) {
+      tfsDestroy();
+      return -1;
+    }
+  }
+
+  if (tfsCheck() < 0) {
+    tfsDestroy();
+    return -1;
+  }
+
+  tfsUpdateInfo(NULL);
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsPosNextId(TFS_TIER_AT(level));
+  }
+
+  return 0;
+}
+
+void tfsDestroy() {
+  taosHashCleanup(pfs->map);
+  pfs->map = NULL;
+
+  pthread_spin_destroy(&(pfs->lock));
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsDestroyTier(TFS_TIER_AT(level));
+  }
+}
+
+void tfsUpdateInfo(SFSMeta *pFSMeta) {
+  SFSMeta   fsMeta;
+  STierMeta tierMeta;
+
+  if (pFSMeta == NULL) {
+    pFSMeta = &fsMeta;
+  }
+
+  memset(pFSMeta, 0, sizeof(*pFSMeta));
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    tfsUpdateTierInfo(pTier, &tierMeta);
+    pFSMeta->tsize += tierMeta.size;
+    pFSMeta->avail += tierMeta.free;
+  }
+
+  tfsLock();
+  pfs->meta = *pFSMeta;
+  tfsUnLock();
+}
+
+void tfsGetMeta(SFSMeta *pMeta) {
+  ASSERT(pMeta);
+
+  tfsLock();
+  *pMeta = pfs->meta;
+  tfsUnLock();
+}
+
+/* Allocate an existing available tier level
+ */
+void tfsAllocDisk(int expLevel, int *level, int *id) {
+  ASSERT(expLevel >= 0);
+
+  *level = expLevel;
+  *id = TFS_UNDECIDED_ID;
+
+  if (*level >= TFS_NLEVEL()) {
+    *level = TFS_NLEVEL() - 1;
+  }
+
+  while (*level >= 0) {
+    *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level));
+    if (*id == TFS_UNDECIDED_ID) {
+      (*level)--;
+      continue;
+    }
+
+    return;
+  }
+
+  *level = TFS_UNDECIDED_LEVEL;
+  *id = TFS_UNDECIDED_ID;
+}
+
+const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); }
+const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
+
+// TFILE APIs ====================================
+void tfsInitFile(TFILE *pf, int level, int id, const char *bname) {
+  ASSERT(TFS_IS_VALID_DISK(level, id));
+
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+
+  pf->level = level;
+  pf->id = id;
+  strncpy(pf->rname, bname, TSDB_FILENAME_LEN);
+
+  char tmpName[TMPNAME_LEN] = {0};
+  snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname);
+  tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN);
+}
+
+bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) {
+  ASSERT(pf1 != NULL || pf2 != NULL);
+  if (pf1 == NULL || pf2 == NULL) return false;
+  if (pf1->level != pf2->level) return false;
+  if (pf1->id != pf2->id) return false;
+  if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false;
+  return true;
+}
+
+int tfsEncodeFile(void **buf, TFILE *pf) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pf->level);
+  tlen += taosEncodeVariantI32(buf, pf->id);
+  tlen += taosEncodeString(buf, pf->rname);
+
+  return tlen;
+}
+
+void *tfsDecodeFile(void *buf, TFILE *pf) {
+  int32_t level, id;
+  char *  rname;
+
+  buf = taosDecodeVariantI32(buf, &(level));
+  buf = taosDecodeVariantI32(buf, &(id));
+  buf = taosDecodeString(buf, &rname);
+
+  tfsInitFile(pf, level, id, rname);
+
+  tfree(rname);
+  return buf;
+}
+
+void tfsbasename(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, basename(tname), TSDB_FILENAME_LEN);
+}
+
+void tfsdirname(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
+}
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id) {
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+  char   aname[TMPNAME_LEN];
+
+  snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+  if (taosMkDir(aname, 0755) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tfsMkdirRecurAt(const char *rname, int level, int id) {
+  if (tfsMkdirAt(rname, level, id) < 0) {
+    if (errno == ENOENT) {
+      // Try to create upper
+      char *s = strdup(rname);
+
+      if (tfsMkdirRecurAt(dirname(s), level, id) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
+
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    } else {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tfsMkdir(const char *rname) {
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int tfsRmdir(const char *rname) {
+  char aname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+
+      taosRemoveDir(aname);
+    }
+  }
+
+  return 0;
+}
+
+int tfsRename(char *orname, char *nrname) {
+  char oaname[TMPNAME_LEN] = "\0";
+  char naname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < pfs->nlevel; level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
+      snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname);
+
+      taosRename(oaname, naname);
+    }
+  }
+
+  return 0;
+}
+
+struct TDIR {
+  SDiskIter iter;
+  int       level;
+  int       id;
+  char      dirname[TSDB_FILENAME_LEN];
+  TFILE     tfile;
+  DIR *     dir;
+};
+
+TDIR *tfsOpendir(const char *rname) {
+  TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
+  if (tdir == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  tfsInitDiskIter(&(tdir->iter));
+  strncpy(tdir->dirname, rname, TSDB_FILENAME_LEN);
+
+  if (tfsOpendirImpl(tdir) < 0) {
+    free(tdir);
+    return NULL;
+  }
+
+  return tdir;
+}
+
+const TFILE *tfsReaddir(TDIR *tdir) {
+  if (tdir == NULL || tdir->dir == NULL) return NULL;
+  char bname[TMPNAME_LEN * 2] = "\0";
+
+  while (true) {
+    struct dirent *dp = NULL;
+    dp = readdir(tdir->dir);
+    if (dp != NULL) {
+      // Skip . and ..
+      if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
+
+      snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name);
+      tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname);
+      return &(tdir->tfile);
+    }
+
+    if (tfsOpendirImpl(tdir) < 0) {
+      return NULL;
+    }
+
+    if (tdir->dir == NULL) {
+      terrno = TSDB_CODE_SUCCESS;
+      return NULL;
+    }
+  }
+}
+
+void tfsClosedir(TDIR *tdir) {
+  if (tdir) {
+    if (tdir->dir != NULL) {
+      closedir(tdir->dir);
+      tdir->dir = NULL;
+    }
+    free(tdir);
+  }
+}
+
+// private
+static int tfsMount(SDiskCfg *pCfg) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+
+  if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
+
+  did.level = pCfg->level;
+  pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg);
+  if (pDisk == NULL) {
+    fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
+    return -1;
+  }
+  did.id = DISK_ID(pDisk);
+
+  taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
+  if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1;
+
+  return 0;
+}
+
+static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
+  char        dirName[TSDB_FILENAME_LEN] = "\0";
+  struct stat pstat;
+
+  if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) {
+    fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (pCfg->primary) {
+    if (pCfg->level != 0) {
+      fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level);
+      terrno = TSDB_CODE_FS_INVLD_CFG;
+      return -1;
+    }
+
+    if (TFS_PRIMARY_DISK() != NULL) {
+      fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir);
+      terrno = TSDB_CODE_FS_DUP_PRIMARY;
+      return -1;
+    }
+  }
+
+  if (tfsFormatDir(pCfg->dir, dirName) < 0) {
+    fError("failed to mount %s to FS since invalid dir format", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (tfsGetDiskByName(dirName) != NULL) {
+    fError("failed to mount %s to FS since duplicate mount", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (access(dirName, W_OK | R_OK | F_OK) != 0) {
+    fError("failed to mount %s to FS since no R/W access rights", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (stat(dirName, &pstat) < 0) {
+    fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (!S_ISDIR(pstat.st_mode)) {
+    fError("failed to mount %s to FS since not a directory", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN);
+
+  return 0;
+}
+
+static int tfsFormatDir(char *idir, char *odir) {
+  wordexp_t wep = {0};
+
+  int code = wordexp(idir, &wep, 0);
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  if (realpath(wep.we_wordv[0], odir) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    wordfree(&wep);
+    return -1;
+  }
+
+  wordfree(&wep);
+  return 0;
+
+}
+
+static int tfsCheck() {
+  if (TFS_PRIMARY_DISK() == NULL) {
+    fError("no primary disk is set");
+    terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
+    return -1;
+  }
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) {
+      fError("no disk at level %d", level);
+      terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); }
+static SDisk *tfsGetDiskByName(const char *dir) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+  void *  pr = NULL;
+
+  pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
+  if (pr == NULL) return NULL;
+
+  did = *(SDiskID *)pr;
+  pDisk = tfsGetDiskByID(did);
+  ASSERT(pDisk != NULL);
+
+  return pDisk;
+}
+
+static int tfsOpendirImpl(TDIR *tdir) {
+  SDisk *pDisk = NULL;
+  char   adir[TMPNAME_LEN * 2] = "\0";
+
+  if (tdir->dir != NULL) {
+    closedir(tdir->dir);
+    tdir->dir = NULL;
+  }
+
+  while (true) {
+    pDisk = tfsNextDisk(&(tdir->iter));
+    if (pDisk == NULL) return 0;
+
+    tdir->level = DISK_LEVEL(pDisk);
+    tdir->id = DISK_ID(pDisk);
+
+    snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname);
+    tdir->dir = opendir(adir);
+    if (tdir->dir != NULL) break;
+  }
+
+  return 0;
+}
+
+static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); }
+
+static SDisk *tfsNextDisk(SDiskIter *pIter) {
+  SDisk *pDisk = pIter->pDisk;
+
+  if (pDisk == NULL) return NULL;
+
+  int level = DISK_LEVEL(pDisk);
+  int id = DISK_ID(pDisk);
+
+  id++;
+  if (id < TIER_NDISKS(TFS_TIER_AT(level))) {
+    pIter->pDisk = TFS_DISK_AT(level, id);
+    ASSERT(pIter->pDisk != NULL);
+  } else {
+    level++;
+    id = 0;
+    if (level < TFS_NLEVEL()) {
+      pIter->pDisk = TFS_DISK_AT(level, id);
+      ASSERT(pIter->pDisk != NULL);
+    } else {
+      pIter->pDisk = NULL;
+    }
+  }
+
+  return pDisk;
+}
+
+// OTHER FUNCTIONS ===================================
+void taosGetDisk() {
+  const double unit = 1024 * 1024 * 1024;
+  SysDiskSize  diskSize;
+  SFSMeta      fsMeta;
+
+  if (tscEmbedded) {
+    tfsUpdateInfo(&fsMeta);
+    tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
+    tsAvailDataDirGB = (float)(fsMeta.avail / unit);
+  }
+
+  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
+    tsTotalLogDirGB = (float)(diskSize.tsize / unit);
+    tsAvailLogDirGB = (float)(diskSize.avail / unit);
+  }
+
+  if (taosGetDiskSize("/tmp", &diskSize) == 0) {
+    tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
+    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
+  }
+}

src/tfs/src/ttier.c

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfsint.h"
+
+#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock))
+#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock))
+
+// PROTECTED ==========================================
+int tfsInitTier(STier *pTier, int level) {
+  memset((void *)pTier, 0, sizeof(*pTier));
+
+  int code = pthread_spin_init(&(pTier->lock), 0);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  pTier->level = level;
+
+  return 0;
+}
+
+void tfsDestroyTier(STier *pTier) {
+  for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
+    DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id));
+  }
+
+  pTier->ndisk = 0;
+
+  pthread_spin_destroy(&(pTier->lock));
+}
+
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
+  ASSERT(pTier->level == pCfg->level);
+
+  int    id = 0;
+  SDisk *pDisk;
+
+  if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) {
+    terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+    return NULL;
+  }
+
+  if (pTier->level == 0) {
+    if (DISK_AT_TIER(pTier, 0) != NULL) {
+      id = pTier->ndisk;
+    } else {
+      if (pCfg->primary) {
+        id = 0;
+      } else {
+        id = pTier->ndisk + 1;
+      }
+      if (id >= TSDB_MAX_DISKS_PER_TIER) {
+        terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+        return NULL;
+      }
+    }
+  } else {
+    id = pTier->ndisk;
+  }
+
+  pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
+  if (pDisk == NULL) return NULL;
+  DISK_AT_TIER(pTier, id) = pDisk;
+  pTier->ndisk++;
+
+  fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
+
+  return DISK_AT_TIER(pTier, id);
+}
+
+void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) {
+  STierMeta tmeta;
+
+  if (pTierMeta == NULL) {
+    pTierMeta = &tmeta;
+  }
+  memset(pTierMeta, 0, sizeof(*pTierMeta));
+
+  tfsLockTier(pTier);
+
+  for (int id = 0; id < pTier->ndisk; id++) {
+    if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) {
+      continue;
+    }
+    pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->nAvailDisks++;
+  }
+
+  pTier->tmeta = *pTierMeta;
+
+  tfsUnLockTier(pTier);
+}
+
+// Round-Robin to allocate disk on a tier
+int tfsAllocDiskOnTier(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int    id = TFS_UNDECIDED_ID;
+  SDisk *pDisk;
+
+  tfsLockTier(pTier);
+
+  if (TIER_AVAIL_DISKS(pTier) <= 0) {
+    tfsUnLockTier(pTier);
+    return id;
+  }
+
+  id = pTier->nextid;
+  while (true) {
+    pDisk = DISK_AT_TIER(pTier, id);
+    ASSERT(pDisk != NULL);
+
+    if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) {
+      id = (id + 1) % pTier->ndisk;
+      if (id == pTier->nextid) {
+        tfsUnLockTier(pTier);
+        return TFS_UNDECIDED_ID;
+      } else {
+        continue;
+      }
+    } else {
+      pTier->nextid = (id + 1) % pTier->ndisk;
+      break;
+    }
+  }
+
+  tfsUnLockTier(pTier);
+  return id;
+}
+
+void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) {
+  ASSERT(pTierMeta != NULL);
+
+  tfsLockTier(pTier);
+  *pTierMeta = pTier->tmeta;
+  tfsUnLockTier(pTier);
+}
+
+void tfsPosNextId(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int nextid = 0;
+
+  for (int id = 1; id < pTier->ndisk; id++) {
+    SDisk *pLDisk = DISK_AT_TIER(pTier, nextid);
+    SDisk *pDisk = DISK_AT_TIER(pTier, id);
+    if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) {
+      nextid = id;
+    }
+  }
+
+  pTier->nextid = nextid;
+}

src/tsdb/CMakeLists.txt

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tsdb ${SRC})
-TARGET_LINK_LIBRARIES(tsdb common tutil)
+TARGET_LINK_LIBRARIES(tsdb tfs common tutil)
 
 IF (TD_LINUX)
   # Someone has no gtest directory, so comment it

src/tsdb/inc/tsdbBuffer.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_BUFFER_H_
+#define _TD_TSDB_BUFFER_H_
+
+typedef struct {
+  int64_t blockId;
+  int     offset;
+  int     remain;
+  char    data[];
+} STsdbBufBlock;
+
+typedef struct {
+  pthread_cond_t poolNotEmpty;
+  int            bufBlockSize;
+  int            tBufBlocks;
+  int            nBufBlocks;
+  int64_t        index;
+  SList*         bufBlockList;
+} STsdbBufPool;
+
+#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
+
+STsdbBufPool* tsdbNewBufPool();
+void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
+int           tsdbOpenBufPool(STsdbRepo* pRepo);
+void          tsdbCloseBufPool(STsdbRepo* pRepo);
+SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
+
+#endif /* _TD_TSDB_BUFFER_H_ */

src/tsdb/inc/tsdbCommit.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_COMMIT_H_
+#define _TD_TSDB_COMMIT_H_
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
+int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
+void *tsdbCommitData(STsdbRepo *pRepo);
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
+}
+
+#endif /* _TD_TSDB_COMMIT_H_ */

src/tsdb/inc/tsdbCommitQueue.h

@@ -13,26 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSTEM_H
+#ifndef _TD_TSDB_COMMIT_QUEUE_H_
-#define TDENGINE_TSYSTEM_H
+#define _TD_TSDB_COMMIT_QUEUE_H_
 
-#ifdef __cplusplus
+int tsdbScheduleCommit(STsdbRepo *pRepo);
-extern "C" {
-#endif
 
-bool taosGetSysMemory(float *memoryUsedMB);
+#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */
-bool taosGetProcMemory(float *memoryUsedMB);
-bool taosGetDisk();
-bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
-bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetProcIO(float *readKB, float *writeKB);
-void taosGetSystemInfo();
-void taosPrintOsInfo();
-void taosKillSystem();
-void taosSetCoreDump();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/inc/tsdbFS.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_FS_H_
+#define _TD_TSDB_FS_H_
+
+#define TSDB_FS_VERSION 0
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SMFile*     pmf;   // meta file pointer
+  SMFile      mf;    // meta file
+  SArray*     df;    // data file array
+} SFSStatus;
+
+typedef struct {
+  pthread_rwlock_t lock;
+
+  SFSStatus* cstatus;    // current status
+  SHashObj*  metaCache;  // meta cache
+  bool       intxn;
+  SFSStatus* nstatus;  // new status
+} STsdbFS;
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS*   pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet* pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdbRepo *pRepo);
+void     tsdbCloseFS(STsdbRepo *pRepo);
+void     tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdbRepo *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_rdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_wrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_unlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+#endif /* _TD_TSDB_FS_H_ */

src/tsdb/inc/tsdbFile.h

@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TS_TSDB_FILE_H_
+#define _TS_TSDB_FILE_H_
+
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
+#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
+#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf))
+
+typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
+
+// =============== SMFile
+typedef struct {
+  int64_t  size;
+  int64_t  tombSize;
+  int64_t  nRecords;
+  int64_t  nDels;
+  uint32_t magic;
+} SMFInfo;
+
+typedef struct {
+  SMFInfo info;
+  TFILE   f;
+  int     fd;
+} SMFile;
+
+void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
+void  tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile);
+int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
+int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
+int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
+int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
+int   tsdbUpdateMFileHeader(SMFile* pMFile);
+int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
+int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
+int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
+void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
+
+static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
+  ASSERT(TSDB_FILE_CLOSED(pMFile));
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
+  if (pMFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
+  if (TSDB_FILE_OPENED(pMFile)) {
+    close(pMFile->fd);
+    TSDB_FILE_SET_CLOSED(pMFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pMFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pMFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
+
+static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+// =============== SDFile
+typedef struct {
+  uint32_t magic;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo info;
+  TFILE   f;
+  int     fd;
+} SDFile;
+
+void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(void* buf, SDFile* pDFile);
+int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
+  if (pDFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    close(pDFile->fd);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+typedef struct {
+  int    fid;
+  int    state;
+  SDFile files[TSDB_FILE_MAX];
+} SDFileSet;
+
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                     \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
+    }                                                                          \
+  } while (0);
+
+void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsMsPerDay[precision];
+  *maxKey = *minKey + days * tsMsPerDay[precision] - 1;
+}
+
+#endif /* _TS_TSDB_FILE_H_ */

src/tsdb/inc/tsdbLog.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */

src/tsdb/inc/tsdbMain.h

@@ -1,619 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_TSDB_MAIN_H_
-#define _TD_TSDB_MAIN_H_
-
-#include "os.h"
-#include "hash.h"
-#include "tcoding.h"
-#include "tglobal.h"
-#include "tkvstore.h"
-#include "tlist.h"
-#include "tlog.h"
-#include "tlockfree.h"
-#include "tsdb.h"
-#include "tskiplist.h"
-#include "tutil.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int32_t tsdbDebugFlag;
-
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-
-#define TSDB_MAX_TABLE_SCHEMAS 16
-#define TSDB_FILE_HEAD_SIZE    512
-#define TSDB_FILE_DELIMITER    0xF00AFA0F
-#define TSDB_FILE_INIT_MAGIC   0xFFFFFFFF
-
-#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))
-
-// NOTE: Any file format change must increase this version number by 1
-//       Also, implement the convert function
-#define TSDB_FILE_VERSION ((uint32_t)0)
-
-// Definitions
-// ------------------ tsdbMeta.c
-typedef struct STable {
-  STableId       tableId;
-  ETableType     type;
-  tstr*          name;  // NOTE: there a flexible string here
-  uint64_t       suid;
-  struct STable* pSuper;  // super table pointer
-  uint8_t        numOfSchemas;
-  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
-  STSchema*      tagSchema;
-  SKVRow         tagVal;
-  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
-  void*          eventHandler;   // TODO
-  void*          streamHandler;  // TODO
-  TSKEY          lastKey;
-  SDataRow       lastRow;
-  char*          sql;
-  void*          cqhandle;
-  SRWLatch       latch;  // TODO: implementa latch functions
-  T_REF_DECLARE()
-} STable;
-
-typedef struct {
-  pthread_rwlock_t rwLock;
-
-  int32_t   nTables;
-  int32_t   maxTables;
-  STable**  tables;
-  SList*    superList;
-  SHashObj* uidMap;
-  SKVStore* pStore;
-  int       maxRowBytes;
-  int       maxCols;
-} STsdbMeta;
-
-// ------------------ tsdbBuffer.c
-typedef struct {
-  int64_t blockId;
-  int     offset;
-  int     remain;
-  char    data[];
-} STsdbBufBlock;
-
-typedef struct {
-  pthread_cond_t poolNotEmpty;
-  int            bufBlockSize;
-  int            tBufBlocks;
-  int            nBufBlocks;
-  int64_t        index;
-  SList*         bufBlockList;
-} STsdbBufPool;
-
-// ------------------ tsdbMemTable.c
-typedef struct {
-  STable *           pTable;
-  SSkipListIterator *pIter;
-} SCommitIter;
-
-typedef struct {
-  uint64_t   uid;
-  TSKEY      keyFirst;
-  TSKEY      keyLast;
-  int64_t    numOfRows;
-  SSkipList* pData;
-} STableData;
-
-typedef struct {
-  T_REF_DECLARE()
-  SRWLatch     latch;
-  TSKEY        keyFirst;
-  TSKEY        keyLast;
-  int64_t      numOfRows;
-  int32_t      maxTables;
-  STableData** tData;
-  SList*       actList;
-  SList*       extraBuffList;
-  SList*       bufBlockList;
-} SMemTable;
-
-enum { TSDB_UPDATE_META, TSDB_DROP_META };
-
-#ifdef WINDOWS
-#pragma pack(push ,1) 
-typedef struct {
-#else
-typedef struct __attribute__((packed)){
-#endif
-  char     act;
-  uint64_t uid;
-} SActObj;
-#ifdef WINDOWS
-#pragma pack(pop) 
-#endif
-
-typedef struct {
-  int  len;
-  char cont[];
-} SActCont;
-
-// ------------------ tsdbFile.c
-extern const char* tsdbFileSuffix[];
-typedef enum {
-  TSDB_FILE_TYPE_HEAD = 0,
-  TSDB_FILE_TYPE_DATA,
-  TSDB_FILE_TYPE_LAST,
-  TSDB_FILE_TYPE_STAT,
-  TSDB_FILE_TYPE_NHEAD,
-  TSDB_FILE_TYPE_NDATA,
-  TSDB_FILE_TYPE_NLAST,
-  TSDB_FILE_TYPE_NSTAT
-} TSDB_FILE_TYPE;
-
-#ifndef TDINTERNAL
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1)
-#else
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1)
-#endif
-
-typedef struct {
-  uint32_t magic;
-  uint32_t len;
-  uint32_t totalBlocks;
-  uint32_t totalSubBlocks;
-  uint32_t offset;
-  uint64_t size;      // total size of the file
-  uint64_t tombSize;  // unused file size
-} STsdbFileInfo;
-
-typedef struct {
-  char  fname[TSDB_FILENAME_LEN];
-  int   fd;
-
-  STsdbFileInfo info;
-} SFile;
-
-typedef struct {
-  int   fileId;
-  int   state; // 0 for health, 1 for problem
-  SFile files[TSDB_FILE_TYPE_MAX];
-} SFileGroup;
-
-typedef struct {
-  pthread_rwlock_t fhlock;
-
-  int         maxFGroups;
-  int         nFGroups;
-  SFileGroup* pFGroup;
-} STsdbFileH;
-
-typedef struct {
-  int         direction;
-  STsdbFileH* pFileH;
-  int         fileId;
-  int         index;
-} SFileGroupIter;
-
-// ------------------ tsdbMain.c
-typedef struct {
-  int32_t  totalLen;
-  int32_t  len;
-  SDataRow row;
-} SSubmitBlkIter;
-
-typedef struct {
-  int32_t totalLen;
-  int32_t len;
-  void *  pMsg;
-} SSubmitMsgIter;
-
-typedef struct {
-  int8_t state;
-
-  char*           rootDir;
-  STsdbCfg        config;
-  STsdbAppH       appH;
-  STsdbStat       stat;
-  STsdbMeta*      tsdbMeta;
-  STsdbBufPool*   pPool;
-  SMemTable*      mem;
-  SMemTable*      imem;
-  STsdbFileH*     tsdbFileH;
-  tsem_t          readyToCommit;
-  pthread_mutex_t mutex;
-  bool            repoLocked;
-  int32_t         code; // Commit code
-} STsdbRepo;
-
-// ------------------ tsdbRWHelper.c
-typedef struct {
-  int32_t  tid;
-  uint32_t len;
-  uint32_t offset;
-  uint32_t hasLast : 2;
-  uint32_t numOfBlocks : 30;
-  uint64_t uid;
-  TSKEY    maxKey;
-} SCompIdx;
-
-typedef struct {
-  int64_t last : 1;
-  int64_t offset : 63;
-  int32_t algorithm : 8;
-  int32_t numOfRows : 24;
-  int32_t len;
-  int32_t keyLen;     // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols
-  int16_t numOfSubBlocks;
-  int16_t numOfCols; // not including timestamp column
-  TSKEY   keyFirst;
-  TSKEY   keyLast;
-} SCompBlock;
-
-typedef struct {
-  int32_t    delimiter;  // For recovery usage
-  int32_t    tid;
-  uint64_t   uid;
-  SCompBlock blocks[];
-} SCompInfo;
-
-typedef struct {
-  int16_t colId;
-  int32_t len;
-  int32_t type : 8;
-  int32_t offset : 24;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  char    padding[2];
-} SCompCol;
-
-typedef struct {
-  int32_t  delimiter;  // For recovery usage
-  int32_t  numOfCols;  // For recovery usage
-  uint64_t uid;        // For recovery usage
-  SCompCol cols[];
-} SCompData;
-
-typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
-
-typedef struct {
-  TSKEY      minKey;
-  TSKEY      maxKey;
-  SFileGroup fGroup;
-  SFile      nHeadF;
-  SFile      nLastF;
-} SHelperFile;
-
-typedef struct {
-  uint64_t uid;
-  int32_t  tid;
-} SHelperTable;
-
-typedef struct {
-  SCompIdx* pIdxArray;
-  int       numOfIdx;
-  int       curIdx;
-} SIdxH;
-
-typedef struct {
-  tsdb_rw_helper_t type;
-
-  STsdbRepo* pRepo;
-  int8_t     state;
-  // For file set usage
-  SHelperFile files;
-  SIdxH       idxH;
-  SCompIdx    curCompIdx;
-  void*       pWIdx;
-  // For table set usage
-  SHelperTable tableInfo;
-  SCompInfo*   pCompInfo;
-  bool         hasOldLastBlock;
-  // For block set usage
-  SCompData* pCompData;
-  SDataCols* pDataCols[2];
-  void*      pBuffer;     // Buffer to hold the whole data block
-  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
-} SRWHelper;
-
-typedef struct {
-  int   rowsInserted;
-  int   rowsUpdated;
-  int   rowsDeleteSucceed;
-  int   rowsDeleteFailed;
-  int   nOperations;
-  TSKEY keyFirst;
-  TSKEY keyLast;
-} SMergeInfo;
-// ------------------ tsdbScan.c
-typedef struct {
-  SFileGroup fGroup;
-  int        numOfIdx;
-  SCompIdx*  pCompIdx;
-  SCompInfo* pCompInfo;
-  void*      pBuf;
-  FILE*      tLogStream;
-} STsdbScanHandle;
-
-// Operations
-// ------------------ tsdbMeta.c
-#define TSDB_INIT_NTABLES 1024
-#define TABLE_TYPE(t) (t)->type
-#define TABLE_NAME(t) (t)->name
-#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
-#define TABLE_UID(t) (t)->tableId.uid
-#define TABLE_TID(t) (t)->tableId.tid
-#define TABLE_SUID(t) (t)->suid
-#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
-#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
-#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
-#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
-#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
-
-STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
-void       tsdbFreeMeta(STsdbMeta* pMeta);
-int        tsdbOpenMeta(STsdbRepo* pRepo);
-int        tsdbCloseMeta(STsdbRepo* pRepo);
-STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
-STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
-int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
-void       tsdbRefTable(STable* pTable);
-void       tsdbUnRefTable(STable* pTable);
-void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
-
-static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
-    return -1;
-  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
-  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
-  STSchema* pSchema = NULL;
-  STSchema* pTSchema = NULL;
-
-  if (lock) TSDB_RLOCK_TABLE(pDTable);
-  if (version < 0) {  // get the latest version of schema
-    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
-  } else {  // get the schema with version
-    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
-                            tsdbCompareSchemaVersion, TD_EQ);
-    if (ptr == NULL) {
-      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-      goto _exit;
-    }
-    pTSchema = *(STSchema**)ptr;
-  }
-
-  ASSERT(pTSchema != NULL);
-
-  if (copy) {
-    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-  } else {
-    pSchema = pTSchema;
-  }
-
-_exit:
-  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
-  return pSchema;
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
-  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
-}
-
-static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
-  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
-    STable *pSuper = pTable->pSuper;
-    if (pSuper == NULL) return NULL;
-    return pSuper->tagSchema;
-  } else if (pTable->type == TSDB_SUPER_TABLE) {
-    return pTable->tagSchema;
-  } else {
-    return NULL;
-  }
-}
-
-static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
-  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
-  return pTable->lastKey;
-}
-
-// ------------------ tsdbBuffer.c
-#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
-
-STsdbBufPool* tsdbNewBufPool();
-void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
-int           tsdbOpenBufPool(STsdbRepo* pRepo);
-void          tsdbCloseBufPool(STsdbRepo* pRepo);
-SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
-
-// ------------------ tsdbMemTable.c
-int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
-void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
-void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
-int   tsdbAsyncCommit(STsdbRepo* pRepo);
-int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
-                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
-void* tsdbCommitData(STsdbRepo* pRepo);
-
-static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
-  if (pIter == NULL) return NULL;
-
-  SSkipListNode* node = tSkipListIterGet(pIter);
-  if (node == NULL) return NULL;
-
-  return (SDataRow)SL_GET_NODE_DATA(node);
-}
-
-static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
-
-  return dataRowKey(row);
-}
-
-static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TKEY_NULL;
-
-  return dataRowTKey(row);
-}
-
-static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
-  ASSERT(pRepo != NULL);
-  if (pRepo->mem == NULL) return NULL;
-
-  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
-  if (pNode == NULL) return NULL;
-
-  STsdbBufBlock* pBufBlock = NULL;
-  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
-
-  return pBufBlock;
-}
-
-// ------------------ tsdbFile.c
-#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
-#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
-#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
-#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
-#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
-#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
-
-STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
-void        tsdbFreeFileH(STsdbFileH* pFileH);
-int         tsdbOpenFileH(STsdbRepo* pRepo);
-void        tsdbCloseFileH(STsdbRepo* pRepo);
-SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid);
-void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
-void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
-SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
-int         tsdbOpenFile(SFile* pFile, int oflag);
-void        tsdbCloseFile(SFile* pFile);
-int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
-SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
-void        tsdbFitRetention(STsdbRepo* pRepo);
-int         tsdbUpdateFileHeader(SFile* pFile);
-int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
-void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
-void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
-int         tsdbLoadFileHeader(SFile* pFile, uint32_t* version);
-void        tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size);
-void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
-
-// ------------------ tsdbRWHelper.c
-#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
-#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
-#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
-#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
-#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
-#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
-#define helperSetState(h, s) (((h)->state) |= (s))
-#define helperClearState(h, s) ((h)->state &= (~(s)))
-#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
-#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
-#define TSDB_MAX_SUBBLOCKS 8
-#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
-#define helperType(h) (h)->type
-#define helperRepo(h) (h)->pRepo
-#define helperState(h) (h)->state
-#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
-#define helperFileId(h) ((h)->files.fGroup.fileId)
-#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
-#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
-#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
-#define helperNewHeadF(h) (&((h)->files.nHeadF))
-#define helperNewLastF(h) (&((h)->files.nLastF))
-
-int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-void tsdbDestroyHelper(SRWHelper* pHelper);
-void tsdbResetHelper(SRWHelper* pHelper);
-int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
-int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup);
-int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
-int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
-int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
-int  tsdbWriteCompInfo(SRWHelper* pHelper);
-int  tsdbWriteCompIdx(SRWHelper* pHelper);
-int  tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer);
-int  tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx);
-int  tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo);
-int  tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
-void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
-int  tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
-                           int numOfColIds);
-int  tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);
-
-static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
-  if (*(TSKEY*)key1 > *(TSKEY*)key2) {
-    return 1;
-  } else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
-    return 0;
-  } else {
-    return -1;
-  }
-}
-
-// ------------------ tsdbMain.c
-#define REPO_ID(r) (r)->config.tsdbId
-#define IS_REPO_LOCKED(r) (r)->repoLocked
-#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
-
-char*       tsdbGetMetaFileName(char* rootDir);
-void        tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname);
-int         tsdbLockRepo(STsdbRepo* pRepo);
-int         tsdbUnlockRepo(STsdbRepo* pRepo);
-char*       tsdbGetDataDirName(char* rootDir);
-int         tsdbGetNextMaxTables(int tid);
-STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
-STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
-int         tsdbCheckCommit(STsdbRepo* pRepo);
-
-// ------------------ tsdbScan.c
-int              tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-STsdbScanHandle* tsdbNewScanHandle();
-void             tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream);
-int              tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-int              tsdbScanSCompIdx(STsdbScanHandle* pScanHandle);
-int              tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
-int              tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
-void             tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
-
-// ------------------ tsdbCommitQueue.c
-int tsdbScheduleCommit(STsdbRepo *pRepo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/inc/tsdbMemTable.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMTABLE_H_
+#define _TD_TSDB_MEMTABLE_H_
+
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
+
+typedef struct {
+  STable *           pTable;
+  SSkipListIterator *pIter;
+} SCommitIter;
+
+typedef struct {
+  uint64_t   uid;
+  TSKEY      keyFirst;
+  TSKEY      keyLast;
+  int64_t    numOfRows;
+  SSkipList* pData;
+} STableData;
+
+typedef struct {
+  T_REF_DECLARE()
+  SRWLatch     latch;
+  TSKEY        keyFirst;
+  TSKEY        keyLast;
+  int64_t      numOfRows;
+  int32_t      maxTables;
+  STableData** tData;
+  SList*       actList;
+  SList*       extraBuffList;
+  SList*       bufBlockList;
+  int64_t      pointsAdd;   // TODO
+  int64_t      storageAdd;  // TODO
+} SMemTable;
+
+enum { TSDB_UPDATE_META, TSDB_DROP_META };
+
+#ifdef WINDOWS
+#pragma pack(push ,1) 
+typedef struct {
+#else
+typedef struct __attribute__((packed)){
+#endif
+  char     act;
+  uint64_t uid;
+} SActObj;
+#ifdef WINDOWS
+#pragma pack(pop) 
+#endif
+
+typedef struct {
+  int  len;
+  char cont[];
+} SActCont;
+
+int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
+void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
+void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
+int   tsdbAsyncCommit(STsdbRepo* pRepo);
+int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
+                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdbRepo* pRepo);
+
+static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
+  if (pIter == NULL) return NULL;
+
+  SSkipListNode* node = tSkipListIterGet(pIter);
+  if (node == NULL) return NULL;
+
+  return (SDataRow)SL_GET_NODE_DATA(node);
+}
+
+static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
+  SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
+
+  return dataRowKey(row);
+}
+
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+  SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TKEY_NULL;
+
+  return dataRowTKey(row);
+}
+
+#endif /* _TD_TSDB_MEMTABLE_H_ */

src/tsdb/inc/tsdbMeta.h

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_META_H_
+#define _TD_TSDB_META_H_
+
+#define TSDB_MAX_TABLE_SCHEMAS 16
+
+typedef struct STable {
+  STableId       tableId;
+  ETableType     type;
+  tstr*          name;  // NOTE: there a flexible string here
+  uint64_t       suid;
+  struct STable* pSuper;  // super table pointer
+  uint8_t        numOfSchemas;
+  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
+  STSchema*      tagSchema;
+  SKVRow         tagVal;
+  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
+  void*          eventHandler;   // TODO
+  void*          streamHandler;  // TODO
+  TSKEY          lastKey;
+  SDataRow       lastRow;
+  char*          sql;
+  void*          cqhandle;
+  SRWLatch       latch;  // TODO: implementa latch functions
+  T_REF_DECLARE()
+} STable;
+
+typedef struct {
+  pthread_rwlock_t rwLock;
+
+  int32_t   nTables;
+  int32_t   maxTables;
+  STable**  tables;
+  SList*    superList;
+  SHashObj* uidMap;
+  int       maxRowBytes;
+  int       maxCols;
+} STsdbMeta;
+
+#define TSDB_INIT_NTABLES 1024
+#define TABLE_TYPE(t) (t)->type
+#define TABLE_NAME(t) (t)->name
+#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
+#define TABLE_UID(t) (t)->tableId.uid
+#define TABLE_TID(t) (t)->tableId.tid
+#define TABLE_SUID(t) (t)->suid
+// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
+#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
+#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
+#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
+#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
+
+STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
+void       tsdbFreeMeta(STsdbMeta* pMeta);
+int        tsdbOpenMeta(STsdbRepo* pRepo);
+int        tsdbCloseMeta(STsdbRepo* pRepo);
+STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
+STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
+int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
+void       tsdbRefTable(STable* pTable);
+void       tsdbUnRefTable(STable* pTable);
+void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
+int        tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen);
+void       tsdbOrgMeta(STsdbRepo* pRepo);
+
+static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
+    return -1;
+  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
+  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
+  STSchema* pSchema = NULL;
+  STSchema* pTSchema = NULL;
+
+  if (lock) TSDB_RLOCK_TABLE(pDTable);
+  if (version < 0) {  // get the latest version of schema
+    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
+  } else {  // get the schema with version
+    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
+                            tsdbCompareSchemaVersion, TD_EQ);
+    if (ptr == NULL) {
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      goto _exit;
+    }
+    pTSchema = *(STSchema**)ptr;
+  }
+
+  ASSERT(pTSchema != NULL);
+
+  if (copy) {
+    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  } else {
+    pSchema = pTSchema;
+  }
+
+_exit:
+  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
+  return pSchema;
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
+  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
+}
+
+static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
+  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
+    STable *pSuper = pTable->pSuper;
+    if (pSuper == NULL) return NULL;
+    return pSuper->tagSchema;
+  } else if (pTable->type == TSDB_SUPER_TABLE) {
+    return pTable->tagSchema;
+  } else {
+    return NULL;
+  }
+}
+
+static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
+  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
+  return pTable->lastKey;
+}
+
+#endif /* _TD_TSDB_META_H_ */

src/tsdb/inc/tsdbReadImpl.h

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_READ_IMPL_H_
+#define _TD_TSDB_READ_IMPL_H_
+
+typedef struct SReadH SReadH;
+
+typedef struct {
+  int32_t  tid;
+  uint32_t len;
+  uint32_t offset;
+  uint32_t hasLast : 2;
+  uint32_t numOfBlocks : 30;
+  uint64_t uid;
+  TSKEY    maxKey;
+} SBlockIdx;
+
+typedef struct {
+  int64_t last : 1;
+  int64_t offset : 63;
+  int32_t algorithm : 8;
+  int32_t numOfRows : 24;
+  int32_t len;
+  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  int16_t numOfSubBlocks;
+  int16_t numOfCols;  // not including timestamp column
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+} SBlock;
+
+typedef struct {
+  int32_t  delimiter;  // For recovery usage
+  int32_t  tid;
+  uint64_t uid;
+  SBlock   blocks[];
+} SBlockInfo;
+
+typedef struct {
+  int16_t colId;
+  int32_t len;
+  int32_t type : 8;
+  int32_t offset : 24;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  char    padding[2];
+} SBlockCol;
+
+typedef struct {
+  int32_t   delimiter;  // For recovery usage
+  int32_t   numOfCols;  // For recovery usage
+  uint64_t  uid;        // For recovery usage
+  SBlockCol cols[];
+} SBlockData;
+
+struct SReadH {
+  STsdbRepo * pRepo;
+  SDFileSet   rSet;     // FSET to read
+  SArray *    aBlkIdx;  // SBlockIdx array
+  STable *    pTable;   // table to read
+  SBlockIdx * pBlkIdx;  // current reading table SBlockIdx
+  int         cidx;
+  SBlockInfo *pBlkInfo;
+  SBlockData *pBlkData;  // Block info
+  SDataCols * pDCols[2];
+  void *      pBuf;   // buffer
+  void *      pCBuf;  // compression buffer
+};
+
+#define TSDB_READ_REPO(rh) ((rh)->pRepo)
+#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
+#define TSDB_READ_FSET(rh) (&((rh)->rSet))
+#define TSDB_READ_TABLE(rh) ((rh)->pTable)
+#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
+#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
+#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
+#define TSDB_READ_BUF(rh) ((rh)->pBuf)
+#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
+
+#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))
+
+int   tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo);
+void  tsdbDestroyReadH(SReadH *pReadh);
+int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
+void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
+int   tsdbLoadBlockIdx(SReadH *pReadh);
+int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
+int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
+int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
+void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols);
+
+static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
+  void * pBuf = *ppBuf;
+  size_t tsize = taosTSizeof(pBuf);
+
+  if (tsize < size) {
+    if (tsize == 0) tsize = 1024;
+
+    while (tsize < size) {
+      tsize *= 2;
+    }
+
+    *ppBuf = taosTRealloc(pBuf, tsize);
+    if (*ppBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+#endif /*_TD_TSDB_READ_IMPL_H_*/

src/tsdb/inc/tsdbint.h

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_INT_H_
+#define _TD_TSDB_INT_H_
+
+// // TODO: remove the include
+// #include <errno.h>
+// #include <fcntl.h>
+// #include <limits.h>
+// #include <inttypes.h>
+// #include <sys/stat.h>
+// #include <sys/types.h>
+// #include <semaphore.h>
+// #include <dirent.h>
+
+#include "os.h"
+#include "tlog.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "tskiplist.h"
+#include "tdataformat.h"
+#include "tcoding.h"
+#include "tscompression.h"
+#include "tlockfree.h"
+#include "tlist.h"
+#include "hash.h"
+#include "tarray.h"
+#include "tfs.h"
+#include "tsocket.h"
+
+#include "tsdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Log
+#include "tsdbLog.h"
+// Meta
+#include "tsdbMeta.h"
+// Buffer
+#include "tsdbBuffer.h"
+// MemTable
+#include "tsdbMemTable.h"
+// File
+#include "tsdbFile.h"
+// FS
+#include "tsdbFS.h"
+// ReadImpl
+#include "tsdbReadImpl.h"
+// Commit
+#include "tsdbCommit.h"
+// Commit Queue
+#include "tsdbCommitQueue.h"
+// Main definitions
+struct STsdbRepo {
+  uint8_t state;
+
+  STsdbCfg        config;
+  STsdbAppH       appH;
+  STsdbStat       stat;
+  STsdbMeta*      tsdbMeta;
+  STsdbBufPool*   pPool;
+  SMemTable*      mem;
+  SMemTable*      imem;
+  STsdbFS*        fs;
+  tsem_t          readyToCommit;
+  pthread_mutex_t mutex;
+  bool            repoLocked;
+  int32_t         code;  // Commit code
+};
+
+#define REPO_ID(r) (r)->config.tsdbId
+#define REPO_CFG(r) (&((r)->config))
+#define REPO_FS(r) ((r)->fs)
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+int        tsdbLockRepo(STsdbRepo* pRepo);
+int        tsdbUnlockRepo(STsdbRepo* pRepo);
+STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo);
+int        tsdbCheckCommit(STsdbRepo* pRepo);
+int        tsdbRestoreInfo(STsdbRepo* pRepo);
+void       tsdbGetRootDir(int repoid, char dirName[]);
+void       tsdbGetDataDir(int repoid, char dirName[]);
+
+static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
+  ASSERT(pRepo != NULL);
+  if (pRepo->mem == NULL) return NULL;
+
+  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
+  if (pNode == NULL) return NULL;
+
+  STsdbBufBlock* pBufBlock = NULL;
+  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
+
+  return pBufBlock;
+}
+
+static FORCE_INLINE int tsdbGetNextMaxTables(int tid) {
+  ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES);
+  int maxTables = TSDB_INIT_NTABLES;
+  while (true) {
+    maxTables = MIN(maxTables, TSDB_MAX_TABLES);
+    if (tid <= maxTables) break;
+    maxTables *= 2;
+  }
+
+  return maxTables + 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TD_TSDB_INT_H_ */

src/tsdb/src/tsdbBuffer.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdb.h"
+#include "tsdbint.h"
-#include "tsdbMain.h"
 
 #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
 

src/tsdb/src/tsdbCommitQueue.c

@@ -13,11 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
+#include "tsdbint.h"
-#include "tglobal.h"
-#include "tlist.h"
-#include "tref.h"
-#include "tsdbMain.h"
 
 typedef struct {
   bool            stop;
@@ -35,7 +31,7 @@ typedef struct {
 
 static void *tsdbLoopCommit(void *arg);
 
-SCommitQueue tsCommitQueue = {0};
+static SCommitQueue tsCommitQueue = {0};
 
 int tsdbInitCommitQueue() {
   int nthreads = tsNumOfCommitThreads;

src/tsdb/src/tsdbMemTable.c

@@ -13,12 +13,23 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdb.h"
+#include "tsdbint.h"
-#include "tsdbMain.h"
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
 #define TSDB_MAX_INSERT_BATCH 512
 
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SDataRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  void *  pMsg;
+} SSubmitMsgIter;
+
 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
 static void        tsdbFreeMemTable(SMemTable *pMemTable);
 static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
@@ -41,8 +52,8 @@ static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable,
 static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now);
 
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
-  STsdbRepo *    pRepo = (STsdbRepo *)repo;
+  STsdbRepo *    pRepo = repo;
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk *   pBlock = NULL;
   int32_t        affectedrows = 0;
@@ -225,8 +236,8 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
   return 0;
 }
 
-int tsdbSyncCommit(TSDB_REPO_T *repo) {
+int tsdbSyncCommit(STsdbRepo *repo) {
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  STsdbRepo *pRepo = repo;
 
   tsdbAsyncCommit(pRepo);
   tsem_wait(&(pRepo->readyToCommit));
@@ -254,14 +265,17 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) {
  */
 int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
                           TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
-  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
+  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
   if (pIter == NULL) return 0;
-  STSchema *pSchema = NULL;
+  STSchema * pSchema = NULL;
-  TSKEY     rowKey = 0;
+  TSKEY      rowKey = 0;
-  TSKEY     fKey = 0;
+  TSKEY      fKey = 0;
-  bool      isRowDel = false;
+  bool       isRowDel = false;
-  int       filterIter = 0;
+  int        filterIter = 0;
-  SDataRow  row = NULL;
+  SDataRow   row = NULL;
+  SMergeInfo mInfo;
+
+  if (pMergeInfo == NULL) pMergeInfo = &mInfo;
 
   memset(pMergeInfo, 0, sizeof(*pMergeInfo));
   pMergeInfo->keyFirst = INT64_MAX;
@@ -452,11 +466,6 @@ static void tsdbFreeTableData(STableData *pTableData) {
 
 static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
 
-void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fileId * daysPerFile * tsMsPerDay[precision];
-  *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
-}
-
 static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
   ASSERT(pMemTable->maxTables < maxTables);
 

src/tsdb/src/tsdbMeta.c

@@ -12,20 +12,12 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
+#include "tsdbint.h"
-#include "hash.h"
-#include "taosdef.h"
-#include "tchecksum.h"
-#include "tsdb.h"
-#include "tsdbMain.h"
-#include "tskiplist.h"
 
 #define TSDB_SUPER_TABLE_SL_LEVEL 5
 #define DEFAULT_TAG_INDEX_COLUMN 0
 
 static int     tsdbCompareSchemaVersion(const void *key1, const void *key2);
-static int     tsdbRestoreTable(void *pHandle, void *cont, int contLen);
-static void    tsdbOrgMeta(void *pHandle);
 static char *  getTagIndexKey(const void *pData);
 static STable *tsdbNewTable();
 static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper);
@@ -53,7 +45,7 @@ static int     tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable);
 static int     tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid);
 
 // ------------------ OUTER FUNCTIONS ------------------
-int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   STable *   super = NULL;
@@ -148,7 +140,7 @@ _err:
   return -1;
 }
 
-int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
+int tsdbDropTable(STsdbRepo *repo, STableId tableId) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   uint64_t   uid = tableId.uid;
@@ -301,7 +293,7 @@ static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) {
   return (colId < p2->colId)? -1:1;
 }
 
-int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) {
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   STSchema * pNewSchema = NULL;
@@ -469,6 +461,8 @@ void tsdbFreeMeta(STsdbMeta *pMeta) {
 }
 
 int tsdbOpenMeta(STsdbRepo *pRepo) {
+  return 0;
+#if 0
   char *     fname = NULL;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   ASSERT(pMeta != NULL);
@@ -479,11 +473,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
     goto _err;
   }
 
-  pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo);
+  // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo);
-  if (pMeta->pStore == NULL) {
+  // if (pMeta->pStore == NULL) {
-    tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno));
+  //   tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  //   goto _err;
-  }
+  // }
 
   tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
   tfree(fname);
@@ -492,6 +486,7 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
 _err:
   tfree(fname);
   return -1;
+#endif
 }
 
 int tsdbCloseMeta(STsdbRepo *pRepo) {
@@ -500,7 +495,7 @@ int tsdbCloseMeta(STsdbRepo *pRepo) {
   STable *   pTable = NULL;
 
   if (pMeta == NULL) return 0;
-  tdCloseKVStore(pMeta->pStore);
+  // tdCloseKVStore(pMeta->pStore);
   for (int i = 1; i < pMeta->maxTables; i++) {
     tsdbFreeTable(pMeta->tables[i]);
   }
@@ -609,10 +604,8 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema,
   }
 }
 
-// ------------------ LOCAL FUNCTIONS ------------------
+int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) {
-static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
+  STable *pTable = NULL;
-  STsdbRepo *pRepo = (STsdbRepo *)pHandle;
-  STable *   pTable = NULL;
 
   if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
@@ -631,8 +624,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
   return 0;
 }
 
-static void tsdbOrgMeta(void *pHandle) {
+void tsdbOrgMeta(STsdbRepo *pRepo) {
-  STsdbRepo *pRepo = (STsdbRepo *)pHandle;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
 
   for (int i = 1; i < pMeta->maxTables; i++) {
@@ -643,6 +635,7 @@ static void tsdbOrgMeta(void *pHandle) {
   }
 }
 
+// ------------------ LOCAL FUNCTIONS ------------------
 static char *getTagIndexKey(const void *pData) {
   STable *pTable = (STable *)pData;
 

src/tsdb/src/tsdbRead.c

@@ -20,8 +20,7 @@
 #include "exception.h"
 
 #include "tlosertree.h"
-#include "tsdb.h"
+#include "tsdbint.h"
-#include "tsdbMain.h"
 #include "texpr.h"
 
 #define EXTRA_BYTES 2
@@ -54,7 +53,7 @@ typedef struct SQueryFilePos {
 } SQueryFilePos;
 
 typedef struct SDataBlockLoadInfo {
-  SFileGroup* fileGroup;
+  SDFileSet*  fileGroup;
   int32_t     slot;
   int32_t     tid;
   SArray*     pLoadedCols;
@@ -69,7 +68,7 @@ typedef struct STableCheckInfo {
   STableId      tableId;
   TSKEY         lastKey;
   STable*       pTableObj;
-  SCompInfo*    pCompInfo;
+  SBlockInfo*    pCompInfo;
   int32_t       compSize;
   int32_t       numOfBlocks:29; // number of qualified data blocks not the original blocks
   int8_t        chosen:2;       // indicate which iterator should move forward
@@ -79,7 +78,7 @@ typedef struct STableCheckInfo {
 } STableCheckInfo;
 
 typedef struct STableBlockInfo {
-  SCompBlock*        compBlock;
+  SBlock*        compBlock;
   STableCheckInfo*   pTableCheckInfo;
 } STableBlockInfo;
 
@@ -114,9 +113,9 @@ typedef struct STsdbQueryHandle {
   bool           loadExternalRow;  // load time window external data rows
   void*          qinfo;            // query info handle, for debug purpose
   int32_t        type;             // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
-  SFileGroup*    pFileGroup;
+  SDFileSet*     pFileGroup;
-  SFileGroupIter fileIter;
+  SFSIter        fileIter;
-  SRWHelper      rhelper;
+  SReadH         rhelper;
   STableBlockInfo* pDataBlockInfo;
 
   SDataCols     *pDataCols;        // in order to hold current file data block
@@ -142,7 +141,7 @@ static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroup
 static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey);
 
 static void    changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle);
-static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock);
+static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock);
 static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
 static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle);
 static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
@@ -308,7 +307,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
   return pNew;
 }
 
-static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
+static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
   STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
   if (pQueryHandle == NULL) {
     goto out_of_memory;
@@ -329,7 +328,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   pQueryHandle->pMemRef     = pMemRef;
   pQueryHandle->loadExternalRow = pCond->loadExternalRows;
 
-  if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
+  if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) {
     goto out_of_memory;
   }
 
@@ -388,7 +387,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   return NULL;
 }
 
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
+TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
   STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
 
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
@@ -406,7 +405,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   return (TsdbQueryHandleT) pQueryHandle;
 }
 
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
   pCond->twindow = updateLastrowForEachGroup(groupList);
 
   // no qualified table
@@ -442,7 +441,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   return res;
 }
 
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
   pQueryHandle->loadExternalRow = true;
   if (pQueryHandle != NULL) {
@@ -713,7 +712,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   return (int32_t)fid;
 }
 
-static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
+static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
   int32_t firstSlot = 0;
   int32_t lastSlot = numOfBlocks - 1;
 
@@ -751,16 +750,16 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     pCheckInfo->numOfBlocks = 0;
 
-    if (tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb) != TSDB_CODE_SUCCESS) {
+    if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) {
       code = terrno;
       break;
     }
 
-    SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx;
+    SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx;
 
     // no data block in this file, try next file
-    if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) {
+    if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) {
-      continue; // no data blocks in the file belongs to pCheckInfo->pTable
+      continue;  // no data blocks in the file belongs to pCheckInfo->pTable
     }
 
     if (pCheckInfo->compSize < (int32_t)compIndex->len) {
@@ -773,12 +772,12 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
         break;
       }
 
-      pCheckInfo->pCompInfo = (SCompInfo*) t;
+      pCheckInfo->pCompInfo = (SBlockInfo*) t;
       pCheckInfo->compSize = compIndex->len;
     }
 
-    tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
+    tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
-    SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
+    SBlockInfo* pCompInfo = pCheckInfo->pCompInfo;
 
     TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL;
 
@@ -807,7 +806,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
     pCheckInfo->numOfBlocks = (end - start);
 
     if (start > 0) {
-      memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SCompBlock));
+      memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock));
     }
 
     (*numOfBlocks) += pCheckInfo->numOfBlocks;
@@ -816,7 +815,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
   return code;
 }
 
-static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
+static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
   int64_t st = taosGetTimestampUs();
 
   STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj);
@@ -827,14 +826,14 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
     goto _error;
   }
 
-  code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema);
+  code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %p", pQueryHandle, pQueryHandle->qinfo);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     goto _error;
   }
 
-  code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema);
+  code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %p", pQueryHandle, pQueryHandle->qinfo);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -856,7 +855,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
   pBlockLoadInfo->slot = pQueryHandle->cur.slot;
   pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
 
   pBlock->numOfRows = pCols->numOfRows;
@@ -882,7 +881,7 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i
 static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle);
 static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos);
 
-static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
+static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){
   SQueryFilePos* cur = &pQueryHandle->cur;
   STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
   SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
@@ -965,7 +964,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
   return code;
 }
 
-static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) {
+static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -977,7 +976,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
 
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
@@ -1000,7 +999,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
         cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
@@ -1085,7 +1084,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
   char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
 
   int32_t num = end - start + 1;
@@ -1309,7 +1308,7 @@ static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) {
 static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) {
   SQueryFilePos* cur = &pQueryHandle->cur;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
 
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
@@ -1352,7 +1351,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
   int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
 
   SQueryFilePos* cur = &pQueryHandle->cur;
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
 
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) {
     endPos = pBlockInfo->rows - 1;
@@ -1371,14 +1370,14 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
 
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
-static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
+static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
   STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
 
   initTableMemIterator(pQueryHandle, pCheckInfo);
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
       cur->pos >= 0 && cur->pos < pBlock->numOfRows);
 
@@ -1670,7 +1669,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
       continue;
     }
 
-    SCompBlock* pBlock = pTableCheck->pCompInfo->blocks;
+    SBlock* pBlock = pTableCheck->pCompInfo->blocks;
     sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks;
 
     char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks);
@@ -1786,19 +1785,19 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
   STimeWindow win = TSWINDOW_INITIALIZER;
 
   while (true) {
-    pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb));
 
-    if ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) == NULL) {
+    if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       break;
     }
 
-    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey);
+    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey);
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
         (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
                 pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
       pQueryHandle->pFileGroup = NULL;
@@ -1806,15 +1805,15 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
       break;
     }
 
-    if (tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) {
+    if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       code = terrno;
       break;
     }
 
-    pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
 
-    if (tsdbLoadCompIdx(&pQueryHandle->rhelper, NULL) < 0) {
+    if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) {
       code = terrno;
       break;
     }
@@ -1824,7 +1823,7 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
     }
 
     tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
-              pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
+              pQueryHandle->pFileGroup->fid, pQueryHandle->qinfo);
 
     assert(numOfBlocks >= 0);
     if (numOfBlocks == 0) {
@@ -1855,7 +1854,7 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
 
   assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0);
   cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1;
-  cur->fid = pQueryHandle->pFileGroup->fileId;
+  cur->fid = pQueryHandle->pFileGroup->fid;
 
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
   return getDataBlockRv(pQueryHandle, pBlockInfo, exists);
@@ -1878,7 +1877,7 @@ static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
 }
 
 static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
-  STsdbFileH*    pFileHandle = tsdbGetFile(pQueryHandle->pTsdb);
+  STsdbFS*       pFileHandle = REPO_FS(pQueryHandle->pTsdb);
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   // find the start data block in file
@@ -1887,10 +1886,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
     int32_t   fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
 
-    pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbRLockFS(pFileHandle);
-    tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
+    tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order);
-    tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
+    tsdbFSIterSeek(&pQueryHandle->fileIter, fid);
-    pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbUnLockFS(pFileHandle);
 
     return getFirstFileDataBlock(pQueryHandle, exists);
   } else {
@@ -2482,7 +2481,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   }
 
   int64_t stime = taosGetTimestampUs();
-  tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
+  tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock);
 
   int16_t* colIds = pHandle->defaultLoadColumn->pData;
 
@@ -2492,7 +2491,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
     pHandle->statis[i].colId = colIds[i];
   }
 
-  tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols);
+  tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols);
 
   // always load the first primary timestamp column data
   SDataStatis* pPrimaryColStatis = &pHandle->statis[0];
@@ -2544,11 +2543,11 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
 
-      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
+      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
-        SCompBlock* pBlock = pBlockInfo->compBlock;
+        SBlock* pBlock = pBlockInfo->compBlock;
         if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) {
           return NULL;
         }
@@ -2820,7 +2819,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
                                  int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
                                  SColIndex* pColIndex, int32_t numOfCols) {
   if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
@@ -2915,7 +2914,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co
   return terrno;
 }
 
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
   if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
 
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@@ -2945,7 +2944,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, ST
   return terrno;
 }
 
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
   if (tsdbRLockRepoMeta(tsdb) < 0) {
     return terrno;
   }
@@ -3031,7 +3030,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   // todo check error
   tsdbMayUnTakeMemSnapshot(pQueryHandle);
 
-  tsdbDestroyHelper(&pQueryHandle->rhelper);
+  tsdbDestroyReadH(&pQueryHandle->rhelper);
 
   tdFreeDataCols(pQueryHandle->pDataCols);
   pQueryHandle->pDataCols = NULL;

src/tsdb/src/tsdbReadImpl.c

@@ -0,0 +1,660 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+
+#define TSDB_KEY_COL_OFFSET 0
+
+static void tsdbResetReadTable(SReadH *pReadh);
+static void tsdbResetReadFile(SReadH *pReadh);
+static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
+static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                         int maxPoints, char *buffer, int bufferSize);
+static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                      int numOfColIds);
+static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
+
+int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) {
+  ASSERT(pReadh != NULL && pRepo != NULL);
+
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+  memset((void *)pReadh, 0, sizeof(*pReadh));
+  pReadh->pRepo = pRepo;
+
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+
+  pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+  if (pReadh->aBlkIdx == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[0] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[1] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbDestroyReadH(SReadH *pReadh) {
+  if (pReadh == NULL) return;
+
+  pReadh->pCBuf = taosTZfree(pReadh->pCBuf);
+  pReadh->pBuf = taosTZfree(pReadh->pBuf);
+  pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]);
+  pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]);
+  pReadh->pBlkData = taosTZfree(pReadh->pBlkData);
+  pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+  pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+  pReadh->pRepo = NULL;
+}
+
+int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
+  ASSERT(pSet != NULL);
+  tsdbResetReadFile(pReadh);
+
+  pReadh->rSet = *pSet;
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+  if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) {
+    tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet),
+              tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
+
+int tsdbLoadBlockIdx(SReadH *pReadh) {
+  SDFile *  pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx blkIdx;
+
+  ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
+
+  // No data at all, just return
+  if (pHeadf->info.offset <= 0) return 0;
+
+  if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (nread < pHeadf->info.len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len);
+    return -1;
+  }
+
+  void *ptr = TSDB_READ_BUF(pReadh);
+  int   tsize = 0;
+  while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) {
+    ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx);
+    ASSERT(ptr != NULL);
+
+    if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) < 0) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    tsize++;
+    ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
+                             ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
+  }
+
+  return 0;
+}
+
+int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
+
+  pReadh->pTable = pTable;
+
+  if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  size_t size = taosArrayGetSize(pReadh->aBlkIdx);
+  if (size > 0) {
+    while (true) {
+      if (pReadh->cidx >= size) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      }
+
+      SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
+      if (pBlkIdx->tid == TABLE_TID(pTable)) {
+        if (pBlkIdx->uid == TABLE_UID(pTable)) {
+          pReadh->pBlkIdx = pBlkIdx;
+        } else {
+          pReadh->pBlkIdx = NULL;
+        }
+        pReadh->cidx++;
+        break;
+      } else if (pBlkIdx->tid > TABLE_TID(pTable)) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      } else {
+        pReadh->cidx++;
+      }
+    }
+  } else {
+    pReadh->pBlkIdx = NULL;
+  }
+
+  return 0;
+}
+
+int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
+  ASSERT(pReadh->pBlkIdx != NULL);
+
+  SDFile *   pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
+
+  if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (nread < pBlkIdx->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
+
+  if (pTarget) {
+    memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  }
+
+  return 0;
+}
+
+int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
+  ASSERT(pBlock->numOfSubBlocks <= 1);
+
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols);
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+
+  if (nread < size) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pIdx->tid);
+  tlen += taosEncodeVariantU32(buf, pIdx->len);
+  tlen += taosEncodeVariantU32(buf, pIdx->offset);
+  tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
+  tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
+  tlen += taosEncodeFixedU64(buf, pIdx->uid);
+  tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
+
+  return tlen;
+}
+
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
+  uint8_t  hasLast = 0;
+  uint32_t numOfBlocks = 0;
+  uint64_t value = 0;
+
+  if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
+  if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
+  pIdx->hasLast = hasLast;
+  if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
+  pIdx->numOfBlocks = numOfBlocks;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->uid = (int64_t)value;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->maxKey = (TSKEY)value;
+
+  return buf;
+}
+
+void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) {
+  SBlockData *pBlockData = pReadh->pBlkData;
+
+  for (int i = 0, j = 0; i < numOfCols;) {
+    if (j >= pBlockData->numOfCols) {
+      pStatis[i].numOfNull = -1;
+      i++;
+      continue;
+    }
+
+    if (pStatis[i].colId == pBlockData->cols[j].colId) {
+      pStatis[i].sum = pBlockData->cols[j].sum;
+      pStatis[i].max = pBlockData->cols[j].max;
+      pStatis[i].min = pBlockData->cols[j].min;
+      pStatis[i].maxIndex = pBlockData->cols[j].maxIndex;
+      pStatis[i].minIndex = pBlockData->cols[j].minIndex;
+      pStatis[i].numOfNull = pBlockData->cols[j].numOfNull;
+      i++;
+      j++;
+    } else if (pStatis[i].colId < pBlockData->cols[j].colId) {
+      pStatis[i].numOfNull = -1;
+      i++;
+    } else {
+      j++;
+    }
+  }
+}
+
+static void tsdbResetReadTable(SReadH *pReadh) {
+  tdResetDataCols(pReadh->pDCols[0]);
+  tdResetDataCols(pReadh->pDCols[1]);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+}
+
+static void tsdbResetReadFile(SReadH *pReadh) {
+  tsdbResetReadTable(pReadh);
+  taosArrayClear(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+}
+
+static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+
+  tdResetDataCols(pDataCols);
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1;
+
+  SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh);
+
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset,
+              pBlock->len);
+    return -1;
+  }
+
+  if (nread < pBlock->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64
+              " expected bytes:%d read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread);
+    return -1;
+  }
+
+  int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols);
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
+    return -1;
+  }
+
+  ASSERT(tsize < pBlock->len);
+  ASSERT(pBlockData->numOfCols == pBlock->numOfCols);
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  // Recover the data
+  int ccol = 0;  // loop iter for SBlockCol object
+  int dcol = 0;  // loop iter for SDataCols object
+  while (dcol < pDataCols->numOfCols) {
+    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+    if (dcol != 0 && ccol >= pBlockData->numOfCols) {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+      dcol++;
+      continue;
+    }
+
+    int16_t tcolId = 0;
+    int32_t toffset = TSDB_KEY_COL_OFFSET;
+    int32_t tlen = pBlock->keyLen;
+
+    if (dcol != 0) {
+      SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
+      tcolId = pBlockCol->colId;
+      toffset = pBlockCol->offset;
+      tlen = pBlockCol->len;
+    } else {
+      ASSERT(pDataCol->colId == tcolId);
+    }
+
+    if (tcolId == pDataCol->colId) {
+      if (pBlock->algorithm == TWO_STAGE_COMP) {
+        int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+        if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
+      }
+
+      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
+                                       pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
+                  TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
+        return -1;
+      }
+
+      if (dcol != 0) {
+        ccol++;
+      }
+      dcol++;
+    } else if (tcolId < pDataCol->colId) {
+      ccol++;
+    } else {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+      dcol++;
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                        int maxPoints, char *buffer, int bufferSize) {
+  if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  // Decode the data
+  if (comp) {
+    // Need to decompress
+    int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData,
+                                                             pDataCol->spaceSize, comp, buffer, bufferSize);
+    if (tlen <= 0) {
+      tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
+                len, comp, numOfRows, maxPoints, bufferSize);
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      return -1;
+    }
+    pDataCol->len = tlen;
+  } else {
+    // No need to decompress, just memcpy it
+    pDataCol->len = len - sizeof(TSCKSUM);
+    memcpy(pDataCol->pData, content, pDataCol->len);
+  }
+
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    dataColSetOffset(pDataCol, numOfRows);
+  }
+  return 0;
+}
+
+static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                     int numOfColIds) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+  ASSERT(colIds[0] == 0);
+
+  SDFile *  pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+  SBlockCol blockCol = {0};
+
+  tdResetDataCols(pDataCols);
+
+  // If only load timestamp column, no need to load SBlockData part
+  if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1;
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  int dcol = 0;
+  int ccol = 0;
+  for (int i = 0; i < numOfColIds; i++) {
+    int16_t    colId = colIds[i];
+    SDataCol * pDataCol = NULL;
+    SBlockCol *pBlockCol = NULL;
+
+    while (true) {
+      if (dcol >= pDataCols->numOfCols) {
+        pDataCol = NULL;
+        break;
+      }
+      pDataCol = &pDataCols->cols[dcol];
+      if (pDataCol->colId > colId) {
+        pDataCol = NULL;
+        break;
+      } else {
+        dcol++;
+        if (pDataCol->colId == colId) break;
+      }
+    }
+
+    if (pDataCol == NULL) continue;
+    ASSERT(pDataCol->colId == colId);
+
+    if (colId == 0) {  // load the key row
+      blockCol.colId = colId;
+      blockCol.len = pBlock->keyLen;
+      blockCol.type = pDataCol->type;
+      blockCol.offset = TSDB_KEY_COL_OFFSET;
+      pBlockCol = &blockCol;
+    } else {  // load non-key rows
+      while (true) {
+        if (ccol >= pBlock->numOfCols) {
+          pBlockCol = NULL;
+          break;
+        }
+
+        pBlockCol = &(pReadh->pBlkData->cols[ccol]);
+        if (pBlockCol->colId > colId) {
+          pBlockCol = NULL;
+          break;
+        } else {
+          ccol++;
+          if (pBlockCol->colId == colId) break;
+        }
+      }
+
+      if (pBlockCol == NULL) {
+        dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+        continue;
+      }
+
+      ASSERT(pBlockCol->colId == pDataCol->colId);
+    }
+
+    if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
+  }
+
+  return 0;
+}
+
+static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
+  ASSERT(pDataCol->colId == pBlockCol->colId);
+
+  STsdbRepo *pRepo = TSDB_READ_REPO(pReadh);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
+  if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
+
+  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + pBlockCol->offset;
+  if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len);
+    return -1;
+  }
+
+  if (nread < pBlockCol->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread);
+    return -1;
+  }
+
+  if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
+                                   pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
+    tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+              pBlockCol->colId, offset);
+    return -1;
+  }
+
+  return 0;
+}

src/tsdb/src/tsdbScan.c

@@ -13,8 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
+#if 0
 #ifndef _TSDB_PLUGINS
 
 int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }
@@ -25,12 +26,13 @@ void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {}
 
 int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }
 
-int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; }
+int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; }
 
-int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; }
+int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; }
 
 int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; }
 
 void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {}
 
+#endif
 #endif

src/tsdb/src/tsdbSync.c

@@ -0,0 +1,694 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tsdbint.h"
+
+// Sync handle
+typedef struct {
+  STsdbRepo *pRepo;
+  SRtn       rtn;
+  SOCKET     socketFd;
+  void *     pBuf;
+  bool       mfChanged;
+  SMFile *   pmf;
+  SMFile     mf;
+  SDFileSet  df;
+  SDFileSet *pdf;
+} SSyncH;
+
+#define SYNC_BUFFER(sh) ((sh)->pBuf)
+
+static void    tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd);
+static void    tsdbDestroySyncH(SSyncH *pSyncH);
+static int32_t tsdbSyncSendMeta(SSyncH *pSynch);
+static int32_t tsdbSyncRecvMeta(SSyncH *pSynch);
+static int32_t tsdbSendMetaInfo(SSyncH *pSynch);
+static int32_t tsdbRecvMetaInfo(SSyncH *pSynch);
+static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend);
+static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend);
+static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch);
+static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch);
+static bool    tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2);
+static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet);
+static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet);
+static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch);
+static int     tsdbReload(STsdbRepo *pRepo, bool isMfChanged);
+
+int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) {
+  STsdbRepo *pRepo = (STsdbRepo *)tsdb;
+  SSyncH     synch = {0};
+
+  tsdbInitSyncH(&synch, pRepo, socketFd);
+  // Disable TSDB commit
+  tsem_wait(&(pRepo->readyToCommit));
+
+  if (tsdbSyncSendMeta(&synch) < 0) {
+    tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbSyncSendDFileSetArray(&synch) < 0) {
+    tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  // Enable TSDB commit
+  tsem_post(&(pRepo->readyToCommit));
+  tsdbDestroySyncH(&synch);
+  return 0;
+
+_err:
+  tsem_post(&(pRepo->readyToCommit));
+  tsdbDestroySyncH(&synch);
+  return -1;
+}
+
+int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
+  STsdbRepo *pRepo = (STsdbRepo *)tsdb;
+  SSyncH synch = {0};
+
+  pRepo->state = TSDB_STATE_OK;
+
+  tsdbInitSyncH(&synch, pRepo, socketFd);
+  tsdbStartFSTxn(pRepo, 0, 0);
+
+  if (tsdbSyncRecvMeta(&synch) < 0) {
+    tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbSyncRecvDFileSetArray(&synch) < 0) {
+    tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  tsdbEndFSTxn(pRepo);
+  tsdbDestroySyncH(&synch);
+
+  // Reload file change
+  tsdbReload(pRepo, synch.mfChanged);
+
+  return 0;
+
+_err:
+  tsdbEndFSTxnWithError(REPO_FS(pRepo));
+  tsdbDestroySyncH(&synch);
+  return -1;
+}
+
+static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) {
+  pSyncH->pRepo = pRepo;
+  pSyncH->socketFd = socketFd;
+  tsdbGetRtnSnap(pRepo, &(pSyncH->rtn));
+}
+
+static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); }
+
+static int32_t tsdbSyncSendMeta(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  bool       toSendMeta = false;
+  SMFile     mf;
+
+  // Send meta info to remote
+  tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo));
+  if (tsdbSendMetaInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (pRepo->fs->cstatus->pmf == NULL) {
+    // No meta file, not need to wait to retrieve meta file
+    tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo));
+    return 0;
+  }
+
+  if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) {
+    tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (toSendMeta) {
+    tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf);
+    if (tsdbOpenMFile(&mf, O_RDONLY) < 0) {
+      tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    int32_t writeLen = (int32_t)mf.info.size;
+    tsdbInfo("vgId:%d, metafile:%s will be sent, size:%d", REPO_ID(pRepo), mf.f.aname, writeLen);
+
+    int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen);
+    if (ret != writeLen) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbError("vgId:%d, failed to send metafile since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+                writeLen);
+      tsdbCloseMFile(&mf);
+      return -1;
+    }
+
+    tsdbCloseMFile(&mf);
+    tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo));
+  } else {
+    tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo));
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  SMFile *   pLMFile = pRepo->fs->cstatus->pmf;
+
+  // Recv meta info from remote
+  if (tsdbRecvMetaInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  // No meta file, do nothing (rm local meta file)
+  if (pSynch->pmf == NULL) {
+    if (pLMFile == NULL) {
+      pSynch->mfChanged = false;
+    } else {
+      pSynch->mfChanged = true;
+    }
+    tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo));
+    return 0;
+  }
+
+  if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0) {
+    // Local has no meta file or has a different meta file, need to copy from remote
+    pSynch->mfChanged = true;
+
+    if (tsdbSendDecision(pSynch, true) < 0) {
+      tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo));
+
+    // Recv from remote
+    SMFile  mf;
+    SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID};
+    tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
+    if (tsdbCreateMFile(&mf, false) < 0) {
+      tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname);
+
+    int32_t readLen = (int32_t)pSynch->pmf->info.size;
+    int32_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen);
+    if (ret != readLen) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbError("vgId:%d, failed to recv metafile since %s, ret:%d readLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+                readLen);
+      tsdbCloseMFile(&mf);
+      tsdbRemoveMFile(&mf);
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile is received, size:%d", REPO_ID(pRepo), readLen);
+
+    mf.info = pSynch->pmf->info;
+    tsdbCloseMFile(&mf);
+    tsdbUpdateMFile(REPO_FS(pRepo), &mf);
+  } else {
+    pSynch->mfChanged = false;
+    tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo));
+    if (tsdbSendDecision(pSynch, false) < 0) {
+      tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+    tsdbUpdateMFile(REPO_FS(pRepo), pLMFile);
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSendMetaInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+  SMFile *   pMFile = pRepo->fs->cstatus->pmf;
+
+  if (pMFile) {
+    tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM);
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  void *ptr = SYNC_BUFFER(pSynch);
+  taosEncodeFixedU32(&ptr, tlen);
+  void *tptr = ptr;
+  if (pMFile) {
+    tsdbEncodeSMFileEx(&ptr, pMFile);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
+  }
+
+  int32_t writeLen = tlen + sizeof(uint32_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+              writeLen);
+    return -1;
+  }
+
+  tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen);
+  return 0;
+}
+
+static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+  char       buf[64] = {0};
+
+  int32_t readLen = sizeof(uint32_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
+    return -1;
+  }
+
+  taosDecodeFixedU32(buf, &tlen);
+
+  tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen);
+  if (tlen == 0) {
+    pSynch->pmf = NULL;
+    return 0;
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
+  if (ret != tlen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen);
+    return -1;
+  }
+
+  tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen);
+  if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
+    terrno = TSDB_CODE_TDB_MESSED_MSG;
+    tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  pSynch->pmf = &(pSynch->mf);
+  tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf);
+
+  return 0;
+}
+
+static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint8_t    decision = toSend;
+
+  int32_t writeLen = sizeof(uint8_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint8_t    decision = 0;
+
+  int32_t readLen = sizeof(uint8_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
+    return -1;
+  }
+
+  *toSend = decision;
+  return 0;
+}
+
+static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SFSIter    fsiter;
+  SDFileSet *pSet;
+
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+
+  do {
+    pSet = tsdbFSIterNext(&fsiter);
+    if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) {
+      tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1,
+                tstrerror(terrno));
+      return -1;
+    }
+
+    // No more file set to send, jut break
+    if (pSet == NULL) {
+      tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo));
+      break;
+    }
+  } while (true);
+
+  return 0;
+}
+
+static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SFSIter    fsiter;
+  SDFileSet *pLSet;  // Local file set
+
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+
+  pLSet = tsdbFSIterNext(&fsiter);
+  if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  while (true) {
+    if (pLSet == NULL && pSynch->pdf == NULL) {
+      tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo));
+      break;
+    } else {
+      tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1,
+               pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
+    }
+
+    if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) {
+      // remote not has pLSet->fid set, just remove local (do nothing to remote the fset)
+      tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid,
+               pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
+      pLSet = tsdbFSIterNext(&fsiter);
+    } else {
+      if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf)) {
+        // Just keep local files and notify remote not to send
+        tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid);
+
+        if (tsdbUpdateDFileSet(pfs, pLSet) < 0) {
+          tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        if (tsdbSendDecision(pSynch, false) < 0) {
+          tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+      } else {
+        // Need to copy from remote
+        tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid);
+
+        // Notify remote to send there file here
+        if (tsdbSendDecision(pSynch, true) < 0) {
+          tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        // Create local files and copy from remote
+        SDiskID   did;
+        SDFileSet fset;
+
+        tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id));
+        if (did.level == TFS_UNDECIDED_LEVEL) {
+          terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+          tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs));
+
+        // Create new FSET
+        if (tsdbCreateDFileSet(&fset, false) < 0) {
+          tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+          SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype);         // local file
+          SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype);  // remote file
+
+          tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo),
+                   pDFile->f.aname, pDFile->info.size, pRDFile->info.size);
+
+          int32_t writeLen = (int32_t)pRDFile->info.size;
+          int32_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen);
+          if (ret != writeLen) {
+            terrno = TAOS_SYSTEM_ERROR(errno);
+            tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), pDFile->f.aname,
+                      tstrerror(terrno), ret, writeLen);
+            tsdbCloseDFileSet(&fset);
+            tsdbRemoveDFileSet(&fset);
+            return -1;
+          }
+
+          // Update new file info
+          pDFile->info = pRDFile->info;
+          tsdbInfo("vgId:%d, file:%s is received, size:%d", REPO_ID(pRepo), pDFile->f.aname, writeLen);
+        }
+
+        tsdbCloseDFileSet(&fset);
+        if (tsdbUpdateDFileSet(pfs, &fset) < 0) {
+          tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno));
+          return -1;
+        }
+
+        tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid);
+      }
+
+      // Move forward
+      if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+        tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+        return -1;
+      }
+
+      if (pLSet) {
+        pLSet = tsdbFSIterNext(&fsiter);
+      }
+    }
+
+#if 0
+    if (pLSet == NULL) {
+      // Copy from remote >>>>>>>>>>>
+    } else {
+      if (pSynch->pdf == NULL) {
+        // Remove local file, just ignore ++++++++++++++
+        pLSet = tsdbFSIterNext(&fsiter);
+      } else {
+        if (pLSet->fid < pSynch->pdf->fid) {
+          // Remove local file, just ignore ++++++++++++
+          pLSet = tsdbFSIterNext(&fsiter);
+        } else if (pLSet->fid > pSynch->pdf->fid){
+          // Copy from remote >>>>>>>>>>>>>>
+          if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+            // TODO
+            return -1;
+          }
+        } else {
+          if (true/*TODO: is same fset*/) {
+            // No need to copy ---------------------
+          } else {
+            // copy from remote >>>>>>>>>>>>>.
+          }
+        }
+      }
+    }
+#endif
+  }
+
+  return 0;
+}
+
+static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype);
+    SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype);
+
+    if (memcmp((void *)(TSDB_FILE_INFO(pDFile1)), (void *)(TSDB_FILE_INFO(pDFile2)), sizeof(SDFInfo)) != 0) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  bool toSend = false;
+
+  if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) {
+    tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+    return -1;
+  }
+
+  // No file any more, no need to send file, just return
+  if (pSet == NULL) {
+    return 0;
+  }
+
+  if (tsdbRecvDecision(pSynch, &toSend) < 0) {
+    tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid,
+              tstrerror(terrno));
+    return -1;
+  }
+
+  if (toSend) {
+    tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid);
+
+    for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+      SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype);
+      
+      if (tsdbOpenDFile(&df, O_RDONLY) < 0) {
+        tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno));
+        return -1;
+      }
+
+      int32_t writeLen = (int32_t)df.info.size;
+      tsdbInfo("vgId:%d, file:%s will be sent, size:%d", REPO_ID(pRepo), df.f.aname, writeLen);
+
+      int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen);
+      if (ret != writeLen) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        tsdbError("vgId:%d, failed to send file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), df.f.aname,
+                  tstrerror(terrno), ret, writeLen);
+        tsdbCloseDFile(&df);
+        return -1;
+      }
+
+      tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname);
+      tsdbCloseDFile(&df);
+    }
+
+    tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid);
+  } else {
+    tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid);
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+
+  if (pSet) {
+    tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM);
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  void *ptr = SYNC_BUFFER(pSynch);
+  taosEncodeFixedU32(&ptr, tlen);
+  void *tptr = ptr;
+  if (pSet) {
+    tsdbEncodeDFileSetEx(&ptr, pSet);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
+  }
+
+  int32_t writeLen = tlen + sizeof(uint32_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen;
+  char       buf[64] = {0};
+
+  int32_t readLen = sizeof(uint32_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  taosDecodeFixedU32(buf, &tlen);
+
+  tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen);
+  if (tlen == 0) {
+    pSynch->pdf = NULL;
+    return 0;
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
+  if (ret != tlen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
+    terrno = TSDB_CODE_TDB_MESSED_MSG;
+    tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  pSynch->pdf = &(pSynch->df);
+  tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf);
+
+  return 0;
+}
+
+static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) {
+  // TODO: may need to stop and restart stream
+  if (isMfChanged) {
+    tsdbCloseMeta(pRepo);
+    tsdbFreeMeta(pRepo->tsdbMeta);
+    pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo));
+    tsdbOpenMeta(pRepo);
+    tsdbLoadMetaCache(pRepo, true);
+  }
+
+  tsdbUnRefMemTable(pRepo, pRepo->mem);
+  tsdbUnRefMemTable(pRepo, pRepo->imem);
+  pRepo->mem = NULL;
+  pRepo->imem = NULL;
+
+  if (tsdbRestoreInfo(pRepo) < 0) {
+    tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}

src/tsdb/tests/tsdbTests.cpp

@@ -12,7 +12,7 @@ static double getCurTime() {
 }
 
 typedef struct {
-  TSDB_REPO_T *pRepo;
+  STsdbRepo *pRepo;
   bool       isAscend;
   int        tid;
   uint64_t   uid;
@@ -143,7 +143,7 @@ TEST(TsdbTest, testInsertSpeed) {
   // Create and open repository
   tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
   tsdbCreateRepo(rootDir, &tsdbCfg);
-  TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL);
+  STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL);
   ASSERT_NE(repo, nullptr);
 
   // Create table

src/util/CMakeLists.txt

@@ -33,3 +33,7 @@ ELSEIF(TD_DARWIN)
   TARGET_LINK_LIBRARIES(tutil m)
   TARGET_LINK_LIBRARIES(tutil iconv)
 ENDIF()
+
+IF (TD_STORAGE)
+  TARGET_LINK_LIBRARIES(tutil storage)
+ENDIF ()

src/util/inc/tarray.h

@@ -21,9 +21,11 @@ extern "C" {
 #endif
 
 #include "os.h"
+#include "talgo.h"
 
 #define TARRAY_MIN_SIZE 8
 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
+#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize)
 
 typedef struct SArray {
   size_t size;
@@ -44,9 +46,20 @@ void* taosArrayInit(size_t size, size_t elemSize);
  *
  * @param pArray
  * @param pData
+ * @param nEles
  * @return
  */
-void* taosArrayPush(SArray* pArray, void* pData);
+void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles);
+
+/**
+ *
+ * @param pArray
+ * @param pData
+ * @return
+ */
+static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) {
+  return taosArrayPushBatch(pArray, pData, 1);
+}
 
 /**
  *
@@ -92,6 +105,14 @@ size_t taosArrayGetSize(const SArray* pArray);
  */
 void* taosArrayInsert(SArray* pArray, size_t index, void* pData);
 
+/**
+ * set data in array
+ * @param pArray
+ * @param index
+ * @param pData
+ */
+void taosArraySet(SArray* pArray, size_t index, void* pData);
+
 /**
  * remove data entry of the given index
  * @param pArray
@@ -122,7 +143,7 @@ void taosArrayClear(SArray* pArray);
  * destroy array list
  * @param pArray
  */
-void taosArrayDestroy(SArray* pArray);
+void* taosArrayDestroy(SArray* pArray);
 
 /**
  *
@@ -150,14 +171,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
  * @param compar
  * @param key
  */
-void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn);
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
 
 /**
  * search the array
  * @param pArray
  * @param key
  */
-char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn);
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags);
 
 #ifdef __cplusplus
 }

src/util/inc/tconfig.h

@@ -48,6 +48,7 @@ enum {
   TAOS_CFG_VTYPE_STRING,
   TAOS_CFG_VTYPE_IPSTR,
   TAOS_CFG_VTYPE_DIRECTORY,
+  TAOS_CFG_VTYPE_DATA_DIRCTORY,
 };
 
 enum {

src/util/inc/tkvstore.h

@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_KVSTORE_H_
-#define _TD_KVSTORE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-#define KVSTORE_FILE_VERSION ((uint32_t)0)
-
-typedef int (*iterFunc)(void *, void *cont, int contLen);
-typedef void (*afterFunc)(void *);
-
-typedef struct {
-  int64_t  size;  // including 512 bytes of header size
-  int64_t  tombSize;
-  int64_t  nRecords;
-  int64_t  nDels;
-  uint32_t magic;
-} SStoreInfo;
-
-typedef struct {
-  char *     fname;
-  int        fd;
-  char *     fsnap;
-  int        sfd;
-  char *     fnew;
-  int        nfd;
-  SHashObj * map;
-  iterFunc   iFunc;
-  afterFunc  aFunc;
-  void *     appH;
-  SStoreInfo info;
-} SKVStore;
-
-#define KVSTORE_MAGIC(s) (s)->info.magic
-
-int       tdCreateKVStore(char *fname);
-int       tdDestroyKVStore(char *fname);
-SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
-void      tdCloseKVStore(SKVStore *pStore);
-int       tdKVStoreStartCommit(SKVStore *pStore);
-int       tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
-int       tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid);
-int       tdKVStoreEndCommit(SKVStore *pStore);
-void      tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/util/inc/tlist.h

@@ -47,7 +47,7 @@ typedef struct {
 #define listNodeFree(n) free(n);
 
 SList *    tdListNew(int eleSize);
-void       tdListFree(SList *list);
+void *     tdListFree(SList *list);
 void       tdListEmpty(SList *list);
 void       tdListPrependNode(SList *list, SListNode *node);
 void       tdListAppendNode(SList *list, SListNode *node);

src/util/src/tarray.c

@@ -55,24 +55,29 @@ static int32_t taosArrayResize(SArray* pArray) {
   return 0;
 }
 
-void* taosArrayPush(SArray* pArray, void* pData) {
+void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) {
   if (pArray == NULL || pData == NULL) {
     return NULL;
   }
 
-  if (pArray->size >= pArray->capacity) {
+  if (pArray->size + nEles > pArray->capacity) {
-    int32_t ret = taosArrayResize(pArray);
+    size_t tsize = (pArray->capacity << 1u);
-    
+    while (pArray->size + nEles > tsize) {
-    // failed to push data into buffer due to the failure of memory allocation
+      tsize = (tsize << 1u);
-    if (ret != 0) {
+    }
+
+    pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize);
+    if (pArray->pData == NULL) {
       return NULL;
     }
+
+    pArray->capacity = tsize;
   }
 
   void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
-  memcpy(dst, pData, pArray->elemSize);
+  memcpy(dst, pData, pArray->elemSize * nEles);
 
-  pArray->size += 1;
+  pArray->size += nEles;
   return dst;
 }
 
@@ -133,6 +138,11 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
   return dst;
 }
 
+void taosArraySet(SArray* pArray, size_t index, void* pData) {
+  assert(index < pArray->size);
+  memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize);
+}
+
 void taosArrayRemove(SArray* pArray, size_t index) {
   assert(index < pArray->size);
   
@@ -184,13 +194,13 @@ void taosArrayClear(SArray* pArray) {
   pArray->size = 0;
 }
 
-void taosArrayDestroy(SArray* pArray) {
+void* taosArrayDestroy(SArray* pArray) {
-  if (pArray == NULL) {
+  if (pArray) {
-    return;
+    free(pArray->pData);
+    free(pArray);
   }
 
-  free(pArray->pData);
+  return NULL;
-  free(pArray);
 }
 
 void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
@@ -217,11 +227,11 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
-void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) {
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) {
   assert(pArray != NULL && comparFn != NULL);
   assert(key != NULL);
 
-  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
+  return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
 }
 
 void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
@@ -229,11 +239,11 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) {
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) {
   assert(pArray != NULL);
   assert(key != NULL);
 
-  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
+  void* p = taosbsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
   if (p == NULL) {
     return NULL;
   }

src/util/src/tcompare.c

@@ -277,7 +277,7 @@ int32_t taosArrayCompareString(const void* a, const void* b) {
 
 static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
   const SArray* arr = (const SArray*) pRight;
-  return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1;
+  return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1;
 }
 
 static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {

src/util/src/tconfig.c

@@ -20,7 +20,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tsystem.h"
+#include "tsocket.h"
 #include "tutil.h"
 
 SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}};
@@ -112,32 +112,34 @@ static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) {
   }
 }
 
-static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
+static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
   int   length = (int)strlen(input_value);
   char *option = (char *)cfg->ptr;
   if (length <= 0 || length > cfg->ptrLength) {
-    uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s",
+    uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option,
-           cfg->option, input_value, cfg->ptrLength, option);
+           input_value, cfg->ptrLength, option);
+    return false;
   } else {
     if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
       wordexp_t full_path;
       if (0 != wordexp(input_value, &full_path, 0)) {
         printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno));
         wordfree(&full_path);
-        return;
+        return false;
       }
-      
+
       if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
         strcpy(option, full_path.we_wordv[0]);
       }
-      
+
       wordfree(&full_path);
 
       int code = taosMkDir(option, 0755);
       if (code != 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
-        uError("config option:%s, input value:%s, directory not exist, create fail:%s",
+        uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value,
-             cfg->option, input_value, strerror(errno)); 
+               strerror(errno));
+        return false;
       }
       cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
     } else {
@@ -145,6 +147,8 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
             tsCfgStatusStr[cfg->cfgStatus], option);
     }
   }
+
+  return true;
 }
 
 static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) {
@@ -214,7 +218,7 @@ SGlobalCfg *taosGetConfigOption(const char *option) {
   return NULL;
 }
 
-static void taosReadConfigOption(const char *option, char *value) {
+static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) {
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
     if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue;
@@ -242,6 +246,11 @@ static void taosReadConfigOption(const char *option, char *value) {
       case TAOS_CFG_VTYPE_DIRECTORY:
         taosReadDirectoryConfig(cfg, value);
         break;
+      case TAOS_CFG_VTYPE_DATA_DIRCTORY:
+        if (taosReadDirectoryConfig(cfg, value)) {
+          taosReadDataDirCfg(value, value2, value3);
+        }
+        break;
       default:
         uError("config option:%s, input value:%s, can't be recognized", option, value);
         break;
@@ -322,8 +331,8 @@ void taosReadGlobalLogCfg() {
 }
 
 bool taosReadGlobalCfg() {
-  char * line, *option, *value, *value1;
+  char * line, *option, *value, *value2, *value3;
-  int    olen, vlen, vlen1;
+  int    olen, vlen, vlen2, vlen3;
   char   fileName[PATH_MAX] = {0};
 
   sprintf(fileName, "%s/taos.cfg", configDir);
@@ -346,8 +355,8 @@ bool taosReadGlobalCfg() {
   while (!feof(fp)) {
     memset(line, 0, len);
 
-    option = value = NULL;
+    option = value = value2 = value3 = NULL;
-    olen = vlen = 0;
+    olen = vlen = vlen2 = vlen3 = 0;
 
     tgetline(&line, &len, fp);
     line[len - 1] = 0;
@@ -360,11 +369,14 @@ bool taosReadGlobalCfg() {
     if (vlen == 0) continue;
     value[vlen] = 0;
 
-    // For dataDir, the format is:
+    paGetToken(value + vlen + 1, &value2, &vlen2);
-    // dataDir    /mnt/disk1    0
+    if (vlen2 != 0) {
-    paGetToken(value + vlen + 1, &value1, &vlen1);
+      value2[vlen2] = 0;
-    
+      paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
-    taosReadConfigOption(option, value);
+      if (vlen3 != 0) value3[vlen3] = 0;
+    }
+
+    taosReadConfigOption(option, value, value2, value3);
   }
 
   fclose(fp);
@@ -419,6 +431,8 @@ void taosPrintGlobalCfg() {
   }
 
   taosPrintOsInfo();
+  taosPrintDataDirCfg();
+  uInfo("==================================");
 }
 
 static void taosDumpCfg(SGlobalCfg *cfg) {

src/util/src/tkvstore.c

@@ -1,621 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#define TAOS_RANDOM_FILE_FAIL_TEST
-#include "os.h"
-#include "hash.h"
-#include "taoserror.h"
-#include "tchecksum.h"
-#include "tcoding.h"
-#include "tkvstore.h"
-#include "tulog.h"
-
-#define TD_KVSTORE_HEADER_SIZE 512
-#define TD_KVSTORE_MAJOR_VERSION 1
-#define TD_KVSTORE_MAINOR_VERSION 0
-#define TD_KVSTORE_SNAP_SUFFIX ".snap"
-#define TD_KVSTORE_NEW_SUFFIX ".new"
-#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF
-
-typedef struct {
-  uint64_t uid;
-  int64_t  offset;
-  int64_t  size;
-} SKVRecord;
-
-static int       tdInitKVStoreHeader(int fd, char *fname);
-static int       tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo);
-static void *    tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
-static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
-static char *    tdGetKVStoreSnapshotFname(char *fdata);
-static char *    tdGetKVStoreNewFname(char *fdata);
-static void      tdFreeKVStore(SKVStore *pStore);
-static int       tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
-static int       tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version);
-static int       tdEncodeKVRecord(void **buf, SKVRecord *pRecord);
-static void *    tdDecodeKVRecord(void *buf, SKVRecord *pRecord);
-static int       tdRestoreKVStore(SKVStore *pStore);
-
-int tdCreateKVStore(char *fname) {
-  int fd = open(fname, O_RDWR | O_CREAT | O_BINARY, 0755);
-  if (fd < 0) {
-    uError("failed to open file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (tdInitKVStoreHeader(fd, fname) < 0) goto _err;
-
-  if (fsync(fd) < 0) {
-    uError("failed to fsync file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (close(fd) < 0) {
-    uError("failed to close file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  return 0;
-
-_err:
-  if (fd >= 0) close(fd);
-  (void)remove(fname);
-  return -1;
-}
-
-int tdDestroyKVStore(char *fname) {
-  if (remove(fname) < 0) {
-    uError("failed to remove file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
-  SStoreInfo info = {0};
-  uint32_t   version = 0;
-
-  SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH);
-  if (pStore == NULL) return NULL;
-
-  pStore->fd = open(pStore->fname, O_RDWR | O_BINARY);
-  if (pStore->fd < 0) {
-    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pStore->sfd = open(pStore->fsnap, O_RDONLY | O_BINARY);
-  if (pStore->sfd < 0) {
-    if (errno != ENOENT) {
-      uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  } else {
-    uDebug("file %s exists, try to recover the KV store", pStore->fsnap);
-    if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info, &version) < 0) {
-      if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err;
-    } else {
-      if (version != KVSTORE_FILE_VERSION) {
-        uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fsnap,
-               version, KVSTORE_FILE_VERSION);
-      }
-
-      if (taosFtruncate(pStore->fd, info.size) < 0) {
-        uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
-      }
-
-      if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
-      if (fsync(pStore->fd) < 0) {
-        uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
-        goto _err;
-      }
-    }
-
-    close(pStore->sfd);
-    pStore->sfd = -1;
-    (void)remove(pStore->fsnap);
-  }
-
-  if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info, &version) < 0) goto _err;
-  if (version != KVSTORE_FILE_VERSION) {
-    uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fname, version,
-           KVSTORE_FILE_VERSION);
-  }
-
-  pStore->info.size = TD_KVSTORE_HEADER_SIZE;
-  pStore->info.magic = info.magic;
-
-  if (tdRestoreKVStore(pStore) < 0) goto _err;
-
-  close(pStore->fd);
-  pStore->fd = -1;
-
-  return pStore;
-
-_err:
-  if (pStore->fd > 0) {
-    close(pStore->fd);
-    pStore->fd = -1;
-  }
-  if (pStore->sfd > 0) {
-    close(pStore->sfd);
-    pStore->sfd = -1;
-  }
-  tdFreeKVStore(pStore);
-  return NULL;
-}
-
-void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); }
-
-int tdKVStoreStartCommit(SKVStore *pStore) {
-  ASSERT(pStore->fd < 0);
-
-  pStore->fd = open(pStore->fname, O_RDWR | O_BINARY);
-  if (pStore->fd < 0) {
-    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT | O_BINARY, 0755);
-  if (pStore->sfd < 0) {
-    uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (fsync(pStore->sfd) < 0) {
-    uError("failed to fsync file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (close(pStore->sfd) < 0) {
-    uError("failed to close file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  pStore->sfd = -1;
-
-  if (lseek(pStore->fd, 0, SEEK_END) < 0) {
-    uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  ASSERT(pStore->info.size == lseek(pStore->fd, 0, SEEK_CUR));
-
-  return 0;
-
-_err:
-  if (pStore->sfd > 0) {
-    close(pStore->sfd);
-    pStore->sfd = -1;
-    (void)remove(pStore->fsnap);
-  }
-  if (pStore->fd > 0) {
-    close(pStore->fd);
-    pStore->fd = -1;
-  }
-  return -1;
-}
-
-int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) {
-  SKVRecord rInfo = {0};
-  char      buf[64] = "\0";
-  char *    pBuf = buf;
-
-  rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
-  if (rInfo.offset < 0) {
-    uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  rInfo.uid = uid;
-  rInfo.size = contLen;
-
-  int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo);
-  ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
-  ASSERT(tlen == sizeof(SKVRecord));
-
-  if (taosWrite(pStore->fd, buf, tlen) < tlen) {
-    uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosWrite(pStore->fd, cont, contLen) < contLen) {
-    uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  pStore->info.magic =
-      taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM));
-  pStore->info.size += (sizeof(SKVRecord) + contLen);
-  SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid));
-  if (pRecord != NULL) {  // just to insert
-    pStore->info.tombSize += pRecord->size;
-  } else {
-    pStore->info.nRecords++;
-  }
-
-  taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
-  uTrace("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname);
-
-  return 0;
-}
-
-int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
-  SKVRecord rInfo = {0};
-  char      buf[128] = "\0";
-
-  SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid));
-  if (pRecord == NULL) {
-    uError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
-    return -1;
-  }
-
-  rInfo.offset = -pRecord->offset;
-  rInfo.uid = pRecord->uid;
-  rInfo.size = pRecord->size;
-
-  void *pBuf = buf;
-  tdEncodeKVRecord(&pBuf, &rInfo);
-
-  if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
-    uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf));
-  pStore->info.size += POINTER_DISTANCE(pBuf, buf);
-  pStore->info.nDels++;
-  pStore->info.nRecords--;
-  pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-
-  taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid));
-  uDebug("drop uid %" PRIu64 " from KV store %s", uid, pStore->fname);
-
-  return 0;
-}
-
-int tdKVStoreEndCommit(SKVStore *pStore) {
-  ASSERT(pStore->fd > 0);
-
-  if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info)) < 0) return -1;
-
-  if (fsync(pStore->fd) < 0) {
-    uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (close(pStore->fd) < 0) {
-    uError("failed to close file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-  pStore->fd = -1;
-
-  (void)remove(pStore->fsnap);
-  return 0;
-}
-
-void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) {
-  char       buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-  SStoreInfo info = {0};
-
-  int fd = open(fname, O_RDONLY | O_BINARY);
-  if (fd < 0) goto _err;
-
-  if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err;
-
-  void *pBuf = (void *)buf;
-  pBuf = tdDecodeStoreInfo(pBuf, &info);
-  off_t offset = lseek(fd, 0, SEEK_END);
-  if (offset < 0) goto _err;
-  close(fd);
-
-  *magic = info.magic;
-  *size = offset;
-
-  return;
-
-_err:
-  if (fd >= 0) close(fd);
-  *magic = TD_KVSTORE_INIT_MAGIC;
-  *size = 0;
-}
-
-static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version) {
-  char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-
-  if (lseek(fd, 0, SEEK_SET) < 0) {
-    uError("failed to lseek file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) {
-    uError("file %s is broken", fname);
-    terrno = TSDB_CODE_COM_FILE_CORRUPTED;
-    return -1;
-  }
-
-  void *pBuf = (void *)buf;
-  pBuf = tdDecodeStoreInfo(pBuf, pInfo);
-  pBuf = taosDecodeFixedU32(pBuf, version);
-
-  return 0;
-}
-
-static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
-  char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-
-  if (lseek(fd, 0, SEEK_SET) < 0) {
-    uError("failed to lseek file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  void *pBuf = buf;
-  tdEncodeStoreInfo(&pBuf, pInfo);
-  taosEncodeFixedU32(&pBuf, KVSTORE_FILE_VERSION);
-  ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
-  if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int tdInitKVStoreHeader(int fd, char *fname) {
-  SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC};
-
-  return tdUpdateKVStoreHeader(fd, fname, &info);
-}
-
-static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) {
-  int tlen = 0;
-  tlen += taosEncodeVariantI64(buf, pInfo->size);
-  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
-  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
-  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
-  tlen += taosEncodeFixedU32(buf, pInfo->magic);
-
-  return tlen;
-}
-
-static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
-  buf = taosDecodeVariantI64(buf, &(pInfo->size));
-  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
-  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
-
-  return buf;
-}
-
-static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
-  SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore));
-  if (pStore == NULL) goto _err;
-
-  pStore->fname = strdup(fname);
-  if (pStore->fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  pStore->fsnap = tdGetKVStoreSnapshotFname(fname);
-  if (pStore->fsnap == NULL) {
-    goto _err;
-  }
-
-  pStore->fnew = tdGetKVStoreNewFname(fname);
-  if (pStore->fnew == NULL) goto _err;
-
-  pStore->fd = -1;
-  pStore->sfd = -1;
-  pStore->nfd = -1;
-  pStore->iFunc = iFunc;
-  pStore->aFunc = aFunc;
-  pStore->appH = appH;
-  pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
-  if (pStore->map == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  return pStore;
-
-_err:
-  tdFreeKVStore(pStore);
-  return NULL;
-}
-
-static void tdFreeKVStore(SKVStore *pStore) {
-  if (pStore) {
-    tfree(pStore->fname);
-    tfree(pStore->fsnap);
-    tfree(pStore->fnew);
-    taosHashCleanup(pStore->map);
-    free(pStore);
-  }
-}
-
-static char *tdGetKVStoreSnapshotFname(char *fdata) {
-  size_t size = strlen(fdata) + strlen(TD_KVSTORE_SNAP_SUFFIX) + 1;
-  char * fname = malloc(size);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    return NULL;
-  }
-  sprintf(fname, "%s%s", fdata, TD_KVSTORE_SNAP_SUFFIX);
-  return fname;
-}
-
-static char *tdGetKVStoreNewFname(char *fdata) {
-  size_t size = strlen(fdata) + strlen(TD_KVSTORE_NEW_SUFFIX) + 1;
-  char * fname = malloc(size);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    return NULL;
-  }
-  sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX);
-  return fname;
-}
-
-static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) {
-  int tlen = 0;
-  tlen += taosEncodeFixedU64(buf, pRecord->uid);
-  tlen += taosEncodeFixedI64(buf, pRecord->offset);
-  tlen += taosEncodeFixedI64(buf, pRecord->size);
-
-  return tlen;
-}
-
-static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) {
-  buf = taosDecodeFixedU64(buf, &(pRecord->uid));
-  buf = taosDecodeFixedI64(buf, &(pRecord->offset));
-  buf = taosDecodeFixedI64(buf, &(pRecord->size));
-
-  return buf;
-}
-
-static int tdRestoreKVStore(SKVStore *pStore) {
-  char                  tbuf[128] = "\0";
-  void *                buf = NULL;
-  int64_t               maxBufSize = 0;
-  SKVRecord             rInfo = {0};
-  SKVRecord            *pRecord = NULL;
-
-  ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR));
-  ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
-
-  while (true) {
-    int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord));
-    if (tsize == 0) break;
-    if (tsize < sizeof(SKVRecord)) {
-      uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
-             pStore->info.size, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    char *pBuf = tdDecodeKVRecord(tbuf, &rInfo);
-    ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord));
-    ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
-
-    if (rInfo.offset < 0) {
-      taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid));
-      pStore->info.size += sizeof(SKVRecord);
-      pStore->info.nRecords--;
-      pStore->info.nDels++;
-      pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-    } else {
-      ASSERT(rInfo.offset > 0 && rInfo.size > 0);
-      if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
-        uError("failed to put record in KV store %s", pStore->fname);
-        terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-        goto _err;
-      }
-
-      maxBufSize = MAX(maxBufSize, rInfo.size);
-
-      if (lseek(pStore->fd, (off_t)rInfo.size, SEEK_CUR) < 0) {
-        uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
-      }
-
-      pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
-      pStore->info.nRecords++;
-    }
-  }
-
-  buf = malloc((size_t)maxBufSize);
-  if (buf == NULL) {
-    uError("failed to allocate %" PRId64 " bytes in KV store %s", maxBufSize, pStore->fname);
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pRecord = taosHashIterate(pStore->map, NULL);
-  while (pRecord) {
-    if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) {
-      uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset);
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
-      uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
-             strerror(errno), pRecord->offset);
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (pStore->iFunc) {
-      if ((*pStore->iFunc)(pStore->appH, buf, (int)pRecord->size) < 0) {
-        uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64
-               " since %s",
-               pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno));
-        goto _err;
-      }
-    }
-
-    pRecord = taosHashIterate(pStore->map, pRecord);
-  }
-
-  if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
-
-  tfree(buf);
-  return 0;
-
-_err:
-  taosHashCancelIterate(pStore->map, pRecord);
-  tfree(buf);
-  return -1;
-}

src/util/src/tlist.c

@@ -38,11 +38,13 @@ void tdListEmpty(SList *list) {
   list->numOfEles = 0;
 }
 
-void tdListFree(SList *list) {
+void *tdListFree(SList *list) {
   if (list) {
     tdListEmpty(list);
     free(list);
   }
+
+  return NULL;
 }
 
 void tdListPrependNode(SList *list, SListNode *node) {

src/util/src/tsocket.c

@@ -483,5 +483,5 @@ int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) {
     leftLen -= readLen;
   }
 
-  return 0;
+  return (int32_t)len;
 }

src/util/src/ttimer.c

@@ -188,7 +188,11 @@ static void removeTimer(uintptr_t id) {
 }
 
 static int64_t getMonotonicMs(void) {
+#ifdef WINDOWS
   return (int64_t) getMonotonicUs() / 1000;
+#else
+  return taosGetTimestampMs();
+#endif
 }
 
 static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
@@ -537,7 +541,8 @@ static void taosTmrModuleInit(void) {
 }
 
 void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) {
-  tmrInfo("ttimer monotonic clock source:%s", monotonicInit());
+  const char* ret = monotonicInit();
+  tmrInfo("ttimer monotonic clock source:%s", ret);
 
   pthread_once(&tmrModuleInit, taosTmrModuleInit);
 

src/vnode/CMakeLists.txt

@@ -11,4 +11,4 @@ INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 ADD_LIBRARY(vnode ${SRC})
-TARGET_LINK_LIBRARIES(vnode tsdb tcq)
+TARGET_LINK_LIBRARIES(vnode tsdb tcq common)

src/vnode/inc/vnodeMain.h

@@ -26,8 +26,6 @@ int32_t vnodeDrop(int32_t vgId);
 int32_t vnodeOpen(int32_t vgId);
 int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);
-
-int32_t vnodeReset(SVnodeObj *pVnode);
 void    vnodeCleanUp(SVnodeObj *pVnode);
 void    vnodeDestroy(SVnodeObj *pVnode);
 

src/vnode/inc/vnodeSync.h

@@ -25,7 +25,8 @@ uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t ei
 int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
 void     vnodeNotifyRole(int32_t vgId, int8_t role);
 void     vnodeCtrlFlow(int32_t vgId, int32_t level);
-int32_t  vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion);
+void     vnodeStartSyncFile(int32_t vgId);
+void     vnodeStopSyncFile(int32_t vgId, uint64_t fversion);
 void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
 int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
 int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);

src/vnode/src/vnodeMain.c

@@ -18,7 +18,7 @@
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tglobal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "query.h"
 #include "dnode.h"
 #include "vnodeCfg.h"
@@ -41,32 +41,19 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (mkdir(tsVnodeDir, 0755) != 0 && errno != EEXIST) {
+  if (tfsMkdir("vnode") < 0) {
-    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), tsVnodeDir);
+    vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
-    if (errno == EACCES) {
+    return terrno;
-      return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
-    } else if (errno == ENOSPC) {
-      return TSDB_CODE_VND_NO_DISKSPACE;
-    } else if (errno == ENOENT) {
-      return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
-    } else {
-      return TSDB_CODE_VND_INIT_FAILED;
-    }
   }
 
   char rootDir[TSDB_FILENAME_LEN] = {0};
   sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
-  if (mkdir(rootDir, 0755) != 0 && errno != EEXIST) {
+
-    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
+  char vnodeDir[TSDB_FILENAME_LEN] = "\0";
-    if (errno == EACCES) {
+  snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId);
-      return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
+  if (tfsMkdir(vnodeDir) < 0) {
-    } else if (errno == ENOSPC) {
+    vError("vgId:%d, failed to create vnode dir %s, reason:%s", pVnodeCfg->cfg.vgId, vnodeDir, strerror(errno));
-      return TSDB_CODE_VND_NO_DISKSPACE;
+    return terrno;
-    } else if (errno == ENOENT) {
-      return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
-    } else {
-      return TSDB_CODE_VND_INIT_FAILED;
-    }
   }
 
   code = vnodeWriteCfg(pVnodeCfg);
@@ -75,22 +62,24 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
     return code;
   }
 
-  STsdbCfg tsdbCfg = {0};
+  // STsdbCfg tsdbCfg = {0};
-  tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
+  // tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
-  tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
+  // tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
-  tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
+  // tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
-  tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
+  // tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
-  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
+  // tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
-  tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
+  // tsdbCfg.keep1               = pVnodeCfg->cfg.daysToKeep1;
-  tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
+  // tsdbCfg.keep2               = pVnodeCfg->cfg.daysToKeep2;
-  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
+  // tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
-  tsdbCfg.compression         = pVnodeCfg->cfg.compression;
+  // tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
-  tsdbCfg.update              = pVnodeCfg->cfg.update;
+  // tsdbCfg.precision           = pVnodeCfg->cfg.precision;
-  tsdbCfg.cacheLastRow        = pVnodeCfg->cfg.cacheLastRow;
+  // tsdbCfg.compression         = pVnodeCfg->cfg.compression;
+  // tsdbCfg.update              = pVnodeCfg->cfg.update;
+  // tsdbCfg.cacheLastRow        = pVnodeCfg->cfg.cacheLastRow;
 
-  char tsdbDir[TSDB_FILENAME_LEN] = {0};
+  // char tsdbDir[TSDB_FILENAME_LEN] = {0};
-  sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
+  // sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId);
-  if (tsdbCreateRepo(tsdbDir, &tsdbCfg) < 0) {
+  if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) {
     vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
     return TSDB_CODE_VND_INIT_FAILED;
   }
@@ -247,14 +236,13 @@ int32_t vnodeOpen(int32_t vgId) {
   appH.cqH = pVnode->cq;
   appH.cqCreateFunc = cqCreate;
   appH.cqDropFunc = cqDrop;
-  sprintf(temp, "%s/tsdb", rootDir);
 
   terrno = 0;
-  pVnode->tsdb = tsdbOpenRepo(temp, &appH);
+  pVnode->tsdb = tsdbOpenRepo(&(pVnode->tsdbCfg), &appH);
   if (pVnode->tsdb == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
-  } else if (terrno != TSDB_CODE_SUCCESS) {
+  } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) {
     vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
     if (pVnode->syncCfg.replica <= 1) {
@@ -301,20 +289,23 @@ int32_t vnodeOpen(int32_t vgId) {
   vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
 
   vnodeAddIntoHash(pVnode);
-
+  
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
   syncInfo.version = pVnode->version;
   syncInfo.syncCfg = pVnode->syncCfg;
   tstrncpy(syncInfo.path, rootDir, TSDB_FILENAME_LEN);
-  syncInfo.getWalInfo = vnodeGetWalInfo;
+  syncInfo.getWalInfoFp = vnodeGetWalInfo;
-  syncInfo.getFileInfo = vnodeGetFileInfo;
+  syncInfo.writeToCacheFp = vnodeWriteToCache;
-  syncInfo.writeToCache = vnodeWriteToCache;
   syncInfo.confirmForward = vnodeConfirmForard; 
-  syncInfo.notifyRole = vnodeNotifyRole;
+  syncInfo.notifyRoleFp = vnodeNotifyRole;
-  syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
+  syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow;
-  syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
+  syncInfo.startSyncFileFp = vnodeStartSyncFile;
-  syncInfo.getVersion = vnodeGetVersion;
+  syncInfo.stopSyncFileFp = vnodeStopSyncFile;
+  syncInfo.getVersionFp = vnodeGetVersion;
+  syncInfo.sendFileFp = tsdbSyncSend;
+  syncInfo.recvFileFp = tsdbSyncRecv;
+  syncInfo.pTsdb = pVnode->tsdb;
   pVnode->sync = syncStart(&syncInfo);
 
   if (pVnode->sync <= 0) {
@@ -344,7 +335,7 @@ int32_t vnodeClose(int32_t vgId) {
 void vnodeDestroy(SVnodeObj *pVnode) {
   int32_t code = 0;
   int32_t vgId = pVnode->vgId;
-
+  
   if (pVnode->qMgmt) {
     qCleanupQueryMgmt(pVnode->qMgmt);
     pVnode->qMgmt = NULL;
@@ -396,17 +387,17 @@ void vnodeDestroy(SVnodeObj *pVnode) {
   if (pVnode->dropped) {
     char rootDir[TSDB_FILENAME_LEN] = {0};    
     char newDir[TSDB_FILENAME_LEN] = {0};
-    sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId);
+    sprintf(rootDir, "%s/vnode%d", "vnode", vgId);
-    sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId);
+    sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
 
     if (0 == tsEnableVnodeBak) {
       vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
     } else {
-      taosRemoveDir(newDir);
+      tfsRmdir(newDir);
-      taosRename(rootDir, newDir);
+      tfsRename(rootDir, newDir);
     }
 
-    taosRemoveDir(rootDir);
+    tfsRmdir(rootDir);
     dnodeSendStatusMsgToMnode();
   }
 
@@ -466,37 +457,3 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
 
   return 0;
 }
-
-int32_t vnodeReset(SVnodeObj *pVnode) {
-  char rootDir[128] = "\0";
-  sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
-
-  if (!vnodeSetResetStatus(pVnode)) {
-    return -1;
-  }
-
-  void *tsdb = pVnode->tsdb;
-  pVnode->tsdb = NULL;
-  
-  // acquire vnode
-  int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
-
-  if (refCount > 3) {
-    tsem_wait(&pVnode->sem);
-  }
-
-  // close tsdb, then open tsdb
-  tsdbCloseRepo(tsdb, 0);
-  STsdbAppH appH = {0};
-  appH.appH = (void *)pVnode;
-  appH.notifyStatus = vnodeProcessTsdbStatus;
-  appH.cqH = pVnode->cq;
-  appH.cqCreateFunc = cqCreate;
-  appH.cqDropFunc = cqDrop;
-  pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
-
-  vnodeSetReadyStatus(pVnode);
-  vnodeRelease(pVnode);
-
-  return 0;
-}

src/vnode/src/vnodeSync.c

@@ -20,6 +20,7 @@
 #include "dnode.h"
 #include "vnodeVersion.h"
 #include "vnodeMain.h"
+#include "vnodeStatus.h"
 
 uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) {
   SVnodeObj *pVnode = vnodeAcquire(vgId);
@@ -83,22 +84,34 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) {
   vnodeRelease(pVnode);
 }
 
-int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) {
+void vnodeStartSyncFile(int32_t vgId) {
   SVnodeObj *pVnode = vnodeAcquire(vgId);
   if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while notify file synced", vgId);
+    vError("vgId:%d, vnode not found while start filesync", vgId);
-    return 0;
+    return;
+  }
+
+  vDebug("vgId:%d, datafile will be synced", vgId);
+  vnodeSetResetStatus(pVnode);
+
+  vnodeRelease(pVnode);
+}
+
+void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while stop filesync", vgId);
+    return;
   }
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;
   vnodeSaveVersion(pVnode);
 
-  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
+  vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
-  int32_t code = vnodeReset(pVnode);
+  vnodeSetReadyStatus(pVnode);
 
   vnodeRelease(pVnode);
-  return code;
 }
 
 void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {

tests/pytest/pytest_1.sh

@@ -16,7 +16,7 @@ python3 ./test.py -f insert/nchar.py
 python3 ./test.py -f insert/nchar-unicode.py
 python3 ./test.py -f insert/multi.py
 python3 ./test.py -f insert/randomNullCommit.py
-python3 insert/retentionpolicy.py
+#python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 #python3 ./test.py -f insert/before_1970.py
@@ -135,106 +135,4 @@ python3 ./test.py -f import_merge/importTORestart.py
 python3 ./test.py -f import_merge/importTPORestart.py
 python3 ./test.py -f import_merge/importTRestart.py
 python3 ./test.py -f import_merge/importInsertThenImport.py
-python3 ./test.py -f import_merge/importCSV.py
+python3 ./test.py -f import_merge/importCSV.py
-# user
-python3 ./test.py -f user/user_create.py
-python3 ./test.py -f user/pass_len.py
-
-# stable
-python3 ./test.py -f stable/query_after_reset.py
-
-#query
-python3 ./test.py -f query/filter.py
-python3 ./test.py -f query/filterCombo.py
-python3 ./test.py -f query/queryNormal.py
-python3 ./test.py -f query/queryError.py
-python3 ./test.py -f query/filterAllIntTypes.py
-python3 ./test.py -f query/filterFloatAndDouble.py
-python3 ./test.py -f query/filterOtherTypes.py
-python3 ./test.py -f query/querySort.py
-python3 ./test.py -f query/queryJoin.py
-python3 ./test.py -f query/select_last_crash.py
-python3 ./test.py -f query/queryNullValueTest.py
-python3 ./test.py -f query/queryInsertValue.py
-python3 ./test.py -f query/queryConnection.py
-python3 ./test.py -f query/queryCountCSVData.py
-python3 ./test.py -f query/natualInterval.py
-python3 ./test.py -f query/bug1471.py
-#python3 ./test.py -f query/dataLossTest.py
-python3 ./test.py -f query/bug1874.py
-python3 ./test.py -f query/bug1875.py
-python3 ./test.py -f query/bug1876.py
-python3 ./test.py -f query/bug2218.py
-python3 ./test.py -f query/bug2117.py
-python3 ./test.py -f query/bug2118.py
-python3 ./test.py -f query/bug2143.py
-python3 ./test.py -f query/sliding.py
-python3 ./test.py -f query/unionAllTest.py
-python3 ./test.py -f query/bug2281.py
-python3 ./test.py -f query/bug2119.py
-python3 ./test.py -f query/isNullTest.py
-python3 ./test.py -f query/queryWithTaosdKilled.py
-python3 ./test.py -f query/floatCompare.py
-
-#stream
-python3 ./test.py -f stream/metric_1.py
-python3 ./test.py -f stream/metric_n.py
-python3 ./test.py -f stream/new.py
-python3 ./test.py -f stream/stream1.py
-python3 ./test.py -f stream/stream2.py
-#python3 ./test.py -f stream/parser.py
-python3 ./test.py -f stream/history.py
-python3 ./test.py -f stream/sys.py
-python3 ./test.py -f stream/table_1.py
-python3 ./test.py -f stream/table_n.py
-
-#alter table
-python3 ./test.py -f alter/alter_table_crash.py
-
-# client
-python3 ./test.py -f client/client.py
-python3 ./test.py -f client/version.py
-python3 ./test.py -f client/alterDatabase.py
-python3 ./test.py -f client/noConnectionErrorTest.py
-
-# Misc
-python3 testCompress.py
-python3 testNoCompress.py
-python3 testMinTablesPerVnode.py
-
-# functions
-python3 ./test.py -f functions/function_avg.py -r 1
-python3 ./test.py -f functions/function_bottom.py -r 1
-python3 ./test.py -f functions/function_count.py -r 1
-python3 ./test.py -f functions/function_diff.py -r 1
-python3 ./test.py -f functions/function_first.py -r 1
-python3 ./test.py -f functions/function_last.py -r 1
-python3 ./test.py -f functions/function_last_row.py -r 1
-python3 ./test.py -f functions/function_leastsquares.py -r 1
-python3 ./test.py -f functions/function_max.py -r 1
-python3 ./test.py -f functions/function_min.py -r 1
-python3 ./test.py -f functions/function_operations.py -r 1 
-python3 ./test.py -f functions/function_percentile.py -r 1
-python3 ./test.py -f functions/function_spread.py -r 1
-python3 ./test.py -f functions/function_stddev.py -r 1
-python3 ./test.py -f functions/function_sum.py -r 1
-python3 ./test.py -f functions/function_top.py -r 1
-python3 ./test.py -f functions/function_twa.py -r 1
-python3 ./test.py -f functions/function_twa_test2.py
-python3 queryCount.py
-python3 ./test.py -f query/queryGroupbyWithInterval.py
-python3 client/twoClients.py
-python3 test.py -f query/queryInterval.py
-python3 test.py -f query/queryFillTest.py
-
-# tools
-python3 test.py -f tools/taosdemoTest.py
-python3 test.py -f tools/taosdumpTest.py
-python3 test.py -f tools/lowaTest.py
-python3 test.py -f tools/taosdemoTest2.py
-
-# subscribe
-python3 test.py -f subscribe/singlemeter.py
-#python3 test.py -f subscribe/stability.py  
-python3 test.py -f subscribe/supertable.py
-

tests/pytest/pytest_2.sh

@@ -1,18 +1,18 @@
 
 
 # update
-python3 ./test.py -f update/allow_update.py
+#python3 ./test.py -f update/allow_update.py
 python3 ./test.py -f update/allow_update-0.py
 python3 ./test.py -f update/append_commit_data.py
 python3 ./test.py -f update/append_commit_last-0.py
 python3 ./test.py -f update/append_commit_last.py
-python3 ./test.py -f update/merge_commit_data.py
+#python3 ./test.py -f update/merge_commit_data.py
-python3 ./test.py -f update/merge_commit_data-0.py
+#python3 ./test.py -f update/merge_commit_data-0.py
-python3 ./test.py -f update/merge_commit_data2.py
+#python3 ./test.py -f update/merge_commit_data2.py
-python3 ./test.py -f update/merge_commit_data2_update0.py
+#python3 ./test.py -f update/merge_commit_data2_update0.py
-python3 ./test.py -f update/merge_commit_last-0.py
+#python3 ./test.py -f update/merge_commit_last-0.py
-python3 ./test.py -f update/merge_commit_last.py
+#python3 ./test.py -f update/merge_commit_last.py
-python3 ./test.py -f update/bug_td2279.py
+#python3 ./test.py -f update/bug_td2279.py
 
 # wal
 python3 ./test.py -f wal/addOldWalTest.py

tests/pytest/pytest_3.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+ulimit -c unlimited
+
+
+python3 ./test.py -f insert/randomNullCommit.py
+
+# user
+python3 ./test.py -f user/user_create.py
+python3 ./test.py -f user/pass_len.py
+
+# stable
+python3 ./test.py -f stable/query_after_reset.py
+
+#query
+python3 ./test.py -f query/filter.py
+python3 ./test.py -f query/filterCombo.py
+python3 ./test.py -f query/queryNormal.py
+python3 ./test.py -f query/queryError.py
+python3 ./test.py -f query/filterAllIntTypes.py
+python3 ./test.py -f query/filterFloatAndDouble.py
+python3 ./test.py -f query/filterOtherTypes.py
+python3 ./test.py -f query/querySort.py
+python3 ./test.py -f query/queryJoin.py
+python3 ./test.py -f query/select_last_crash.py
+python3 ./test.py -f query/queryNullValueTest.py
+python3 ./test.py -f query/queryInsertValue.py
+python3 ./test.py -f query/queryConnection.py
+python3 ./test.py -f query/queryCountCSVData.py
+python3 ./test.py -f query/natualInterval.py
+python3 ./test.py -f query/bug1471.py
+#python3 ./test.py -f query/dataLossTest.py
+python3 ./test.py -f query/bug1874.py
+python3 ./test.py -f query/bug1875.py
+python3 ./test.py -f query/bug1876.py
+python3 ./test.py -f query/bug2218.py
+python3 ./test.py -f query/bug2117.py
+python3 ./test.py -f query/bug2118.py
+python3 ./test.py -f query/bug2143.py
+python3 ./test.py -f query/sliding.py
+python3 ./test.py -f query/unionAllTest.py
+python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
+python3 ./test.py -f query/isNullTest.py
+python3 ./test.py -f query/queryWithTaosdKilled.py
+python3 ./test.py -f query/floatCompare.py
+
+#stream
+python3 ./test.py -f stream/metric_1.py
+python3 ./test.py -f stream/metric_n.py
+python3 ./test.py -f stream/new.py
+python3 ./test.py -f stream/stream1.py
+python3 ./test.py -f stream/stream2.py
+#python3 ./test.py -f stream/parser.py
+python3 ./test.py -f stream/history.py
+python3 ./test.py -f stream/sys.py
+python3 ./test.py -f stream/table_1.py
+python3 ./test.py -f stream/table_n.py
+
+#alter table
+python3 ./test.py -f alter/alter_table_crash.py
+
+# client
+python3 ./test.py -f client/client.py
+python3 ./test.py -f client/version.py
+python3 ./test.py -f client/alterDatabase.py
+python3 ./test.py -f client/noConnectionErrorTest.py
+
+# Misc
+python3 testCompress.py
+python3 testNoCompress.py
+python3 testMinTablesPerVnode.py
+
+# functions
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
+python3 ./test.py -f functions/function_percentile.py -r 1
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+python3 ./test.py -f functions/function_twa.py -r 1
+python3 ./test.py -f functions/function_twa_test2.py
+python3 queryCount.py
+python3 ./test.py -f query/queryGroupbyWithInterval.py
+python3 client/twoClients.py
+python3 test.py -f query/queryInterval.py
+python3 test.py -f query/queryFillTest.py
+
+# tools
+python3 test.py -f tools/taosdemoTest.py
+python3 test.py -f tools/taosdumpTest.py
+python3 test.py -f tools/lowaTest.py
+python3 test.py -f tools/taosdemoTest2.py
+
+# subscribe
+python3 test.py -f subscribe/singlemeter.py
+#python3 test.py -f subscribe/stability.py  
+python3 test.py -f subscribe/supertable.py
+