diff --git a/.travis.yml b/.travis.yml
index 6a4acce451..4afdb6154e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,7 +52,8 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
-          pip3 install numpy
+          py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev
+          pip3 install psutil
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e53cca2ea8..8dd99f0b58 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,6 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-SET(TD_SYNC FALSE)
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
@@ -17,8 +16,8 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
 
 INCLUDE(cmake/input.inc)
 INCLUDE(cmake/platform.inc)
-INCLUDE(cmake/env.inc)
 INCLUDE(cmake/define.inc)
+INCLUDE(cmake/env.inc)
 INCLUDE(cmake/install.inc)
 
 ADD_SUBDIRECTORY(deps)
diff --git a/cmake/define.inc b/cmake/define.inc
index 93bf602610..1bb1692a2b 100755
--- a/cmake/define.inc
+++ b/cmake/define.inc
@@ -1,10 +1,6 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
 IF (TD_ACCOUNT)
   ADD_DEFINITIONS(-D_ACCT)
 ENDIF ()
@@ -32,3 +28,107 @@ ENDIF ()
 IF (TD_RANDOM_NETWORK_FAIL)
   ADD_DEFINITIONS(-DTAOS_RANDOM_NETWORK_FAIL)
 ENDIF ()
+
+IF (TD_ARM)
+  ADD_DEFINITIONS(-D_TD_ARM_)
+  IF (TD_ARM_64)
+    ADD_DEFINITIONS(-D_TD_ARM_64_)
+  ENDIF ()
+  IF (TD_ARM_32)
+    ADD_DEFINITIONS(-D_TD_ARM_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_MIPS)
+  ADD_DEFINITIONS(-D_TD_MIPS_)
+  IF (TD_MIPS_64)
+    ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  ENDIF ()
+  IF (TD_MIPS_32)
+    ADD_DEFINITIONS(-D_TD_MIPS_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_LINUX_64)
+  IF (NOT TD_ARM)
+    IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ELSE ()
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ENDIF ()
+  ELSE ()
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  ENDIF ()
+  FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
+  IF (ICONV_INCLUDE_EXIST)
+    ADD_DEFINITIONS(-DUSE_LIBICONV)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_32)
+  ADD_DEFINITIONS(-D_TD_LINUX_32)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  IF (NOT TD_ARM)
+    EXIT ()
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX)
+  ADD_DEFINITIONS(-DLINUX)
+  ADD_DEFINITIONS(-D_TD_LINUX)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+  
+  IF (${COVER} MATCHES "true")
+    MESSAGE(STATUS "Test coverage mode, add extra flags")
+    SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+    SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+    SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+  ENDIF ()
+ENDIF ()
+
+IF (TD_APLHINE)
+  SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
+  link_libraries(/usr/lib/libargp.a)
+  ADD_DEFINITIONS(-D_ALPINE)
+ENDIF ()
+
+IF (TD_DARWIN_64)
+  ADD_DEFINITIONS(-D_TD_DARWIN_64)
+  ADD_DEFINITIONS(-DDARWIN)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+ENDIF ()
+
+IF (TD_WINDOWS)
+  ADD_DEFINITIONS(-DWINDOWS)
+  ADD_DEFINITIONS(-D__CLEANUP_C)
+  ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
+  ADD_DEFINITIONS(-DPTW32_BUILD)
+  ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
+  SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+  IF (NOT TD_GODLL)
+    SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(DEBUG_FLAGS "/Zi /W3 /GL")
+    SET(RELEASE_FLAGS "/W0 /GL")
+  ENDIF ()
+ENDIF () 
+
+IF (TD_WINDOWS_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_64)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+ENDIF ()
+
+IF (TD_WINDOWS_32)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_32)
+ENDIF ()
+
diff --git a/cmake/env.inc b/cmake/env.inc
index fbfa40e5ae..67b934119d 100755
--- a/cmake/env.inc
+++ b/cmake/env.inc
@@ -9,7 +9,6 @@ SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
 SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
 
-MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
 MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
 MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
diff --git a/cmake/platform.inc b/cmake/platform.inc
index 2e0e2d6af0..a0668de7b5 100755
--- a/cmake/platform.inc
+++ b/cmake/platform.inc
@@ -14,6 +14,7 @@ PROJECT(TDengine)
 #
 
 # Set macro definitions according to os platform
+SET(TD_LINUX FALSE)
 SET(TD_LINUX_64 FALSE)
 SET(TD_LINUX_32 FALSE)
 SET(TD_ARM FALSE)
@@ -22,8 +23,11 @@ SET(TD_ARM_32 FALSE)
 SET(TD_MIPS FALSE)
 SET(TD_MIPS_64 FALSE)
 SET(TD_MIPS_32 FALSE)
-SET(TD_DARWIN_64 FALSE)
+SET(TD_WINDOWS FALSE)
 SET(TD_WINDOWS_64 FALSE)
+SET(TD_WINDOWS_32 FALSE)
+SET(TD_DARWIN_64 FALSE)
+SET(TD_APLHINE FALSE)
 
 # if generate ARM version:
 #   cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
@@ -31,24 +35,21 @@ IF (${CPUTYPE} MATCHES "aarch32")
   SET(TD_ARM TRUE)
   SET(TD_ARM_32 TRUE)
   SET(TD_PAGMODE_LITE TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_32_)
+  MESSAGE(STATUS "input cpuType: aarch32")
 ELSEIF (${CPUTYPE} MATCHES "aarch64")
   SET(TD_ARM TRUE)
   SET(TD_ARM_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_64_)
+  MESSAGE(STATUS "input cpuType: aarch64")
 ELSEIF (${CPUTYPE} MATCHES "mips64")
   SET(TD_MIPS TRUE)
   SET(TD_MIPS_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_MIPS_)
-  ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  MESSAGE(STATUS "input cpuType: mips64")
 ELSEIF (${CPUTYPE} MATCHES "x64")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x64")
 ELSEIF (${CPUTYPE} MATCHES "x86")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x86")
 ELSE ()
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
 ENDIF ()
 
 #
@@ -59,16 +60,13 @@ execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_
 MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+  SET(TD_LINUX TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_LINUX_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
-    ADD_DEFINITIONS(-D_M_X64)
     MESSAGE(STATUS "The current platform is Linux 64-bit")
   ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
+    SET(TD_LINUX_32 TRUE)
     IF (TD_ARM)
-      SET(TD_LINUX_32 TRUE)
-      SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
-      #ADD_DEFINITIONS(-D_M_IX86)
       MESSAGE(STATUS "The current platform is Linux 32-bit")
     ELSE ()
       MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
@@ -78,22 +76,26 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
     EXIT ()
   ENDIF ()
+
+  IF (${TD_OS_INFO} MATCHES "Alpine")
+    SET(TD_APLHINE TRUE)
+    MESSAGE(STATUS "The current OS is Alpine, append extra flags")
+  ENDIF()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_DARWIN_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
     MESSAGE(STATUS "The current platform is Darwin 64-bit")
   ELSE ()
     MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+  SET(TD_WINDOWS TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_WINDOWS_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
-    ADD_DEFINITIONS(-D_M_X64)
     MESSAGE(STATUS "The current platform is Windows 64-bit")
   ELSE ()
+    SET(TD_WINDOWS_32 TRUE)
     MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
@@ -101,74 +103,3 @@ ELSE()
   MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
   EXIT ()
 ENDIF ()
-
-IF (TD_LINUX_64)
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    IF (NOT TD_ARM)
-      IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ELSE ()
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ENDIF ()
-    ELSE ()
-      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ENDIF ()
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, append extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_libraries(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_LINUX_32)
-    IF (NOT TD_ARM)
-      EXIT ()
-    ENDIF ()
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, add extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_library(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_WINDOWS_64)
-    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
-    IF (NOT TD_GODLL)
-      SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
-      SET(DEBUG_FLAGS "/Zi /W3 /GL")
-      SET(RELEASE_FLAGS "/W0 /GL")
-    ENDIF ()
-    ADD_DEFINITIONS(-DWINDOWS)
-    ADD_DEFINITIONS(-D__CLEANUP_C)
-    ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
-    ADD_DEFINITIONS(-DPTW32_BUILD)
-    ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
-  ELSEIF (TD_DARWIN_64)
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    ADD_DEFINITIONS(-DDARWIN)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-  ELSE ()
-    MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
-    EXIT ()
-  ENDIF ()
diff --git a/documentation/webdocs/markdowndocs/Super Table-ch.md b/documentation/webdocs/markdowndocs/Super Table-ch.md
index 5f592ec339..f45f2f91b1 100644
--- a/documentation/webdocs/markdowndocs/Super Table-ch.md	
+++ b/documentation/webdocs/markdowndocs/Super Table-ch.md	
@@ -4,7 +4,7 @@ TDengine要求每个数据采集点单独建表。独立建表的模式能够避
 
 ## 什么是超级表
 
-STable是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。
+超级表是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。
 
 TDengine扩展标准SQL语法用于定义STable，使用关键词tags指定标签信息。语法如下：
 
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md
index 343ce80422..e494a1be45 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md	
+++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md	
@@ -556,7 +556,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
 
 - **APERCENTILE**
     ```mysql
@@ -565,7 +565,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
     
 - **LAST_ROW**
     ```mysql
diff --git a/documentation/webdocs/markdowndocs/TAOS SQL.md b/documentation/webdocs/markdowndocs/TAOS SQL.md
index c0d35e9afc..f94047fb6a 100644
--- a/documentation/webdocs/markdowndocs/TAOS SQL.md	
+++ b/documentation/webdocs/markdowndocs/TAOS SQL.md	
@@ -289,10 +289,10 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable. 
 
 
-- **WAVG**
+- **TWA**
   
     ```mysql
-    SELECT WAVG(field_name) FROM tb_name WHERE clause
+    SELECT TWA(field_name) FROM tb_name WHERE clause
     ```
     Function: return the time-weighted average value of a specific column  
     Return Data Type: `double`  
@@ -324,7 +324,7 @@ TDengine supports aggregations over numerical values, they are listed below:
 
 - **LEASTSQUARES**
     ```mysql
-    SELECT LEASTSQUARES(field_name) FROM tb_name [WHERE clause]
+    SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
     ```
     Function: performs a linear fit to the primary timestamp and the specified column. 
     Return Data Type: return a string of the coefficient and the interception of the fitted line.  
@@ -417,6 +417,15 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable.  
     Note: The range of `P` is `[0, 100]`. When `P=0` , `PERCENTILE` returns the equal value as `MIN`; when `P=100`, `PERCENTILE` returns the equal value as `MAX`. 
 
+- **APERCENTILE**
+    ```mysql
+    SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: the value of the specified column below which `P` percent of the data points fall, it returns approximate value of percentile.
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. 
+    Applied to: table/STable.  
+    Note: The range of `P` is `[0, 100]`. When `P=0` , `APERCENTILE` returns the equal value as `MIN`; when `P=100`, `APERCENTILE` returns the equal value as `MAX`. `APERCENTILE` has a much better performance than `PERCENTILE`.
 
 - **LAST_ROW**
     ```mysql
diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 289f2dbf91..b48cf23995 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -16,9 +16,9 @@
 
 TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括：
 
-- _taosd_：TDengine服务端可执行文件
-- _taos_： TDengine Shell可执行文件
-- _taosdump_：数据导出工具
+- _taosd_：TDengine服务端可执行文件。
+- _taos_： TDengine Shell可执行文件。
+- _taosdump_：数据导出工具。
 - *rmtaos*： 卸载TDengine的脚本, 该脚本会删除全部的程序和数据文件。请务必谨慎执行，如非必须不建议使用。
 
 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index f652c65f56..f2dbc04353 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -11,114 +11,107 @@
 # second full-qualified domain name (FQDN) for TDengine system, for cluster edition only
 # second                cluster_hostname2:6030
 
-# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only   
-# arbitrator            arbitrator_hostname:6030     
-
 # the full-qualified domain name (FQDN) of dnode 
-# fqdn                  hostname:6030
+# fqdn                  hostname
 
-# port for MNode connect to Client, default udp[6030-6055] tcp[6030]
+# port for MNode connect to Client, default udp/tcp [6030-6040]
 # serverPort            6030
 
-# http service port, default tcp[6020]
+# http service port, default tcp [6020]
 # httpPort              6020
 
-# set socket type ("udp" and "tcp")
-# the server and client should have the same socket type. Otherwise, connect will fail
-# sockettype            udp
-
-# for the cluster version, data file's directory is configured this way 
-# option                mount_path           tier_level
-# dataDir               /mnt/disk1/taos      0
-# dataDir               /mnt/disk2/taos      0
-# dataDir               /mnt/disk3/taos      0
-# dataDir               /mnt/disk4/taos      0
-# dataDir               /mnt/disk5/taos      0
-# dataDir               /mnt/disk6/taos      1
-# dataDir               /mnt/disk7/taos      1
-# for the stand-alone version, data file's directory is configured this way 
-# dataDir               /var/lib/taos
-
 # log file's directory
 # logDir                /var/log/taos
 
-# number of management nodes in the system
-# numOfMnodes           3
+# scriptDir file's directory
+# scriptDir             /var/log/taos
 
-# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode
-# alternativeRole       0
+# data file's directory
+# dataDir               /var/lib/taos
+
+# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only   
+# arbitrator            arbitrator_hostname:6030     
 
 # number of threads per CPU core
 # numOfThreadsPerCore   1.0
 
-# number of vgroups per db
-# maxVgroupsPerDb       0
-
-# max number of tables per vnode
-# maxTablesPerVnode     1000000
-
 # the ratio of threads responsible for querying in the total thread
 # ratioOfQueryThreads   0.5
 
-# interval of check load balance when the management node is in normal operation
-# balanceInterval       300
+# number of management nodes in the system
+# numOfMnodes           3
 
-# interval of DNode report status to MNode, unit is Second, for cluster version only 
-# statusInterval        1
+# if backup vnode directory when remove dnode
+# vnodeBak              1
 
-# interval of Shell send HB to MNode, unit is Second
-# shellActivityTimer    3
+# Whether to start load balancing
+# balance               1
+
+# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode
+# role                  0
+
+# max timer control block
+# maxTmrCtrl            512
 
 # interval of system monitor 
 # monitorInterval       30
 
+# number of seconds allowed for a dnode to be offline, for cluster version only 
+# offlineThreshold      8640000
+
 # RPC re-try timer, millisecond
 # rpcTimer              300
 
 # RPC maximum time for ack, seconds
 # rpcMaxTime            600
 
+# interval of DNode report status to MNode, unit is Second, for cluster version only 
+# statusInterval        1
+
+# interval of Shell send HB to MNode, unit is Second
+# shellActivityTimer    3
+
 # duration of to keep tableMeta kept in Cache, seconds
 # tableMetaKeepTimer    7200
 
 # Minimum sliding window time
-# minSlidingTime         10
+# minSlidingTime        10
 
 # Time window minimum
-# minIntervalTime        10
+# minIntervalTime       10
 
-# max length of an SQL
-# maxSQLLength          65380
+# the max allowed delayed time for launching continuous query. 20ms by default
+# maxStreamCompDelay    20000
 
-# Support the maximum number of records allowed for super table time sorting
-# maxNumOfOrderedRes    100000
+# The minimum time to wait before the first stream execution
+# maxFirstStreamCompDelay   10000
 
-# system locale
-# locale                en_US.UTF-8
+# Retry wait time benchmark
+# retryStreamCompDelay      10
 
-# default system charset
-# charset               UTF-8
+# the delayed time for launching each continuous query. 10% of the whole computing time window by default.
+# streamCompDelayRatio      0.1
 
-# system time zone
-# timezone              Asia/Shanghai (CST, +0800)
+# max number of vgroups per db
+# maxVgroupsPerDb           0
 
-# set write ahead log (WAL) level
-# walLevel              1
+# max number of tables per vnode
+# maxTablesPerVnode         1000000
 
-# enable/disable async log
-# asyncLog              1
-
-# the compressed rpc message, option:
-#  -1 (no compression)
-#   0 (all message compressed),
-# > 0 (rpc message body which larger than this value will be compressed)
-# compressMsgSize       -1
+# Step size of increasing table number in vnode
+# tableIncStepPerVnode      1000
 
 # cache block size (Mbyte)
-# cache                 16
+# cache                     16
 
 # number of cache blocks per vnode
-# blocks                4
+# blocks                    6
+
+# number of days per DB file
+# days                  10
+
+# number of days to keep DB file
+# keep                  3650
 
 # min row of records in file block
 # minRows               100
@@ -127,17 +120,44 @@
 # maxRows               4096
 
 # enable/disable compression
-# comp                  1
+# comp                  2
 
-# number of days per DB file
-# days                  10
+# set write ahead log (WAL) level
+# walLevel              1
 
-# number of days to keep DB file
-# keep                  3650
+# When walLevel is set to 2, the cycle of fsync is executed
+# fsync                 3000
 
 # number of replications, for cluster version only 
 # replica               1
 
+# mqtt uri  
+# mqttBrokerAddress     mqtt://username:password@hostname:1883/taos/
+
+# mqtt client name 
+# mqttBrokerClientId    taos_mqtt
+
+# the compressed rpc message, option:
+#  -1 (no compression)
+#   0 (all message compressed),
+# > 0 (rpc message body which larger than this value will be compressed)
+# compressMsgSize       -1
+
+# max length of an SQL
+# maxSQLLength          65480
+
+# Support the maximum number of records allowed for super table time sorting
+# maxNumOfOrderedRes    100000
+
+# system time zone
+# timezone              Asia/Shanghai (CST, +0800)
+
+# system locale
+# locale                en_US.UTF-8
+
+# default system charset
+# charset               UTF-8
+
 # max number of connections from client for dnode
 # maxShellConns         5000
 
@@ -153,67 +173,53 @@
 # Stop writing data when the disk size of the log folder is less than this value
 # minimalDataDirGB      0.1
 
-# mnode take into account while balance, for cluster version only 
-# mnodeEqualVnodeNum    4
-
-# number of seconds allowed for a dnode to be offline, for cluster version only 
-# offlineThreshold      864000
-
 # start http service
 # http                  1
 
+# start muqq service
+# mqtt                  0
+
 # start system monitor module
 # monitor               1
 
-# start http service
-# mqtt                  0
-
-# mqtt uri  
-# mqttBrokerAddress     mqtt://username:password@hostname:1883/taos/
-
-# mqtt client name 
-# mqttBrokerClientId    taos_mqtt
-
-# maximum number of rows returned by the restful interface
-# restfulRowLimit       10240
+# Record the SQL through restful interface
+# httpEnableRecordSql   0
 
 # number of threads used to process http requests
 # httpMaxThreads        2
 
-# The minimum time to wait before the first stream execution
-# maxFirstStreamCompDelay   10000
-
-# Retry wait time benchmark
-# retryStreamCompDelay      10
-
-# the delayed time for launching each continuous query. 10% of the whole computing time window by default.
-# streamCompDelayRatio  0.1
-
-# the max allowed delayed time for launching continuous query. 20ms by default
-# maxStreamCompDelay    20000
+# maximum number of rows returned by the restful interface
+# restfulRowLimit       10240
 
 # The following parameter is used to limit the maximum number of lines in log files.
 # max number of rows per log filters
 # numOfLogLines         10000000
 
+# enable/disable async log
+# asyncLog              1
+
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
-# 131: output warning and error，135: output info, warning and error to log.
-# 199: output debug, info, warning and error to both screen and file
+# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log.
+# 199: output debug, warning and error to both screen and file
+# 207: output trace, debug, warning and error to both screen and file
 
-# debug flag for basic utils
-# debugFlag             131
+# debug flag for all log type, take effect when non-zero value
+# debugFlag             0
 
 # debug flag for meta management messages
 # mDebugFlag            135
 
 # debug flag for dnode messages
-# dDebugFlag            131
+# dDebugFlag            135
 
-# debug flag for TDengine SDB
+# debug flag for sync module
 # sDebugFlag            135
 
-# debug flag for TDengine SDB
+# debug flag for WAL
+# wDebugFlag            135
+
+# debug flag for SDB
 # sdbDebugFlag          135
 
 # debug flag for RPC 
@@ -246,12 +252,12 @@
 # debug flag for query
 # qDebugflag            131
 
+# debug flag for vnode
+# vDebugflag            131
+
 # debug flag for http server
 # tsdbDebugFlag         131
 
-# Record the SQL through restful interface
-# httpEnableRecordSql   0
-
 # Record the SQL in taos client
 # tscEnableRecordSql    0
 
diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord
new file mode 100644
index 0000000000..bf4dba7258
--- /dev/null
+++ b/packaging/deb/tarbitratord
@@ -0,0 +1,88 @@
+#!/bin/bash 
+#
+# Modified from original source: Elastic Search
+# https://github.com/elasticsearch/elasticsearch
+# Thank you to the Elastic Search authors
+#
+# chkconfig: 2345 99 01
+#
+### BEGIN INIT INFO
+# Provides:          TDEngine
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:     $local_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts TDEngine tarbitrator
+# Description:       Starts TDEngine tarbitrator, a arbitrator
+### END INIT INFO
+
+set -e
+
+PATH="/bin:/usr/bin:/sbin:/usr/sbin"
+NAME="tarbitrator"
+USER="root"
+GROUP="root"
+DAEMON="/usr/local/taos/bin/tarbitrator"
+DAEMON_OPTS=""
+PID_FILE="/var/run/$NAME.pid"
+APPARGS=""
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+. /lib/lsb/init-functions
+
+case "$1" in
+    start)
+
+        log_action_begin_msg "Starting tarbitrator..."
+        if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then
+
+            touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE"
+
+            if [ -n "$MAX_OPEN_FILES" ]; then
+                ulimit -n $MAX_OPEN_FILES
+            fi
+
+            start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS
+
+            log_end_msg $?
+        fi
+        ;;
+
+    stop)
+        log_action_begin_msg "Stopping tarbitrator..."
+        set +e
+        if [ -f "$PID_FILE" ]; then
+            start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null
+            if [ $? -eq 1 ]; then
+                log_action_cont_msg "TSD is not running but pid file exists, cleaning up"
+            elif [ $? -eq 3 ]; then
+                PID="`cat $PID_FILE`"
+                log_failure_msg "Failed to stop tarbitrator (pid $PID)"
+                exit 1
+            fi
+            rm -f "$PID_FILE"
+        else
+            log_action_cont_msg "tarbitrator was not running"
+        fi
+        log_action_end_msg 0
+        set -e
+        ;;
+
+    restart|force-reload)
+        if [ -f "$PID_FILE" ]; then
+            $0 stop
+            sleep 1
+        fi
+        $0 start
+        ;;
+    status)
+        status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME"
+        ;;
+    *)
+        exit 1
+        ;;
+esac
+
+exit 0
diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord
new file mode 100644
index 0000000000..5d3f2b9c79
--- /dev/null
+++ b/packaging/rpm/tarbitratord
@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# tarbitratord      This shell script takes care of starting and stopping tarbitrator.
+#
+# chkconfig: 2345 99 01
+# description: tarbitrator is a arbitrator used in TDengine cluster.
+#
+#
+### BEGIN INIT INFO
+# Provides: TDEngine
+# Required-Start: $network $local_fs $remote_fs
+# Required-Stop: $network $local_fs $remote_fs
+# Short-Description: start and stop taosd
+# Description: tarbitrator is a arbitrator used in TDengine cluster.
+### END INIT INFO
+
+# Source init functions
+. /etc/init.d/functions
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+# Default program options
+NAME=tarbitrator
+PROG=/usr/local/taos/bin/tarbitrator
+USER=root
+GROUP=root
+
+# Default directories
+LOCK_DIR=/var/lock/subsys
+PID_DIR=/var/run/$NAME
+
+# Set file names
+LOCK_FILE=$LOCK_DIR/$NAME
+PID_FILE=$PID_DIR/$NAME.pid
+
+[ -e $PID_DIR ] || mkdir -p $PID_DIR
+
+PROG_OPTS=""
+
+start() {
+    echo -n "Starting ${NAME}: "
+    # check identity
+    curid="`id -u -n`"
+    if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then
+        echo "Must be run as root or $USER, but was run as $curid"
+        return 1
+    fi
+    # Sets the maximum number of open file descriptors allowed.
+    ulimit -n $MAX_OPEN_FILES
+    curulimit="`ulimit -n`"
+    if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then
+        echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit"
+        return 1
+    fi
+
+    if [ "`id -u -n`" == root ] ; then
+        # Changes the owner of the lock, and the pid files to allow
+        # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py.
+        touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE
+        touch $PID_FILE && chown $USER:$GROUP $PID_FILE
+        daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
+    else
+        # Don't have to change user.
+        daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &"
+    fi
+    retval=$?
+    sleep 2
+    echo
+    [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE)
+    return $retval
+}
+
+stop() {
+    echo -n "Stopping ${NAME}: "
+    killproc -p $PID_FILE $NAME
+    retval=$?
+    echo
+    # Non-root users don't have enough permission to remove pid and lock files.
+    # So, the opentsdb_restart.py cannot get rid of the files, and the command
+    # "service opentsdb status" will complain about the existing pid file.
+    # Makes the pid file empty.
+    echo > $PID_FILE
+    [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE)
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status -p $PID_FILE -l $LOCK_FILE $NAME
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+
+exit $?
diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh
index c573a2086f..b2613a7cb6 100644
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@@ -99,6 +99,44 @@ else
   os_type=1
 fi
 
+
+# =============================  get input parameters =================================================
+
+# install.sh -v [server | client]  -e [yes | no] -i [systemd | service | ...]
+
+# set parameters by default value
+interactiveFqdn=yes   # [yes | no]
+verType=server        # [server | client]
+initType=systemd      # [systemd | service | ...]
+
+while getopts "hv:e:i:" arg
+do
+  case $arg in
+    e)
+      #echo "interactiveFqdn=$OPTARG"
+      interactiveFqdn=$( echo $OPTARG )
+      ;;
+    v)
+      #echo "verType=$OPTARG"
+      verType=$(echo $OPTARG)
+      ;;
+    i)
+      #echo "initType=$OPTARG"
+      initType=$(echo $OPTARG)
+      ;;
+    h)
+      echo "Usage: `basename $0` -v [server | client]  -e [yes | no]"
+      exit 0
+      ;;
+    ?) #unknow option 
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
+
 function kill_taosd() {
   pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
   if [ -n "$pid" ]; then
@@ -129,15 +167,17 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
     ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
+    ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
 
     ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/*
 
     #Make link
-    [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos             || :
-    [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd          || :
-    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
-    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
-    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
+    [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                      || :
+    [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd                   || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump          || :
+    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo          || :
+    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos          || :
+    [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
 
     if [ "$verMode" == "cluster" ]; then
         ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/*
@@ -189,50 +229,55 @@ function install_config() {
         if ((${update_flag}==1)); then
             return 0
         fi
+        
+        if [ "$interactiveFqdn" == "no" ]; then
+            return 0
+        fi
 
-        IP_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
-        IP_PATTERN="\b$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\b"
+        #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+        #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)"
+        #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)"
+        #FQDN_PATTERN=":[0-9]{1,5}$"
 
+        # first full-qualified domain name (FQDN) for TDengine cluster system
         echo
-        echo -e -n "${GREEN}Enter the IP address of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :"
-        read masterIp
+        echo -e -n "${GREEN}Enter the FQDN of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :"
+        read firstFqdn
         while true; do
-            if [ ! -z "$masterIp" ]; then
-                # check the format of the masterIp
-                if [[ $masterIp =~ $IP_PATTERN ]]; then
-                    # Write the first IP to configuration file
-                    sudo sed -i -r "s/#*\s*(masterIp\s*).*/\1$masterIp/" ${cfg_dir}/taos.cfg
-
-                    # Get the second IP address
+            if [ ! -z "$firstFqdn" ]; then
+                # check the format of the firstFqdn
+                #if [[ $firstFqdn == $FQDN_PATTERN ]]; then
+                    # Write the first FQDN to configuration file
+                    ${csudo} sed -i -r "s/#*\s*(first\s*).*/\1$firstFqdn/" ${cfg_install_dir}/taos.cfg
 
+                    # Get the second FQDN
                     echo
-                    echo -e -n "${GREEN}Enter the IP address of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: "
-                    read secondIp
+                    echo -e -n "${GREEN}Enter the FQDN of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: "
+                    read secondFqdn
                     while true; do
-
-                        if [ ! -z "$secondIp" ]; then
-                            if [[ $secondIp =~ $IP_PATTERN ]]; then
-                                # Write the second IP to configuration file
-                                sudo sed -i -r "s/#*\s*(secondIp\s*).*/\1$secondIp/" ${cfg_dir}/taos.cfg
+                        if [ ! -z "$secondFqdn" ]; then
+                            #if [[ $secondFqdn == $FQDN_PATTERN ]]; then
+                                # Write the second FQDN to configuration file
+                                ${csudo} sed -i -r "s/#*\s*(second\s*).*/\1$secondFqdn/" ${cfg_install_dir}/taos.cfg
                                 break
-                            else
-                                read -p "Please enter the correct IP address: " secondIp
-                            fi
+                            #else
+                            #    read -p "Please enter the correct FQDN: " secondFqdn
+                            #fi
                         else
                             break
                         fi
                     done
     
                     break
-                else
-                    read -p "Please enter the correct IP address: " masterIp
-                fi
+                #else
+                #    read -p "Please enter the correct FQDN: " firstFqdn
+                #fi
             else
                 break
             fi
         done
 	
-	fi
+	  fi
 }
 
 
@@ -266,16 +311,24 @@ function clean_service_on_sysvinit() {
     if pidof taosd &> /dev/null; then
         ${csudo} service taosd stop || :
     fi
+    
+    if pidof tarbitrator &> /dev/null; then
+        ${csudo} service tarbitratord stop || :
+    fi
 
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --del taosd || :
+        ${csudo} chkconfig --del tarbitratord || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv -r taosd || :
+        ${csudo} insserv -r tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d -f taosd remove || :
+        ${csudo} update-rc.d -f tarbitratord remove || :
     fi
     
     ${csudo} rm -f ${service_config_dir}/taosd || :
+    ${csudo} rm -f ${service_config_dir}/tarbitratord || :
     
     if $(which init &> /dev/null); then
         ${csudo} init q || :
@@ -292,9 +345,13 @@ function install_service_on_sysvinit() {
     if ((${os_type}==1)); then
         ${csudo} cp -f ${script_dir}/init.d/taosd.deb ${install_main_dir}/init.d/taosd
         ${csudo} cp    ${script_dir}/init.d/taosd.deb ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd
+        ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
+        ${csudo} cp    ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord
     elif ((${os_type}==2)); then
         ${csudo} cp -f ${script_dir}/init.d/taosd.rpm ${install_main_dir}/init.d/taosd
         ${csudo} cp    ${script_dir}/init.d/taosd.rpm ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd
+        ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
+        ${csudo} cp    ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord
     fi
     
     #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"
@@ -303,11 +360,16 @@ function install_service_on_sysvinit() {
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --add taosd || :
         ${csudo} chkconfig --level 2345 taosd on || :
+        ${csudo} chkconfig --add tarbitratord || :
+        ${csudo} chkconfig --level 2345 tarbitratord on || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv taosd || :
         ${csudo} insserv -d taosd || :
+        ${csudo} insserv tarbitratord || :
+        ${csudo} insserv -d tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d taosd defaults || :
+        ${csudo} update-rc.d tarbitratord defaults || :
     fi
 }
 
@@ -331,8 +393,17 @@ function clean_service_on_systemd() {
         fi
         ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null
 	
-        ${csudo} rm -f ${nginx_service_config}
-	fi
+        ${csudo} rm -f ${nginx_service_config}              
+
+        tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
+        if systemctl is-active --quiet tarbitratord; then
+            echo "tarbitrator is running, stopping it..."
+            ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null
+        fi
+        ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null
+
+        ${csudo} rm -f ${tarbitratord_service_config}
+	  fi
 }
 
 # taos:2345:respawn:/etc/init.d/taosd start
@@ -364,28 +435,52 @@ function install_service_on_systemd() {
     ${csudo} systemctl enable taosd
 
     if [ "$verMode" == "cluster" ]; then		
+    
+        tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
+
+        ${csudo} bash -c "echo '[Unit]'                                  >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'After=network-online.target'             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Wants=network-online.target'             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo                                           >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo '[Service]'                               >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Type=simple'                             >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator'          >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitNOFILE=infinity'                    >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitNPROC=infinity'                     >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'LimitCORE=infinity'                      >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'TimeoutStartSec=0'                       >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StandardOutput=null'                     >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'Restart=always'                          >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StartLimitBurst=3'                       >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'StartLimitInterval=60s'                  >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo                                           >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo '[Install]'                               >> ${tarbitratord_service_config}"
+        ${csudo} bash -c "echo 'WantedBy=multi-user.target'              >> ${tarbitratord_service_config}"
+        ${csudo} systemctl enable tarbitratord  
+    
         nginx_service_config="${service_config_dir}/nginxd.service"
-        ${csudo} bash -c "echo '[Unit]'                             >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'After=network-online.target'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Wants=network-online.target'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo                                      >> ${nginx_service_config}"
-        ${csudo} bash -c "echo '[Service]'                          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Type=forking'                       >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid'        >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx'          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop'   >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitNOFILE=infinity'               >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitNPROC=infinity'                >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'LimitCORE=infinity'                 >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'TimeoutStartSec=0'                  >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StandardOutput=null'                >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'Restart=always'                     >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StartLimitBurst=3'                  >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'StartLimitInterval=60s'             >> ${nginx_service_config}"
-        ${csudo} bash -c "echo                                      >> ${nginx_service_config}"
-        ${csudo} bash -c "echo '[Install]'                          >> ${nginx_service_config}"
-        ${csudo} bash -c "echo 'WantedBy=multi-user.target'         >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Unit]'                                             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Description=Nginx For TDengine Service'             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'After=network-online.target'                        >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Wants=network-online.target'                        >> ${nginx_service_config}"
+        ${csudo} bash -c "echo                                                      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Service]'                                          >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Type=forking'                                       >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid'           >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx'             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop'      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitNOFILE=infinity'                               >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitNPROC=infinity'                                >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'LimitCORE=infinity'                                 >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'TimeoutStartSec=0'                                  >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StandardOutput=null'                                >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'Restart=always'                                     >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StartLimitBurst=3'                                  >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'StartLimitInterval=60s'                             >> ${nginx_service_config}"
+        ${csudo} bash -c "echo                                                      >> ${nginx_service_config}"
+        ${csudo} bash -c "echo '[Install]'                                          >> ${nginx_service_config}"
+        ${csudo} bash -c "echo 'WantedBy=multi-user.target'                         >> ${nginx_service_config}"
         if ! ${csudo} systemctl enable nginxd &> /dev/null; then
             ${csudo} systemctl daemon-reexec
             ${csudo} systemctl enable nginxd
@@ -600,8 +695,8 @@ function install_TDengine() {
                 echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
            else
                 echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
-            fi
-		else
+           fi
+		    else
             echo -e "${GREEN_DARK}To access TDengine    ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}"
         fi
 		
@@ -620,7 +715,7 @@ function install_TDengine() {
 
 
 ## ==============================Main program starts from here============================
-if [ -z $1 ]; then
+if [ "$verType" == "server" ]; then
     # Install server and client
     if [ -x ${bin_dir}/taosd ]; then
         update_flag=1
@@ -628,7 +723,8 @@ if [ -z $1 ]; then
     else
         install_TDengine
     fi
-else
+elif [ "$verType" == "client" ]; then
+    interactiveFqdn=no
     # Only install client
     if [ -x ${bin_dir}/taos ]; then
         update_flag=1
@@ -636,4 +732,6 @@ else
     else
         install_TDengine client
     fi
+else 
+    echo  "please input correct verType"   
 fi
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index c214626efd..1cb412d9dd 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taos
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -54,14 +54,18 @@ nginx_dir="${code_dir}/../../enterprise/src/plugins/web"
 # temp use rpm's taosd. TODO: later modify according to os type
 init_file_deb=${script_dir}/../deb/taosd
 init_file_rpm=${script_dir}/../rpm/taosd
+init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord
+init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord
 
 # make directories.
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg
-mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/*
+mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
 mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/taosd.deb
 mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taosd.rpm
+mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || :
+mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || :
 
 if [ "$verMode" == "cluster" ]; then
     sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >> remove_temp.sh
diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh
index 28cc835f30..9fb8731449 100755
--- a/packaging/tools/remove.sh
+++ b/packaging/tools/remove.sh
@@ -26,6 +26,7 @@ v15_java_app_dir="/usr/local/lib/taos"
 
 service_config_dir="/etc/systemd/system"
 taos_service_name="taosd"
+tarbitrator_service_name="tarbitratord"
 nginx_service_name="nginxd"
 csudo=""
 if command -v sudo > /dev/null; then
@@ -59,6 +60,12 @@ function kill_taosd() {
   fi
 }
 
+function kill_tarbitrator() {
+  pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo} kill -9 $pid   || :
+  fi
+}
 function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/taos      || :
@@ -102,9 +109,18 @@ function clean_service_on_systemd() {
     ${csudo} rm -f ${taosd_service_config}
 
     if [ "$verMode" == "cluster" ]; then
-		nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
-	
-   	 	if [ -d ${bin_dir}/web ]; then
+        tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
+
+        if systemctl is-active --quiet ${tarbitrator_service_name}; then
+            echo "TDengine tarbitrator is running, stopping it..."
+            ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
+        fi
+        ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null
+
+        ${csudo} rm -f ${tarbitratord_service_config}
+        
+		    nginx_service_config="${service_config_dir}/${nginx_service_name}.service"	
+   	 	  if [ -d ${bin_dir}/web ]; then
    	        if systemctl is-active --quiet ${nginx_service_name}; then
    	            echo "Nginx for TDengine is running, stopping it..."
    	            ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null
@@ -124,16 +140,25 @@ function clean_service_on_sysvinit() {
         echo "TDengine taosd is running, stopping it..."
         ${csudo} service taosd stop || :
     fi
-
+    
+    if pidof tarbitrator &> /dev/null; then
+        echo "TDengine tarbitrator is running, stopping it..."
+        ${csudo} service tarbitratord stop || :
+    fi
+    
     if ((${initd_mod}==1)); then
         ${csudo} chkconfig --del taosd || :
+        ${csudo} chkconfig --del tarbitratord || :
     elif ((${initd_mod}==2)); then
         ${csudo} insserv -r taosd || :
+        ${csudo} insserv -r tarbitratord || :
     elif ((${initd_mod}==3)); then
         ${csudo} update-rc.d -f taosd remove || :
+        ${csudo} update-rc.d -f tarbitratord remove || :
     fi
     
     ${csudo} rm -f ${service_config_dir}/taosd || :
+    ${csudo} rm -f ${service_config_dir}/tarbitratord || :
    
     if $(which init &> /dev/null); then
         ${csudo} init q || :
@@ -148,6 +173,7 @@ function clean_service() {
     else
         # must manual stop taosd
         kill_taosd
+        kill_tarbitrator
     fi
 }
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 57d7234379..ede66d95bb 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -10,6 +10,8 @@ ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)
 ADD_SUBDIRECTORY(kit)
 ADD_SUBDIRECTORY(plugins)
+ADD_SUBDIRECTORY(sync)
+ADD_SUBDIRECTORY(balance)
 ADD_SUBDIRECTORY(mnode)
 ADD_SUBDIRECTORY(vnode)
 ADD_SUBDIRECTORY(tsdb)
diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt
new file mode 100644
index 0000000000..45b7c4ed57
--- /dev/null
+++ b/src/balance/CMakeLists.txt
@@ -0,0 +1,16 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sdb/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  ADD_LIBRARY(balance ${SRC})
+ENDIF ()
diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
new file mode 100644
index 0000000000..3ef616bff0
--- /dev/null
+++ b/src/balance/src/balance.c
@@ -0,0 +1,1006 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "tutil.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+extern void *       tsMnodeTmr;
+static void *       tsBalanceTimer = NULL;
+static int32_t      tsBalanceDnodeListSize = 0;
+static SDnodeObj ** tsBalanceDnodeList = NULL;
+static int32_t      tsBalanceDnodeListMallocSize = 16;
+static pthread_mutex_t tsBalanceMutex;
+
+static void  balanceStartTimer(int64_t mseconds);
+static void  balanceInitDnodeList();
+static void  balanceCleanupDnodeList();
+static void  balanceAccquireDnodeList();
+static void  balanceReleaseDnodeList();
+static void  balanceMonitorDnodeModule();
+static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static void balanceLock() {
+  pthread_mutex_lock(&tsBalanceMutex);
+}
+
+static void balanceUnLock() {
+  pthread_mutex_unlock(&tsBalanceMutex);
+}
+
+static bool balanceCheckFree(SDnodeObj *pDnode) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+    mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
+    return false;
+  }
+  
+  if (pDnode->openVnodes >= TSDB_MAX_VNODES) {
+    mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES);
+    return false;
+  }
+
+  if (pDnode->diskAvailable <= tsMinimalDataDirGB) {
+    mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable);
+    return false;
+  }
+
+  if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) {
+    mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId);
+    return false;
+  }
+
+  return true;
+}
+
+static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+  mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
+
+  SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
+  if (pDnode != NULL) {
+    atomic_sub_fetch_32(&pDnode->openVnodes, 1);
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */
+  int32_t   numOfVnodes = 0;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i;
+    if (pTmpVodeGid == pVnodeGid) {
+      continue;
+    }
+    vnodeGid[numOfVnodes] = *pTmpVodeGid;
+    ++numOfVnodes;
+  }
+  memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
+  pVgroup->numOfVnodes = numOfVnodes;
+
+  mnodeUpdateVgroup(pVgroup);
+}
+
+static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+  // SVnodeGid tmp = *pVnodeGid1;
+  // *pVnodeGid1 = *pVnodeGid2;
+  // *pVnodeGid2 = tmp;
+}
+
+int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+  int32_t dnode = 0;
+  int32_t vnodes = 0;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
+      if (balanceCheckFree(pDnode)) {
+        SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
+        pVnodeGid->dnodeId = pDnode->dnodeId;
+        pVnodeGid->pDnode = pDnode;
+        dnode++;
+        vnodes++;
+        break;
+      }
+    }
+  }
+
+  if (vnodes != pVgroup->numOfVnodes) {
+    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
+           pVgroup->numOfVnodes, vnodes);
+    balanceReleaseDnodeList();       
+    balanceUnLock();
+    return -1;
+  }
+
+  /*
+   * make the choice more random.
+   * replica 1: no choice
+   * replica 2: there are 2 combinations
+   * replica 3 or larger: there are 6 combinations
+   */
+  if (pVgroup->numOfVnodes == 1) {
+  } else if (pVgroup->numOfVnodes == 2) {
+    if (rand() % 2 == 0) {
+      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+    }
+  } else {
+    int32_t randVal = rand() % 6;
+    if (randVal == 1) {  // 1, 0, 2
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+    } else if (randVal == 2) {  // 1, 2, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else if (randVal == 3) {  // 2, 1, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+    } else if (randVal == 4) {  // 2, 0, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    }
+    if (randVal == 5) {  // 0, 2, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else {
+    }  // 0, 1, 2
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+  return 0;
+}
+
+static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+  if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
+    return false;
+  }
+
+  bool isReady = false;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
+    if (pVnode == pRmVnode) continue;
+
+    mTrace("vgId:%d, change vgroup status, dnode:%d status:%d", pVgroup->vgId, pVnode->pDnode->dnodeId,
+           pVnode->pDnode->status);
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
+
+    if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+      isReady = true;
+    }
+  }
+
+  return isReady;
+}
+
+/**
+ * desc: remove one vnode from vgroup
+ * all vnodes in vgroup should in ready state, except the balancing one
+ **/
+static void balanceRemoveVnode(SVgObj *pVgroup) {
+  if (pVgroup->numOfVnodes <= 1) return;
+
+  SVnodeGid *pRmVnode = NULL;
+  SVnodeGid *pSelVnode = NULL;
+  int32_t    maxScore = 0;
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]);
+    SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId);
+
+    if (pDnode == NULL) {
+      mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+      break;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else if (pVnode->dnodeId == pVgroup->lbDnodeId) {
+      mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else {
+      if (pSelVnode == NULL) {
+        pSelVnode = pVnode;
+        maxScore = pDnode->score;
+      } else {
+        if (maxScore < pDnode->score) {
+          pSelVnode = pVnode;
+          maxScore = pDnode->score;
+        }
+      }
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  if (pRmVnode != NULL) {
+    pSelVnode = pRmVnode;
+  }
+
+  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+    mDebug("vgId:%d, is not ready", pVgroup->vgId);
+  } else {
+    mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
+    balanceDiscardVnode(pVgroup, pSelVnode);
+  }
+}
+
+static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+ for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pGid = &pVgroup->vnodeGid[i];
+    if (pGid->dnodeId == 0) break;
+    if (pGid->dnodeId == pDnode->dnodeId) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * desc: add vnode to vgroup, find a new one if dest dnode is null
+ **/
+static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+  if (pDestDnode == NULL) {
+    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+      if (pDnode == pSrcDnode) continue;
+      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (!balanceCheckFree(pDnode)) continue;
+      
+      pDestDnode = pDnode;
+      mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
+      break;
+    }
+  }
+
+  if (pDestDnode == NULL) {
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes;
+  pVnodeGid->dnodeId = pDestDnode->dnodeId;
+  pVnodeGid->pDnode = pDestDnode;
+  pVgroup->numOfVnodes++;
+
+  if (pSrcDnode != NULL) {
+    pVgroup->lbDnodeId = pSrcDnode->dnodeId;
+  }
+
+  atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
+
+  mnodeUpdateVgroup(pVgroup);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static bool balanceMonitorBalance() {
+  if (tsBalanceDnodeListSize < 2) return false;
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[src];
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+           pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
+           pDnode->openVnodes);
+  }
+
+  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  if (scoresDiff < 0.01) {
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    return false;
+  }
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
+    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
+    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+    if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
+      continue;
+    }
+
+    void *pIter = NULL;
+    while (1) {
+      SVgObj *pVgroup;
+      pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+      if (pVgroup == NULL) break;
+
+      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+        for (int32_t dest = 0; dest < src; dest++) {
+          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
+          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+
+          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          if (srcScore + 0.0001 < destScore) continue;
+          if (!balanceCheckFree(pDestDnode)) continue;
+          
+          mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
+                 pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
+                 srcScore, pDestDnode->score, destScore);
+          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          mnodeDecVgroupRef(pVgroup);
+          sdbFreeIter(pIter);
+          return true;
+        }
+      }
+
+      mnodeDecVgroupRef(pVgroup);
+    }
+
+    sdbFreeIter(pIter);
+  }
+
+  return false;
+}
+
+// if mgmt changed to master
+// 1. reset balanceAccessSquence to zero
+// 2. reset state of dnodes to offline
+// 3. reset lastAccess of dnodes to zero
+void balanceReset() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    // while master change, should reset dnode to offline
+    mInfo("dnode:%d set access:%d to 0", pDnode->dnodeId, pDnode->lastAccess);
+    pDnode->lastAccess = 0;
+    if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
+      pDnode->status = TAOS_DN_STATUS_OFFLINE;
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  tsAccessSquence = 0;
+}
+
+static int32_t balanceMonitorVgroups() {
+  void *  pIter = NULL;
+  SVgObj *pVgroup = NULL;
+  bool    hasUpdatingVgroup = false;
+
+  while (1) {
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    int32_t dbReplica = pVgroup->pDb->cfg.replications;
+    int32_t vgReplica = pVgroup->numOfVnodes;
+    
+    if (vgReplica > dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceRemoveVnode(pVgroup);
+    } else if (vgReplica < dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceAddVnode(pVgroup, NULL, NULL);
+    }
+
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+
+  return hasUpdatingVgroup;
+}
+
+static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+  mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
+
+  void *  pIter = NULL;
+  bool    hasThisDnode = false;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    mnodeDecVgroupRef(pVgroup);
+
+    if (hasThisDnode) break;
+  }
+
+  sdbFreeIter(pIter);
+
+  if (!hasThisDnode) {
+    mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
+    mnodeDropDnode(pDnode, NULL);
+    return true;
+  }
+
+  return false;
+}
+
+static bool balanceMontiorDropping() {
+  void *pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    mnodeDecDnodeRef(pDnode);
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
+      if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; 
+      if (mnodeGetDnodesNum() <= 1) continue;
+
+      mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
+              tsAccessSquence - pDnode->lastAccess);
+
+      pDnode->status = TAOS_DN_STATUS_DROPPING;
+      mnodeUpdateDnode(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return true;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      bool ret = balanceMonitorDnodeDropping(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return ret;
+    }
+  }
+
+  sdbFreeIter(pIter);
+
+  return false;
+}
+
+static bool balanceStart() {
+  if (!sdbIsMaster()) return false;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  balanceMonitorDnodeModule();
+
+  bool updateSoon = balanceMontiorDropping();
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorVgroups();
+  }
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorBalance();
+  }
+ 
+  balanceReleaseDnodeList();
+
+  balanceUnLock();
+
+  return updateSoon;
+}
+
+static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+      if (pVgroup->vnodeGid[i].pDnode == pDnode) {
+        pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE;
+      }
+    }
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceCheckDnodeAccess() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (tsAccessSquence - pDnode->lastAccess > 3) {
+      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
+        pDnode->status = TAOS_DN_STATUS_OFFLINE;
+        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        balanceSetVgroupOffline(pDnode);
+      }
+    }
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+
+  tsBalanceTimer = NULL;
+  tsAccessSquence ++;
+
+  balanceCheckDnodeAccess();  
+  bool updateSoon = false;
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      updateSoon = balanceStart();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    updateSoon = balanceStart();
+  }
+
+  if (updateSoon) {
+    balanceStartTimer(1000);
+  } else {
+    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
+  }
+}
+
+static void balanceStartTimer(int64_t mseconds) {
+  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
+}
+
+void balanceSyncNotify() {
+  if (sdbIsMaster()) {
+    balanceLock();
+    balanceAccquireDnodeList();
+    balanceMonitorDnodeModule();
+    balanceReleaseDnodeList();
+    balanceUnLock();
+  }
+}
+
+void balanceAsyncNotify() {
+  balanceStartTimer(500); 
+}
+
+int32_t balanceInit() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
+  
+  pthread_mutex_init(&tsBalanceMutex, NULL);
+  balanceInitDnodeList();
+  balanceStartTimer(2000);
+  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
+
+  balanceReset();
+  
+  return 0;
+}
+
+void balanceCleanUp() {
+  if (tsBalanceTimer != NULL) {
+    taosTmrStopA(&tsBalanceTimer);
+    pthread_mutex_destroy(&tsBalanceMutex);
+    tsBalanceTimer = NULL;
+    mDebug("stop balance timer");
+  }
+  balanceCleanupDnodeList();
+}
+
+int32_t balanceDropDnode(SDnodeObj *pDnode) {
+  int32_t    totalFreeVnodes = 0;
+  void *     pIter = NULL;
+  SDnodeObj *pTempDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pTempDnode);
+    if (pTempDnode == NULL) break;
+
+    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+      totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
+    }
+
+    mnodeDecDnodeRef(pTempDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  if (pDnode->openVnodes > totalFreeVnodes) {
+    mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
+    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+  }
+
+  pDnode->status = TAOS_DN_STATUS_DROPPING;
+  mnodeUpdateDnode(pDnode);
+  
+  balanceStartTimer(1100);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float balanceCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+void balanceCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
+                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
+}
+
+float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                        balanceCalcBandwidthScore(pDnode);
+  float moduleScore = balanceCalcModuleScore(pDnode);
+  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+static void balanceInitDnodeList() {
+  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
+}
+
+static void balanceCleanupDnodeList() {
+  if (tsBalanceDnodeList != NULL) {
+    free(tsBalanceDnodeList);
+    tsBalanceDnodeList = NULL;
+  }
+}
+
+static void balanceCheckDnodeListSize(int32_t dnodesNum) {
+  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
+    tsBalanceDnodeListMallocSize = dnodesNum * 2;
+    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
+  }
+}
+
+void balanceAccquireDnodeList() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  balanceCheckDnodeListSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {  
+    if (dnodeIndex >= dnodesNum) break;
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    balanceCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
+        break;
+      }
+      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
+    }
+    tsBalanceDnodeList[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  sdbFreeIter(pIter);
+
+  tsBalanceDnodeListSize = dnodeIndex;
+}
+
+void balanceReleaseDnodeList() {
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0)  {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+  
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                      balanceCalcBandwidthScore(pDnode);
+    float moduleScore = balanceCalcModuleScore(pDnode);
+    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
+
+static void balanceMonitorDnodeModule() {
+  int32_t numOfMnodes = mnodeGetMnodesNum();
+  if (numOfMnodes >= tsNumOfMnodes) return;
+
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode == NULL) break;
+
+    if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      continue;
+    }
+
+    if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE) {
+      continue;
+    }
+
+    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, add mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
+    mnodeAddMnode(pDnode->dnodeId);
+    
+    numOfMnodes = mnodeGetMnodesNum();
+    if (numOfMnodes >= tsNumOfMnodes) return;
+  }
+}
+
+int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+  if (!sdbIsMaster()) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  if (tsEnableBalance != 0) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_BALANCE_ENABLED;
+  }
+
+  SVgObj *pVgroup = mnodeGetVgroup(vnodeId);
+  if (pVgroup == NULL) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_VGROUP_NOT_EXIST;
+  }
+
+  SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId);
+  if (pDestDnode == NULL) {
+    mnodeDecVgroupRef(pVgroup);
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  balanceLock();
+  balanceAccquireDnodeList();
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, pSrcDnode->dnodeId);
+    code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
+  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, dnodeId);
+    code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
+  } else if (!balanceCheckFree(pDestDnode)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
+           dnodeId);
+    code = TSDB_CODE_MND_DNODE_NOT_FREE;
+  } else {
+    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+
+  mnodeDecVgroupRef(pVgroup);
+  mnodeDecDnodeRef(pDestDnode);
+
+  return code;
+}
\ No newline at end of file
diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt
index 00fa1a1479..eaebd2f19d 100644
--- a/src/client/CMakeLists.txt
+++ b/src/client/CMakeLists.txt
@@ -4,11 +4,11 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 INCLUDE_DIRECTORIES(jni)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 17840df4a4..7a44870938 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -31,8 +31,8 @@ extern "C" {
 #include "tutil.h"
 
 #include "qExecutor.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "tcmdtype.h"
 
 // forward declaration
@@ -60,15 +60,14 @@ typedef struct SCMCorVgroupInfo {
 } SCMCorVgroupInfo;
 
 typedef struct STableMeta {
-  STableComInfo tableInfo;
-  uint8_t       tableType;
-  int16_t       sversion;
-  int16_t       tversion;
+  STableComInfo  tableInfo;
+  uint8_t        tableType;
+  int16_t        sversion;
+  int16_t        tversion;
   SCMVgroupInfo  vgroupInfo;
   SCMCorVgroupInfo  corVgroupInfo;
-  int32_t       sid;       // the index of one table in a virtual node
-  uint64_t      uid;       // unique id of a table
-  SSchema       schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
+  STableId       id;
+  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
 } STableMeta;
 
 typedef struct STableMetaInfo {
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 9dd33e03cb..85cff4ba17 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -430,7 +430,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   pRes->code = code;
 
   if (code != TSDB_CODE_SUCCESS) {
-    tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code));
+    tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code));
     goto _error;
   } else {
     tscDebug("%p get tableMeta successfully", pSql);
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 72ccd5adc6..17f6c97ea1 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2131,6 +2131,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
 }
 
 bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
+  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo == NULL) {
+    return true;
+  }
+
   STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx);
   
   // required number of results are not reached, continue load data block
diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 80fc82d90b..bf76b8cbe8 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -691,9 +691,15 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   
   pModel = createColumnModel(pSchema, size, capacity);
 
+  int32_t pg = DEFAULT_PAGE_SIZE;
+  int32_t overhead = sizeof(tFilePage);
+  while((pg - overhead) < pModel->rowSize * 2) {
+    pg *= 2;
+  }
+
   size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
   for (int32_t i = 0; i < numOfSubs; ++i) {
-    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
+    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel);
     (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
   }
 
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 95098bbab1..ae2370cd56 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -104,7 +104,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
   } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) {
     // do nothing
   } else if (pToken->type == TK_INTEGER) {
-    useconds = str2int64(pToken->z);
+    useconds = tsosStr2int64(pToken->z);
   } else {
     // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
     if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
@@ -629,8 +629,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
 }
 
 static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
-  pBlocks->tid = pTableMeta->sid;
-  pBlocks->uid = pTableMeta->uid;
+  pBlocks->tid = pTableMeta->id.tid;
+  pBlocks->uid = pTableMeta->id.uid;
   pBlocks->sversion = pTableMeta->sversion;
   pBlocks->numOfRows += numOfRows;
 }
@@ -686,7 +686,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   
   STableDataBlocks *dataBuf = NULL;
-  int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
+  int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE,
                                         sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name,
                                         pTableMeta, &dataBuf);
   if (ret != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index f58431849c..44236face0 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -635,7 +635,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
   int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    if (pTableMetaInfo->pTableMeta->uid == uid) {
+    if (pTableMetaInfo->pTableMeta->id.uid == uid) {
       tableIndex = i;
       break;
     }
@@ -3053,7 +3053,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
-  pLeft->uid = pTableMetaInfo->pTableMeta->uid;
+  pLeft->uid = pTableMetaInfo->pTableMeta->id.uid;
   pLeft->tagColId = pTagSchema1->colId;
   strcpy(pLeft->tableId, pTableMetaInfo->name);
 
@@ -3065,7 +3065,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
-  pRight->uid = pTableMetaInfo->pTableMeta->uid;
+  pRight->uid = pTableMetaInfo->pTableMeta->id.uid;
   pRight->tagColId = pTagSchema2->colId;
   strcpy(pRight->tableId, pTableMetaInfo->name);
 
@@ -3603,7 +3603,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex);
 
   STagCond* pTagCond = &pQueryInfo->tagCond;
-  pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->uid;
+  pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid;
 
   assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN);
 
@@ -3840,7 +3840,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     
     // add to source column list
     STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    int64_t uid = pTableMetaInfo->pTableMeta->uid;
+    int64_t uid = pTableMetaInfo->pTableMeta->id.uid;
     int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
     
     size_t num = taosArrayGetSize(colList);
@@ -4506,8 +4506,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload;
     pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
-    pUpdateMsg->tid       = htonl(pTableMeta->sid);
-    pUpdateMsg->uid       = htobe64(pTableMeta->uid);
+    pUpdateMsg->tid       = htonl(pTableMeta->id.tid);
+    pUpdateMsg->uid       = htobe64(pTableMeta->id.uid);
     pUpdateMsg->colId     = htons(pTagsSchema->colId);
     pUpdateMsg->type      = pTagsSchema->type;
     pUpdateMsg->bytes     = htons(pTagsSchema->bytes);
@@ -4992,6 +4992,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
   pMsg->compression = pCreateDb->compressionLevel;
   pMsg->walLevel = (char)pCreateDb->walLevel;
   pMsg->replications = pCreateDb->replica;
+  pMsg->quorum = pCreateDb->quorum;
   pMsg->ignoreExist = pCreateDb->ignoreExists;
 }
 
@@ -5045,7 +5046,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
 
     if (pExpr->functionId != TSDB_FUNC_TAG) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
       SColumnIndex    index = {.tableIndex = 0, .columnIndex = columnInfo};
       SSchema*        pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
 
@@ -5522,6 +5523,13 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 
+  if (pCreate->quorum != -1 &&
+      (pCreate->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCreate->quorum > TSDB_MAX_DB_REPLICA_OPTION)) {
+    snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum,
+             TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
+  }
+
   int32_t val = htonl(pCreate->daysPerFile);
   if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
     snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val,
diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c
index 9b8f48b109..63607b7a86 100644
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@@ -162,8 +162,8 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
     .numOfColumns = pTableMetaMsg->numOfColumns,
   };
   
-  pTableMeta->sid = pTableMetaMsg->sid;
-  pTableMeta->uid = pTableMetaMsg->uid;
+  pTableMeta->id.tid = pTableMetaMsg->sid;
+  pTableMeta->id.uid = pTableMetaMsg->uid;
   pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
 
   tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 6c1faf6f9d..521280af87 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -605,9 +605,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     } 
 
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
-    pTableIdInfo->tid = htonl(pTableMeta->sid);
-    pTableIdInfo->uid = htobe64(pTableMeta->uid);
-    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
+    pTableIdInfo->tid = htonl(pTableMeta->id.tid);
+    pTableIdInfo->uid = htobe64(pTableMeta->id.uid);
+    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
     pMsg += sizeof(STableIdInfo);
@@ -640,7 +640,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
   }
   
   tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
-      pTableMeta->sid, pTableMeta->uid);
+      pTableMeta->id.tid, pTableMeta->id.uid);
   
   return pMsg;
 }
@@ -714,8 +714,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
     if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
         pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
-      tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
-          pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
+      tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s",
+          pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex,
                pColSchema->name);
 
       return TSDB_CODE_TSC_INVALID_SQL;
@@ -833,8 +833,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
       if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) ||
           (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) {
-        tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
-                 pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns,
+        tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s",
+                 pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, total, numOfTagColumns,
                  pCol->colIndex.columnIndex, pColSchema->name);
 
         return TSDB_CODE_TSC_INVALID_SQL;
@@ -855,7 +855,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
     STagCond* pTagCond = &pQueryInfo->tagCond;
     
-    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid);
+    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
     if (pCond != NULL && pCond->cond != NULL) {
       pQueryMsg->tagCondLen = htons(pCond->len);
       memcpy(pMsg, pCond->cond, pCond->len);
@@ -1739,7 +1739,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
+  tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name);
   free(pTableMeta);
   
   return TSDB_CODE_SUCCESS;
@@ -2215,7 +2215,7 @@ int tscRenewTableMeta(SSqlObj *pSql, char *tableId) {
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   if (pTableMetaInfo->pTableMeta) {
     tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql,
-             tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta);
+             tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta);
   }
 
   taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 8e554d62ff..6e572d94d1 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -241,7 +241,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
-    SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
+    SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
     SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
@@ -334,7 +334,7 @@ void tscSaveSubscriptionProgress(void* sub) {
 
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
-  if (tmkdir(path, 0777) != 0) {
+  if (taosMkDir(path, 0777) != 0) {
     tscError("failed to create subscribe dir: %s", path);
   }
 
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 5d26d09fae..15e02799aa 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -180,7 +180,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
   pSupporter->limit = pQueryInfo->limit;
 
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index);
-  pSupporter->uid = pTableMetaInfo->pTableMeta->uid;
+  pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid;
   assert (pSupporter->uid != 0);
 
   getTmpfilePath("join-", pSupporter->path);
@@ -355,7 +355,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     // set the join condition tag column info, to do extract method
     if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       assert(pQueryInfo->tagCond.joinInfo.hasJoin);
-      int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+      int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
 
       pExpr->param[0].i64Key = colId;
       pExpr->numOfParams = 1;
@@ -499,7 +499,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
   // set the tags value for ts_comp function
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0);
-    int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+    int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid);
     pExpr->param->i64Key = tagColId;
     pExpr->numOfParams = 1;
   }
@@ -560,7 +560,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
   qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid);
+  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
 
   SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
 
@@ -1034,7 +1034,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
     int32_t tableIndexOfSub = -1;
     for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j);
-      if (pTableMetaInfo->pTableMeta->uid == pExpr->uid) {
+      if (pTableMetaInfo->pTableMeta->id.uid == pExpr->uid) {
         tableIndexOfSub = j;
         break;
       }
@@ -1205,7 +1205,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       STagCond* pTagCond = &pSupporter->tagCond;
       assert(pTagCond->joinInfo.hasJoin);
 
-      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->uid);
+      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
       SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
 
       int16_t bytes = 0;
@@ -1237,7 +1237,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0);
 
       if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-        int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid);
+        int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid);
         pExpr->param->i64Key = tagColId;
         pExpr->numOfParams = 1;
       }
@@ -1505,12 +1505,11 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 
   SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql);
 
-  // todo add to async res or not??
   if (pNew == NULL) {
-    tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
-             trsupport->pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex);
+    tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d",
+             trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex);
 
-    pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    pParentSql->res.code = terrno;
     trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
 
     return pParentSql->res.code;
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 5662b4a885..42356108b1 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -148,7 +148,7 @@ void taos_init_imp() {
   refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
   if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "client");
+    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
   }
 
   tscDebug("client is initialized successfully");
@@ -220,7 +220,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 
         if (strlen(tsLocale) == 0) { // locale does not set yet
           char* defaultLocale = setlocale(LC_CTYPE, "");
-          tstrncpy(tsLocale, defaultLocale, sizeof(tsLocale));
+          tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
         }
 
         // set the user specified locale
@@ -234,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
           tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale);
         }
 
-        tstrncpy(tsLocale, locale, sizeof(tsLocale));
+        tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -249,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
               tscInfo("charset changed from %s to %s", tsCharset, charset);
             }
 
-            tstrncpy(tsCharset, charset, sizeof(tsCharset));
+            tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -286,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
             tscInfo("charset changed from %s to %s", tsCharset, pStr);
           }
 
-          tstrncpy(tsCharset, pStr, sizeof(tsCharset));
+          tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscInfo("charset:%s not valid", pStr);
@@ -304,7 +304,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        tstrncpy(tsTimezone, pStr, sizeof(tsTimezone));
+        tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN);
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 17adc0c03d..73d6f0e592 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -356,9 +356,9 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
   if (pObj->signature == pObj) {
-    pthread_mutex_lock(&pObj->mutex);
+    //pthread_mutex_lock(&pObj->mutex);
     tfree(pSql->sqlstr);
-    pthread_mutex_unlock(&pObj->mutex);
+    //pthread_mutex_unlock(&pObj->mutex);
   }
   
   tscFreeSqlResult(pSql);
@@ -955,7 +955,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
   pExpr->interBytes    = interSize;
   
   if (pTableMetaInfo->pTableMeta) {
-    pExpr->uid = pTableMetaInfo->pTableMeta->uid;
+    pExpr->uid = pTableMetaInfo->pTableMeta->id.uid;
   }
   
   return pExpr;
@@ -1482,7 +1482,7 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i
   int32_t k = -1;
 
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-    if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->uid == uid) {
+    if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->id.uid == uid) {
       k = i;
       break;
     }
@@ -1675,6 +1675,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
   if (pNew == NULL) {
     tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1688,6 +1689,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
 
     free(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1706,6 +1708,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1743,6 +1746,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1756,7 +1760,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery
   }
 
-  uint64_t uid = pTableMetaInfo->pTableMeta->uid;
+  uint64_t uid = pTableMetaInfo->pTableMeta->id.uid;
   tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true);
 
   int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo);
@@ -1827,8 +1831,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   if (pFinalInfo->pTableMeta == NULL) {
-    tscError("%p new subquery failed for get tableMeta is NULL from cache", pSql);
+    tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name);
     tscFreeSqlObj(pNew);
+
+    if (pPrevSql != NULL) {
+      assert(pPrevSql->res.code != TSDB_CODE_SUCCESS);
+      terrno = pPrevSql->res.code;
+    } else {
+      terrno = TSDB_CODE_TSC_APP_ERROR;
+    }
+
     return NULL;
   }
   
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 57a32f3d69..e856481a9a 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -1,12 +1,11 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(common ${SRC})
-  TARGET_LINK_LIBRARIES(common tutil)
-ENDIF ()
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(inc)
+
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(common ${SRC})
+TARGET_LINK_LIBRARIES(common tutil)
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index b9afef65d9..b8d7297ee0 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -43,9 +43,9 @@ extern uint32_t tsMaxTmrCtrl;
 extern float    tsNumOfThreadsPerCore;
 extern float    tsRatioOfQueryThreads;
 extern int8_t   tsDaylight;
-extern char     tsTimezone[64];
-extern char     tsLocale[64];
-extern char     tsCharset[64];  // default encode string
+extern char     tsTimezone[];
+extern char     tsLocale[];
+extern char     tsCharset[];  // default encode string
 extern int32_t  tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
 
@@ -80,6 +80,7 @@ extern int16_t tsCompression;
 extern int16_t tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
+extern int32_t tsQuorum;
 
 // balance
 extern int32_t tsEnableBalance;
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 08349374b1..79ea2b2d46 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -50,7 +50,7 @@ int32_t tsShellActivityTimer = 3;  // second
 float   tsNumOfThreadsPerCore = 1.0;
 float   tsRatioOfQueryThreads = 0.5;
 int8_t  tsDaylight = 0;
-char    tsTimezone[64] = {0};
+char    tsTimezone[TSDB_TIMEZONE_LEN] = {0};
 char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int32_t tsEnableCoreFile = 0;
@@ -109,6 +109,7 @@ int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
 int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
+int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = 100;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -742,6 +743,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "quorum";
+  cfg.ptr = &tsQuorum;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
+  cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "mqttBrokerAddress";
   cfg.ptr = tsMqttBrokerAddress;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
@@ -1338,4 +1349,4 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d
   }
 
   return true;
-}
\ No newline at end of file
+}
diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt
index e8796306f3..4ccdb765c3 100644
--- a/src/cq/CMakeLists.txt
+++ b/src/cq/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(inc)
diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt
index 99c729dff4..82781f531a 100644
--- a/src/cq/test/CMakeLists.txt
+++ b/src/cq/test/CMakeLists.txt
@@ -1,17 +1,12 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
-
-  LIST(APPEND CQTEST_SRC ./cqtest.c)
-  ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
-  TARGET_LINK_LIBRARIES(cqtest tcq)
-
-ENDIF ()
-
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/cq/inc)
 
+LIST(APPEND CQTEST_SRC ./cqtest.c)
+ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
+TARGET_LINK_LIBRARIES(cqtest tcq)
diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt
index de6e15e6b9..f7c2961352 100644
--- a/src/dnode/CMakeLists.txt
+++ b/src/dnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
@@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync)
 
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)
@@ -26,10 +26,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     TARGET_LINK_LIBRARIES(taosd grant)
   ENDIF ()
 
-  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
   SET(PREPARE_ENV_CMD "prepare_env_cmd")
   SET(PREPARE_ENV_TARGET "prepare_env_target")
   ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 9050b9c582..ec35475d73 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -492,6 +492,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
   pCfg->moduleStatus = htonl(pCfg->moduleStatus);
   pCfg->dnodeId      = htonl(pCfg->dnodeId);
+  pCfg->clusterId    = htonl(pCfg->clusterId);
 
   for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
     SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
@@ -697,6 +698,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   //strcpy(pStatus->dnodeName, tsDnodeName);
   pStatus->version          = htonl(tsVersion);
   pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
+  pStatus->clusterId        = htonl(tsDnodeCfg.clusterId);
   strcpy(pStatus->dnodeEp, tsLocalEp);
   pStatus->lastReboot       = htonl(tsRebootTime);
   pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
@@ -767,6 +769,13 @@ static bool dnodeReadDnodeCfg() {
   }
   tsDnodeCfg.dnodeId = dnodeId->valueint;
 
+  cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_Number) {
+    dError("failed to read dnodeCfg.json, clusterId not found");
+    goto PARSE_CFG_OVER;
+  }
+  tsDnodeCfg.clusterId = clusterId->valueint;
+
   ret = true;
 
   dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
@@ -790,7 +799,8 @@ static void dnodeSaveDnodeCfg() {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d\n", tsDnodeCfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": %d\n", tsDnodeCfg.clusterId);
   len += snprintf(content + len, maxLen - len, "}\n"); 
 
   fwrite(content, 1, len, fp);
@@ -803,8 +813,9 @@ static void dnodeSaveDnodeCfg() {
 
 void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
   if (tsDnodeCfg.dnodeId == 0) {
-    dInfo("dnodeId is set to %d", pCfg->dnodeId);  
+    dInfo("dnodeId is set to %d, clusterId is set to %d", pCfg->dnodeId, pCfg->clusterId);  
     tsDnodeCfg.dnodeId = pCfg->dnodeId;
+    tsDnodeCfg.clusterId = pCfg->clusterId;
     dnodeSaveDnodeCfg();
   }
 }
diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c
index 901e0061e9..2519684878 100644
--- a/src/dnode/src/dnodeSystem.c
+++ b/src/dnode/src/dnodeSystem.c
@@ -20,7 +20,6 @@
 #include "tglobal.h"
 #include "dnodeInt.h"
 #include "dnodeMain.h"
-#include "tfile.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static sem_t exitSem;
@@ -40,7 +39,7 @@ int32_t main(int32_t argc, char *argv[]) {
         exit(EXIT_FAILURE);
       }
     } else if (strcmp(argv[i], "-V") == 0) {
-#ifdef _SYNC
+#ifdef _ACCT
       char *versionStr = "enterprise";
 #else
       char *versionStr = "community";
diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c
index bbea1a5e0b..cb53bb5e60 100644
--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@@ -49,7 +49,7 @@ static taos_qset       readQset;
 int32_t dnodeInitVnodeRead() {
   readQset = taosOpenQset();
 
-  readPool.min = 2;
+  readPool.min = tsNumOfCores;
   readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
   if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
   readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
@@ -206,10 +206,14 @@ static void *dnodeProcessReadQueue(void *param) {
            taosMsg[pReadMsg->rpcMsg.msgType], type);
     int32_t code = vnodeProcessRead(pVnode, pReadMsg);
 
-    if (type == TAOS_QTYPE_RPC) {
+    if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
       dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
     } else {
-      dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      if (code == TSDB_CODE_QRY_HAS_RSP) {
+        dnodeSendRpcReadRsp(pVnode, pReadMsg, TSDB_CODE_SUCCESS);
+      } else {
+        dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      }
     }
 
     taosFreeQitem(pReadMsg);
diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index dc09a03e14..51bc8890fc 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -38,6 +38,8 @@ typedef struct {
 
 typedef struct {
   SRspRet  rspRet;
+  int32_t  processedCount;
+  int32_t  code;
   void    *pCont;
   int32_t  contLen;
   SRpcMsg  rpcMsg;
@@ -187,13 +189,16 @@ void dnodeFreeVnodeWqueue(void *wqueue) {
 void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
   SWriteMsg *pWrite = (SWriteMsg *)param;
 
-  if (code > 0) return;
+  if (code < 0) pWrite->code = code;
+  int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
+
+  if (count <= 1) return;
 
   SRpcMsg rpcRsp = {
     .handle  = pWrite->rpcMsg.handle,
     .pCont   = pWrite->rspRet.rsp,
     .contLen = pWrite->rspRet.len,
-    .code    = code,
+    .code    = pWrite->code,
   };
 
   rpcSendResponse(&rpcRsp);
@@ -239,7 +244,10 @@ static void *dnodeProcessWriteQueue(void *param) {
       }
 
       int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
-      if (pWrite) pWrite->rpcMsg.code = code;
+      if (pWrite) { 
+        pWrite->rpcMsg.code = code;
+        if (code <= 0) pWrite->processedCount = 1; 
+      }
     }
 
     walFsync(vnodeGetWal(pVnode));
diff --git a/src/inc/query.h b/src/inc/query.h
index d201b649f9..ec1e458b62 100644
--- a/src/inc/query.h
+++ b/src/inc/query.h
@@ -28,7 +28,7 @@ typedef void* qinfo_t;
  * @param qinfo
  * @return
  */
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo);
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo);
 
 
 /**
@@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs
  * @param qinfo
  * @return
  */
-void qTableQuery(qinfo_t qinfo);
+bool qTableQuery(qinfo_t qinfo);
 
 /**
  * Retrieve the produced results information, if current query is not paused or completed,
@@ -48,7 +48,7 @@ void qTableQuery(qinfo_t qinfo);
  * @param qinfo
  * @return
  */
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext);
 
 /**
  *
@@ -60,16 +60,14 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
  * @param contLen payload length
  * @return
  */
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
 
 /**
- * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
- * so it can be only called once for each retrieve
  *
  * @param qinfo
  * @return
  */
-bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
+void* qGetResultRetrieveMsg(qinfo_t qinfo);
 
 /**
  * kill current ongoing query and free query handle automatically
@@ -89,7 +87,7 @@ void  qQueryMgmtNotifyClosed(void* pExecutor);
 void  qCleanupQueryMgmt(void* pExecutor);
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo);
 void** qAcquireQInfo(void* pMgmt, uint64_t key);
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree);
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle);
 
 #ifdef __cplusplus
 }
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 30523dc25c..c70ca8662e 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -250,7 +250,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_STATE_LEN            20
 #define TSDB_COUNTRY_LEN          20
 #define TSDB_LOCALE_LEN           64
-#define TSDB_TIMEZONE_LEN         64
+#define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
 
 #define TSDB_FQDN_LEN             128
@@ -338,6 +338,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MIN_DB_REPLICA_OPTION      1
 #define TSDB_MAX_DB_REPLICA_OPTION      3
 #define TSDB_DEFAULT_DB_REPLICA_OPTION  1
+#define TSDB_DEFAULT_DB_QUORUM_OPTION   1
 
 #define TSDB_MAX_JOIN_TABLE_NUM         5
 #define TSDB_MAX_UNION_CLAUSE           5
@@ -377,6 +378,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_ORDER_ASC   1
 #define TSDB_ORDER_DESC  2
 
+#define TSDB_DEFAULT_CLUSTER_HASH_SIZE  1
 #define TSDB_DEFAULT_MNODES_HASH_SIZE   5
 #define TSDB_DEFAULT_DNODES_HASH_SIZE   10
 #define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index c440fd3c06..bb3a6ee78e 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -131,6 +131,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED,          0, 0x0337, "balance al
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE,      0, 0x0338, "vgroup not in dnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  0, 0x0339, "vgroup already in dnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE,           0, 0x033A, "dnode not avaliable")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       0, 0x033B, "cluster id not match")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       0, 0x0340, "mnode accounts already exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT,             0, 0x0341, "mnode invalid account")
@@ -215,6 +216,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "query out
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "query app error")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "query duplicated join key")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT,        0, 0x0706, "query tag conditon too many")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY,                0, 0x0707, "query not ready")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP,                  0, 0x0708, "query should response")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "grant expired")
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index e095567d75..edddc717c0 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -139,6 +139,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_GRANTS,
   TSDB_MGMT_TABLE_VNODES,
   TSDB_MGMT_TABLE_STREAMTABLES,
+  TSDB_MGMT_TABLE_CLUSTER,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -521,6 +522,7 @@ typedef struct {
   int8_t   compression;
   int8_t   walLevel;
   int8_t   replications;
+  int8_t   quorum;
   int8_t   ignoreExist;
 } SCMCreateDbMsg, SCMAlterDbMsg;
 
@@ -546,6 +548,7 @@ typedef struct {
 
 typedef struct {
   int32_t  dnodeId;
+  int32_t  clusterId;
   uint32_t moduleStatus;
   uint32_t numOfVnodes;
   uint32_t reserved;
@@ -586,6 +589,7 @@ typedef struct {
   uint16_t    openVnodes;
   uint16_t    numOfCores;
   float       diskAvailable;  // GB
+  int32_t     clusterId;
   uint8_t     alternativeRole;
   uint8_t     reserve2[15];
   SClusterCfg clusterCfg;
diff --git a/src/inc/tsync.h b/src/inc/tsync.h
index 65b91d87e4..2dfac7ec32 100644
--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@@ -26,6 +26,7 @@ extern "C" {
 typedef enum _TAOS_SYNC_ROLE {
   TAOS_SYNC_ROLE_OFFLINE,
   TAOS_SYNC_ROLE_UNSYNCED,
+  TAOS_SYNC_ROLE_SYNCING,
   TAOS_SYNC_ROLE_SLAVE,
   TAOS_SYNC_ROLE_MASTER,
 } ESyncRole;
@@ -78,6 +79,9 @@ typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
 // when role is changed, call this to notify app
 typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
 
+// if a number of retrieving data failed, call this to start flow control 
+typedef void     (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
+
 // when data file is synced successfully, notity app
 typedef int      (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
 
@@ -93,6 +97,7 @@ typedef struct {
   FWriteToCache   writeToCache;
   FConfirmForward confirmForward;
   FNotifyRole     notifyRole;
+  FNotifyFlowCtrl notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
 } SSyncInfo;
 
diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index 0b63e9e71d..e77f5741d3 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -104,123 +104,125 @@
 #define TK_MAXTABLES                       86
 #define TK_CACHE                           87
 #define TK_REPLICA                         88
-#define TK_DAYS                            89
-#define TK_MINROWS                         90
-#define TK_MAXROWS                         91
-#define TK_BLOCKS                          92
-#define TK_CTIME                           93
-#define TK_WAL                             94
-#define TK_FSYNC                           95
-#define TK_COMP                            96
-#define TK_PRECISION                       97
-#define TK_LP                              98
-#define TK_RP                              99
-#define TK_TAGS                           100
-#define TK_USING                          101
-#define TK_AS                             102
-#define TK_COMMA                          103
-#define TK_NULL                           104
-#define TK_SELECT                         105
-#define TK_UNION                          106
-#define TK_ALL                            107
-#define TK_FROM                           108
-#define TK_VARIABLE                       109
-#define TK_INTERVAL                       110
-#define TK_FILL                           111
-#define TK_SLIDING                        112
-#define TK_ORDER                          113
-#define TK_BY                             114
-#define TK_ASC                            115
-#define TK_DESC                           116
-#define TK_GROUP                          117
-#define TK_HAVING                         118
-#define TK_LIMIT                          119
-#define TK_OFFSET                         120
-#define TK_SLIMIT                         121
-#define TK_SOFFSET                        122
-#define TK_WHERE                          123
-#define TK_NOW                            124
-#define TK_RESET                          125
-#define TK_QUERY                          126
-#define TK_ADD                            127
-#define TK_COLUMN                         128
-#define TK_TAG                            129
-#define TK_CHANGE                         130
-#define TK_SET                            131
-#define TK_KILL                           132
-#define TK_CONNECTION                     133
-#define TK_STREAM                         134
-#define TK_COLON                          135
-#define TK_ABORT                          136
-#define TK_AFTER                          137
-#define TK_ATTACH                         138
-#define TK_BEFORE                         139
-#define TK_BEGIN                          140
-#define TK_CASCADE                        141
-#define TK_CLUSTER                        142
-#define TK_CONFLICT                       143
-#define TK_COPY                           144
-#define TK_DEFERRED                       145
-#define TK_DELIMITERS                     146
-#define TK_DETACH                         147
-#define TK_EACH                           148
-#define TK_END                            149
-#define TK_EXPLAIN                        150
-#define TK_FAIL                           151
-#define TK_FOR                            152
-#define TK_IGNORE                         153
-#define TK_IMMEDIATE                      154
-#define TK_INITIALLY                      155
-#define TK_INSTEAD                        156
-#define TK_MATCH                          157
-#define TK_KEY                            158
-#define TK_OF                             159
-#define TK_RAISE                          160
-#define TK_REPLACE                        161
-#define TK_RESTRICT                       162
-#define TK_ROW                            163
-#define TK_STATEMENT                      164
-#define TK_TRIGGER                        165
-#define TK_VIEW                           166
-#define TK_COUNT                          167
-#define TK_SUM                            168
-#define TK_AVG                            169
-#define TK_MIN                            170
-#define TK_MAX                            171
-#define TK_FIRST                          172
-#define TK_LAST                           173
-#define TK_TOP                            174
-#define TK_BOTTOM                         175
-#define TK_STDDEV                         176
-#define TK_PERCENTILE                     177
-#define TK_APERCENTILE                    178
-#define TK_LEASTSQUARES                   179
-#define TK_HISTOGRAM                      180
-#define TK_DIFF                           181
-#define TK_SPREAD                         182
-#define TK_TWA                            183
-#define TK_INTERP                         184
-#define TK_LAST_ROW                       185
-#define TK_RATE                           186
-#define TK_IRATE                          187
-#define TK_SUM_RATE                       188
-#define TK_SUM_IRATE                      189
-#define TK_AVG_RATE                       190
-#define TK_AVG_IRATE                      191
-#define TK_TBID                           192
-#define TK_SEMI                           193
-#define TK_NONE                           194
-#define TK_PREV                           195
-#define TK_LINEAR                         196
-#define TK_IMPORT                         197
-#define TK_METRIC                         198
-#define TK_TBNAME                         199
-#define TK_JOIN                           200
-#define TK_METRICS                        201
-#define TK_STABLE                         202
-#define TK_INSERT                         203
-#define TK_INTO                           204
-#define TK_VALUES                         205
+#define TK_QUORUM                          89
+#define TK_DAYS                            90
+#define TK_MINROWS                         91
+#define TK_MAXROWS                         92
+#define TK_BLOCKS                          93
+#define TK_CTIME                           94
+#define TK_WAL                             95
+#define TK_FSYNC                           96
+#define TK_COMP                            97
+#define TK_PRECISION                       98
+#define TK_LP                              99
+#define TK_RP                             100
+#define TK_TAGS                           101
+#define TK_USING                          102
+#define TK_AS                             103
+#define TK_COMMA                          104
+#define TK_NULL                           105
+#define TK_SELECT                         106
+#define TK_UNION                          107
+#define TK_ALL                            108
+#define TK_FROM                           109
+#define TK_VARIABLE                       110
+#define TK_INTERVAL                       111
+#define TK_FILL                           112
+#define TK_SLIDING                        113
+#define TK_ORDER                          114
+#define TK_BY                             115
+#define TK_ASC                            116
+#define TK_DESC                           117
+#define TK_GROUP                          118
+#define TK_HAVING                         119
+#define TK_LIMIT                          120
+#define TK_OFFSET                         121
+#define TK_SLIMIT                         122
+#define TK_SOFFSET                        123
+#define TK_WHERE                          124
+#define TK_NOW                            125
+#define TK_RESET                          126
+#define TK_QUERY                          127
+#define TK_ADD                            128
+#define TK_COLUMN                         129
+#define TK_TAG                            130
+#define TK_CHANGE                         131
+#define TK_SET                            132
+#define TK_KILL                           133
+#define TK_CONNECTION                     134
+#define TK_STREAM                         135
+#define TK_COLON                          136
+#define TK_ABORT                          137
+#define TK_AFTER                          138
+#define TK_ATTACH                         139
+#define TK_BEFORE                         140
+#define TK_BEGIN                          141
+#define TK_CASCADE                        142
+#define TK_CLUSTER                        143
+#define TK_CONFLICT                       144
+#define TK_COPY                           145
+#define TK_DEFERRED                       146
+#define TK_DELIMITERS                     147
+#define TK_DETACH                         148
+#define TK_EACH                           149
+#define TK_END                            150
+#define TK_EXPLAIN                        151
+#define TK_FAIL                           152
+#define TK_FOR                            153
+#define TK_IGNORE                         154
+#define TK_IMMEDIATE                      155
+#define TK_INITIALLY                      156
+#define TK_INSTEAD                        157
+#define TK_MATCH                          158
+#define TK_KEY                            159
+#define TK_OF                             160
+#define TK_RAISE                          161
+#define TK_REPLACE                        162
+#define TK_RESTRICT                       163
+#define TK_ROW                            164
+#define TK_STATEMENT                      165
+#define TK_TRIGGER                        166
+#define TK_VIEW                           167
+#define TK_COUNT                          168
+#define TK_SUM                            169
+#define TK_AVG                            170
+#define TK_MIN                            171
+#define TK_MAX                            172
+#define TK_FIRST                          173
+#define TK_LAST                           174
+#define TK_TOP                            175
+#define TK_BOTTOM                         176
+#define TK_STDDEV                         177
+#define TK_PERCENTILE                     178
+#define TK_APERCENTILE                    179
+#define TK_LEASTSQUARES                   180
+#define TK_HISTOGRAM                      181
+#define TK_DIFF                           182
+#define TK_SPREAD                         183
+#define TK_TWA                            184
+#define TK_INTERP                         185
+#define TK_LAST_ROW                       186
+#define TK_RATE                           187
+#define TK_IRATE                          188
+#define TK_SUM_RATE                       189
+#define TK_SUM_IRATE                      190
+#define TK_AVG_RATE                       191
+#define TK_AVG_IRATE                      192
+#define TK_TBID                           193
+#define TK_SEMI                           194
+#define TK_NONE                           195
+#define TK_PREV                           196
+#define TK_LINEAR                         197
+#define TK_IMPORT                         198
+#define TK_METRIC                         199
+#define TK_TBNAME                         200
+#define TK_JOIN                           201
+#define TK_METRICS                        202
+#define TK_STABLE                         203
+#define TK_INSERT                         204
+#define TK_INTO                           205
+#define TK_VALUES                         206
+
 
 #define TK_SPACE                          300
 #define TK_COMMENT                        301
diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt
index 9854a44c3b..01c5f5ea3a 100644
--- a/src/kit/shell/CMakeLists.txt
+++ b/src/kit/shell/CMakeLists.txt
@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt
index 627092a85b..a1593ce5a1 100644
--- a/src/kit/taosdemo/CMakeLists.txt
+++ b/src/kit/taosdemo/CMakeLists.txt
@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt
index c63168237c..2a14dbfb4a 100644
--- a/src/kit/taosdump/CMakeLists.txt
+++ b/src/kit/taosdump/CMakeLists.txt
@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c
index a1b1998298..7ae209bbdf 100644
--- a/src/kit/taosdump/taosdump.c
+++ b/src/kit/taosdump/taosdump.c
@@ -545,7 +545,7 @@ int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric
   strcpy(tableRecord.name, meter);
   strcpy(tableRecord.metric, metric);
 
-  twrite(*fd, &tableRecord, sizeof(STableRecord));
+  taosTWrite(*fd, &tableRecord, sizeof(STableRecord));
   return 0;
 }
 
@@ -600,7 +600,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu
     strncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
     strcpy(tableRecord.metric, metric);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 
@@ -1196,7 +1196,7 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
   while ((row = taos_fetch_row(tmpResult)) != NULL) {  
     memset(&tableRecord, 0, sizeof(STableRecord));
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
   }  
   
   taos_free_result(tmpResult);
@@ -1282,7 +1282,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
     strncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 
diff --git a/src/kit/taosmigrate/CMakeLists.txt b/src/kit/taosmigrate/CMakeLists.txt
index 85b2f33f01..86b428c61b 100644
--- a/src/kit/taosmigrate/CMakeLists.txt
+++ b/src/kit/taosmigrate/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) 
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/inc)
diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt
index 2e975f089c..f5b9fbf747 100644
--- a/src/mnode/CMakeLists.txt
+++ b/src/mnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h
new file mode 100644
index 0000000000..3be5d601af
--- /dev/null
+++ b/src/mnode/inc/mnodeCluster.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_CLUSTER_H
+#define TDENGINE_MNODE_CLUSTER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct SClusterObj;
+
+int32_t mnodeInitCluster();
+void    mnodeCleanupCluster();
+int32_t mnodeGetClusterId();
+void    mnodeUpdateClusterId();
+void *  mnodeGetCluster(int32_t clusterId);
+void *  mnodeGetNextCluster(void *pIter, struct SClusterObj **pCluster);
+void    mnodeIncClusterRef(struct SClusterObj *pCluster);
+void    mnodeDecClusterRef(struct SClusterObj *pCluster);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index a454f413f0..02848089cb 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -36,6 +36,14 @@ struct define notes:
 3. The fields behind the updataEnd field can be changed;
 */
 
+typedef struct SClusterObj {
+  int32_t clusterId;
+  int64_t createdTime;
+  int8_t  reserved[36];
+  int8_t  updateEnd[4];
+  int32_t refCount;
+} SClusterObj;
+
 typedef struct SDnodeObj {
   int32_t    dnodeId;
   int32_t    openVnodes;
@@ -163,6 +171,7 @@ typedef struct {
   int8_t  compression;
   int8_t  walLevel;
   int8_t  replications;
+  int8_t  quorum;
   int8_t  reserved[12];
 } SDbCfg;
 
diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h
index 0c47f684f8..cadd1b1f3d 100644
--- a/src/mnode/inc/mnodeSdb.h
+++ b/src/mnode/inc/mnodeSdb.h
@@ -23,15 +23,16 @@ extern "C" {
 struct SMnodeMsg;
 
 typedef enum {
-  SDB_TABLE_DNODE   = 0,
-  SDB_TABLE_MNODE   = 1,
-  SDB_TABLE_ACCOUNT = 2,
-  SDB_TABLE_USER    = 3,
-  SDB_TABLE_DB      = 4,
-  SDB_TABLE_VGROUP  = 5,
-  SDB_TABLE_STABLE  = 6,
-  SDB_TABLE_CTABLE  = 7,
-  SDB_TABLE_MAX     = 8
+  SDB_TABLE_CLUSTER = 0,
+  SDB_TABLE_DNODE   = 1,
+  SDB_TABLE_MNODE   = 2,
+  SDB_TABLE_ACCOUNT = 3,
+  SDB_TABLE_USER    = 4,
+  SDB_TABLE_DB      = 5,
+  SDB_TABLE_VGROUP  = 6,
+  SDB_TABLE_STABLE  = 7,
+  SDB_TABLE_CTABLE  = 8,
+  SDB_TABLE_MAX     = 9
 } ESdbTable;
 
 typedef enum {
diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c
index 2d6e1ae007..287e8bb723 100644
--- a/src/mnode/src/mnodeAcct.c
+++ b/src/mnode/src/mnodeAcct.c
@@ -64,7 +64,7 @@ static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeActionActionEncode(SSdbOper *pOper) {
+static int32_t mnodeAcctActionEncode(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
   memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
   pOper->rowSize = tsAcctUpdateSize;
@@ -109,7 +109,7 @@ int32_t mnodeInitAccts() {
     .insertFp     = mnodeAcctActionInsert,
     .deleteFp     = mnodeAcctActionDelete,
     .updateFp     = mnodeAcctActionUpdate,
-    .encodeFp     = mnodeActionActionEncode,
+    .encodeFp     = mnodeAcctActionEncode,
     .decodeFp     = mnodeAcctActionDecode,
     .destroyFp    = mnodeAcctActionDestroy,
     .restoredFp   = mnodeAcctActionRestored
diff --git a/src/mnode/src/mnodeBalance.c b/src/mnode/src/mnodeBalance.c
deleted file mode 100644
index 95a97bd055..0000000000
--- a/src/mnode/src/mnodeBalance.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tglobal.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
-#include "mnodeSdb.h"
-
-#ifndef _SYNC
-
-int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
-void    balanceCleanUp() {}
-void    balanceAsyncNotify() {}
-void    balanceSyncNotify() {}
-void    balanceReset() {}
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
-
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1000.0;
-
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
-      float openVnodes = pDnode->openVnodes;
-      if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
-
-      float usage = openVnodes / pDnode->numOfCores;
-      if (usage <= vnodeUsage) {
-        pSelDnode = pDnode;
-        vnodeUsage = usage;
-      }
-    }
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  sdbFreeIter(pIter);
-
-  if (pSelDnode == NULL) {
-    mError("failed to alloc vnode to vgroup");
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  }
-
-  pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
-  pVgroup->vnodeGid[0].pDnode = pSelDnode;
-
-  mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
-  return TSDB_CODE_SUCCESS;
-}
-
-#endif 
diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c
new file mode 100644
index 0000000000..8933107dd1
--- /dev/null
+++ b/src/mnode/src/mnodeCluster.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "ttime.h"
+#include "dnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeCluster.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "tglobal.h"
+
+static void *  tsClusterSdb = NULL;
+static int32_t tsClusterUpdateSize;
+static int32_t tsClusterId;
+static int32_t mnodeCreateCluster();
+
+static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) {
+  tfree(pOper->pObj);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionInsert(SSdbOper *pOper) {
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionDelete(SSdbOper *pOper) {
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionUpdate(SSdbOper *pOper) {
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionEncode(SSdbOper *pOper) {
+  SClusterObj *pCluster = pOper->pObj;
+  memcpy(pOper->rowData, pCluster, tsClusterUpdateSize);
+  pOper->rowSize = tsClusterUpdateSize;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionDecode(SSdbOper *pOper) {
+  SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj));
+  if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
+
+  memcpy(pCluster, pOper->rowData, tsClusterUpdateSize);
+  pOper->pObj = pCluster;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeClusterActionRestored() {
+  int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb);
+  if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
+    mInfo("dnode first deploy, create cluster");
+    int32_t code = mnodeCreateCluster();
+    if (code != TSDB_CODE_SUCCESS) {
+      mError("failed to create cluster, reason:%s", tstrerror(code));
+      return code;
+    }
+  }
+
+  mnodeUpdateClusterId();
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mnodeInitCluster() {
+  SClusterObj tObj;
+  tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+
+  SSdbTableDesc tableDesc = {
+    .tableId      = SDB_TABLE_CLUSTER,
+    .tableName    = "cluster",
+    .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE,
+    .maxRowSize   = tsClusterUpdateSize,
+    .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
+    .keyType      = SDB_KEY_INT,
+    .insertFp     = mnodeClusterActionInsert,
+    .deleteFp     = mnodeClusterActionDelete,
+    .updateFp     = mnodeClusterActionUpdate,
+    .encodeFp     = mnodeClusterActionEncode,
+    .decodeFp     = mnodeClusterActionDecode,
+    .destroyFp    = mnodeClusterActionDestroy,
+    .restoredFp   = mnodeClusterActionRestored
+  };
+
+  tsClusterSdb = sdbOpenTable(&tableDesc);
+  if (tsClusterSdb == NULL) {
+    mError("table:%s, failed to create hash", tableDesc.tableName);
+    return -1;
+  }
+
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters);
+
+  mDebug("table:%s, hash is created", tableDesc.tableName);
+  return TSDB_CODE_SUCCESS;
+}
+
+void mnodeCleanupCluster() {
+  sdbCloseTable(tsClusterSdb);
+  tsClusterSdb = NULL;
+}
+
+void *mnodeGetCluster(int32_t clusterId) {
+  return sdbGetRow(tsClusterSdb, &clusterId);
+}
+
+void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) {
+  return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); 
+}
+
+void mnodeIncClusterRef(SClusterObj *pCluster) {
+  sdbIncRef(tsClusterSdb, pCluster);
+}
+
+void mnodeDecClusterRef(SClusterObj *pCluster) {
+  sdbDecRef(tsClusterSdb, pCluster);
+}
+
+static int32_t mnodeCreateCluster() {
+  int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb);
+  if (numOfClusters != 0) return TSDB_CODE_SUCCESS;
+
+  SClusterObj *pCluster = malloc(sizeof(SClusterObj));
+  memset(pCluster, 0, sizeof(SClusterObj));
+  pCluster->createdTime = taosGetTimestampMs();
+  pCluster->clusterId = labs((pCluster->createdTime >> 32) & (pCluster->createdTime)) | (*(int32_t*)tsFirst);
+  
+  SSdbOper oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsClusterSdb,
+    .pObj = pCluster,
+  };
+
+  return sdbInsertRow(&oper);
+}
+
+int32_t mnodeGetClusterId() {
+  return tsClusterId;
+}
+
+void mnodeUpdateClusterId() {
+  SClusterObj *pCluster = NULL;
+  mnodeGetNextCluster(NULL, &pCluster);
+  if (pCluster != NULL) {
+    tsClusterId = pCluster->clusterId;
+    mnodeDecClusterRef(pCluster);
+    mInfo("cluster id is %d", tsClusterId);
+  } else {
+    //assert(false);
+  }
+}
+
+
+static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  int32_t cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "clusterId");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 8;
+  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
+  strcpy(pSchema[cols].name, "create_time");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  strcpy(pMeta->tableId, "show cluster");
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = 1;
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+
+  return 0;
+}
+
+static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t numOfRows = 0;
+  int32_t cols = 0;
+  char *  pWrite;
+  SClusterObj *pCluster = NULL;
+
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster);
+    if (pCluster == NULL) break;
+    
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *) pWrite = pCluster->clusterId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *) pWrite = pCluster->createdTime;
+    cols++;
+
+    mnodeDecClusterRef(pCluster);
+    numOfRows++;
+  }
+
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index a159e98ed5..20d1ce3bab 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -301,12 +301,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
-#ifndef _SYNC
-  if (pCfg->replications != 1) {
-    mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
+  if (pCfg->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCfg->quorum > TSDB_MAX_DB_REPLICA_OPTION) {
+    mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_REPLICA_OPTION,
+           TSDB_MAX_DB_REPLICA_OPTION);
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
-#endif
 
   return TSDB_CODE_SUCCESS;
 }
@@ -327,6 +326,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->compression < 0) pCfg->compression = tsCompression;
   if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
+  if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
 }
 
 static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -376,7 +376,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs
     .precision           = pCreate->precision,
     .compression         = pCreate->compression,
     .walLevel            = pCreate->walLevel,
-    .replications        = pCreate->replications
+    .replications        = pCreate->replications,
+    .quorum              = pCreate->quorum
   };
 
   mnodeSetDefaultDbCfg(&pDb->cfg);
@@ -515,6 +516,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
+    pShow->bytes[cols] = 2;
+    pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+    strcpy(pSchema[cols].name, "quorum");
+    pSchema[cols].bytes = htons(pShow->bytes[cols]);
+    cols++;
+
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
     strcpy(pSchema[cols].name, "days");
@@ -661,6 +668,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
       *(int16_t *)pWrite = pDb->cfg.replications;
       cols++;
 
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      *(int16_t *)pWrite = pDb->cfg.quorum;
+      cols++;
+
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int16_t *)pWrite = pDb->cfg.daysPerFile;
       cols++;
@@ -810,6 +821,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   int8_t  compression    = pAlter->compression;
   int8_t  walLevel       = pAlter->walLevel;
   int8_t  replications   = pAlter->replications;
+  int8_t  quorum         = pAlter->quorum;
   int8_t  precision      = pAlter->precision;
   
   terrno = TSDB_CODE_SUCCESS;
@@ -908,6 +920,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     }
   }
 
+  if (quorum >= 0 && quorum != pDb->cfg.quorum) {
+    mDebug("db:%s, quorum:%d change to %d", pDb->name, pDb->cfg.quorum, quorum);
+    newCfg.compression = quorum;
+  }
+
   return newCfg;
 }
 
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 687bb1873b..aa05eade24 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -37,6 +37,7 @@
 #include "mnodeVgroup.h"
 #include "mnodeWrite.h"
 #include "mnodePeer.h"
+#include "mnodeCluster.h"
 
 int32_t tsAccessSquence = 0;
 static void   *tsDnodeSdb = NULL;
@@ -78,9 +79,6 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
-#ifndef _SYNC 
-  mnodeDropAllDnodeVgroups(pDnode);
-#endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
   balanceAsyncNotify();
 
@@ -355,6 +353,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pStatus->moduleStatus = htonl(pStatus->moduleStatus);
   pStatus->lastReboot   = htonl(pStatus->lastReboot);
   pStatus->numOfCores   = htons(pStatus->numOfCores);
+  pStatus->clusterId    = htonl(pStatus->clusterId);
   
   uint32_t version = htonl(pStatus->version);
   if (version != tsVersion) {
@@ -382,13 +381,19 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pDnode->diskAvailable    = pStatus->diskAvailable;
   pDnode->alternativeRole  = pStatus->alternativeRole;
   pDnode->moduleStatus     = pStatus->moduleStatus;
-  
+
   if (pStatus->dnodeId == 0) {
-    mDebug("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp);
+    mDebug("dnode:%d %s, first access, set clusterId %d", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId());
   } else {
-    mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
+    if (pStatus->clusterId != mnodeGetClusterId()) {
+      mError("dnode:%d, input clusterId %d not match with exist %d", pDnode->dnodeId, pStatus->clusterId,
+             mnodeGetClusterId());
+      return TSDB_CODE_MND_INVALID_CLUSTER_ID;
+    } else {
+      mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
+    }
   }
- 
+
   int32_t openVnodes = htons(pStatus->openVnodes);
   int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
   SDMStatusRsp *pRsp = rpcMallocCont(contLen);
@@ -400,6 +405,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
   pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
   pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
+  pRsp->dnodeCfg.clusterId = htonl(mnodeGetClusterId());
   SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
 
   for (int32_t j = 0; j < openVnodes; ++j) {
@@ -543,12 +549,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 
   mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
 
-#ifndef _SYNC
-  int32_t code = mnodeDropDnode(pDnode, pMsg);
-#else
   int32_t code = balanceDropDnode(pDnode);
-#endif
-
   mnodeDecDnodeRef(pDnode);
   return code;
 }
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index 042e356442..53f3474c3c 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -32,6 +32,7 @@
 #include "mnodeVgroup.h"
 #include "mnodeUser.h"
 #include "mnodeTable.h"
+#include "mnodeCluster.h"
 #include "mnodeShow.h"
 #include "mnodeProfile.h"
 
@@ -46,6 +47,7 @@ static bool tsMgmtIsRunning = false;
 
 static const SMnodeComponent tsMnodeComponents[] = {
   {"profile", mnodeInitProfile, mnodeCleanupProfile},
+  {"cluster", mnodeInitCluster, mnodeCleanupCluster},
   {"accts",   mnodeInitAccts,   mnodeCleanupAccts},
   {"users",   mnodeInitUsers,   mnodeCleanupUsers},
   {"dnodes",  mnodeInitDnodes,  mnodeCleanupDnodes},
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index d2f389ca0b..2fff885fca 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -190,18 +190,7 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) {
 }
 
 char *mnodeGetMnodeRoleStr(int32_t role) {
-  switch (role) {
-    case TAOS_SYNC_ROLE_OFFLINE:
-      return "offline";
-    case TAOS_SYNC_ROLE_UNSYNCED:
-      return "unsynced";
-    case TAOS_SYNC_ROLE_SLAVE:
-      return "slave";
-    case TAOS_SYNC_ROLE_MASTER:
-      return "master";
-    default:
-      return "undefined";
-  }
+  return syncRole[role];
 }
 
 void mnodeUpdateMnodeEpSet() {
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index 9121f31131..30a292f522 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -68,7 +68,7 @@ int32_t mnodeInitProfile() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
 
-  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, false, mnodeFreeConn, "conn");
+  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, true, mnodeFreeConn, "conn");
   return 0;
 }
 
@@ -119,7 +119,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
     return NULL;
   }
 
-  if (pConn->ip != ip || pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
+  if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
     mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user,
            taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port);
     taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false);
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 4b2945152b..26efcfeac0 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -29,6 +29,7 @@
 #include "mnodeInt.h"
 #include "mnodeMnode.h"
 #include "mnodeDnode.h"
+#include "mnodeCluster.h"
 #include "mnodeSdb.h"
 
 #define SDB_TABLE_LEN 12
@@ -214,6 +215,7 @@ void sdbUpdateMnodeRoles() {
     }
   }
 
+  mnodeUpdateClusterId();
   mnodeUpdateMnodeEpSet();
 }
 
diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c
index 1d85a8cacd..d7f058b3a8 100644
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@@ -103,6 +103,8 @@ static char *mnodeGetShowType(int32_t showType) {
     case TSDB_MGMT_TABLE_SCORES:  return "show scores";
     case TSDB_MGMT_TABLE_GRANTS:  return "show grants";
     case TSDB_MGMT_TABLE_VNODES:  return "show vnodes";
+    case TSDB_MGMT_TABLE_CLUSTER: return "show clusters";
+    case TSDB_MGMT_TABLE_STREAMTABLES : return "show streamtables";
     default:                      return "undefined";
   }
 }
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 089168c31e..231e350092 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -784,7 +784,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
   pCfg->walLevel            = pDb->cfg.walLevel;
   pCfg->replications        = (int8_t) pVgroup->numOfVnodes;
   pCfg->wals                = 3;
-  pCfg->quorum              = 1;
+  pCfg->quorum              = pDb->cfg.quorum;
   
   SMDVnodeDesc *pNodes = pVnode->nodes;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt
index f88209ad9b..a8664669d0 100644
--- a/src/os/CMakeLists.txt
+++ b/src/os/CMakeLists.txt
@@ -1,6 +1,16 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-ADD_SUBDIRECTORY(linux)
-ADD_SUBDIRECTORY(windows)
-ADD_SUBDIRECTORY(darwin)
+IF (TD_LINUX_64)
+  ADD_SUBDIRECTORY(src/linux64)
+ELSEIF (TD_LINUX_32)
+  ADD_SUBDIRECTORY(src/linux32)
+ELSEIF (TD_DARWIN_64)
+  ADD_SUBDIRECTORY(src/darwin64)  
+ELSEIF (TD_WINDOWS_64)
+  ADD_SUBDIRECTORY(src/windows64)
+ELSEIF (TD_WINDOWS_32)
+  ADD_SUBDIRECTORY(src/windows32)
+ENDIF ()
+
+ADD_SUBDIRECTORY(src/detail)
diff --git a/src/os/darwin/CMakeLists.txt b/src/os/darwin/CMakeLists.txt
deleted file mode 100644
index 116d0ec8c0..0000000000
--- a/src/os/darwin/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-IF (TD_DARWIN_64)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(os ${SRC})
-ENDIF ()
diff --git a/src/os/inc/os.h b/src/os/inc/os.h
new file mode 100644
index 0000000000..896f3afc7a
--- /dev/null
+++ b/src/os/inc/os.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_H
+#define TDENGINE_OS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _TD_DARWIN_64
+#include "osDarwin64.h"
+#endif
+
+#ifdef _TD_LINUX_64
+#include "osLinux64.h"
+#endif
+
+#ifdef _TD_LINUX_32
+#include "osLinux32.h"
+#endif
+
+#ifdef _TD_ALPINE
+#include "osAlpine.h"
+#endif
+
+#ifdef _TD_WINDOWS_64
+#include "osWindows64.h"
+#endif
+
+#ifdef _TD_WINDOWS_32
+#include "osWindows32.h"
+#endif
+
+#include "osSpec.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osAlpine.h b/src/os/inc/osAlpine.h
new file mode 100644
index 0000000000..b8212373ce
--- /dev/null
+++ b/src/os/inc/osAlpine.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <linux/sysctl.h>
+
+typedef int(*__compar_fn_t)(const void *, const void *);
+void  error (int, int, const char *);
+#ifndef PTHREAD_MUTEX_RECURSIVE_NP
+  #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/darwin/inc/os.h b/src/os/inc/osDarwin64.h
similarity index 96%
rename from src/os/darwin/inc/os.h
rename to src/os/inc/osDarwin64.h
index 1c719933a4..3ceb7ea8f5 100644
--- a/src/os/darwin/inc/os.h
+++ b/src/os/inc/osDarwin64.h
@@ -201,9 +201,10 @@ int tsem_destroy(dispatch_semaphore_t *sem);
 
 void osInit();
 
+ssize_t tread(int fd, void *buf, size_t count);
+
 ssize_t twrite(int fd, void *buf, size_t n);
 
-char *taosCharsetReplace(char *charsetstr);
 
 bool taosCheckPthreadValid(pthread_t thread);
 
@@ -217,8 +218,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
 
 void taosPrintOsInfo();
 
-char *taosCharsetReplace(char *charsetstr);
-
 void taosPrintOsInfo();
 
 void taosGetSystemInfo();
@@ -233,6 +232,8 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
 
 void taosSetCoreDump();
 
+int tSystem(const char * cmd);
+
 typedef int(*__compar_fn_t)(const void *, const void *);
 
 // for send function in tsocket.c
@@ -255,6 +256,17 @@ typedef int(*__compar_fn_t)(const void *, const void *);
 #define BUILDIN_CLZ(val)  __builtin_clz(val)
 #define BUILDIN_CTZ(val)  __builtin_ctz(val)
 
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h
new file mode 100644
index 0000000000..1778b0e315
--- /dev/null
+++ b/src/os/inc/osLinux32.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h
new file mode 100644
index 0000000000..0a99f4b745
--- /dev/null
+++ b/src/os/inc/osLinux64.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+#include <linux/sysctl.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osSpec.h b/src/os/inc/osSpec.h
new file mode 100644
index 0000000000..48ec6ae04f
--- /dev/null
+++ b/src/os/inc/osSpec.h
@@ -0,0 +1,386 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_SPEC_H
+#define TDENGINE_OS_SPEC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define tclose(x) taosCloseSocket(x)
+#define tfree(x)         \
+  do {                   \
+    if (x) {             \
+      free((void *)(x)); \
+      x = 0;             \
+    }                    \
+  } while (0);
+
+#define tstrncpy(dst, src, size)   \
+  do {                             \
+    strncpy((dst), (src), (size)); \
+    (dst)[(size)-1] = 0;           \
+  } while (0);
+
+#ifndef STDERR_FILENO
+#define STDERR_FILENO (2)
+#endif
+
+#define FD_VALID(x) ((x) > STDERR_FILENO)
+#define FD_INITIALIZER  ((int32_t)-1)
+
+#define WCHAR wchar_t
+
+#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
+#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
+
+#ifndef NDEBUG
+#define ASSERT(x) assert(x)
+#else
+#define ASSERT(x)
+#endif
+
+#ifdef UNUSED
+#undefine UNUSED
+#endif
+#define UNUSED(x) ((void)(x))
+
+#ifdef UNUSED_FUNC
+#undefine UNUSED_FUNC
+#endif
+
+#ifdef UNUSED_PARAM
+#undef UNUSED_PARAM
+#endif
+
+#if defined(__GNUC__)
+#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
+#define UNUSED_FUNC __attribute__((unused))
+#else
+#define UNUSED_PARAM(x) x
+#define UNUSED_FUNC
+#endif
+
+#ifdef tListLen
+#undefine tListLen
+#endif
+#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
+
+#if defined(__GNUC__)
+#define FORCE_INLINE inline __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
+  
+#define DEFAULT_COMP(x, y)       \
+  do {                           \
+    if ((x) == (y)) {            \
+      return 0;                  \
+    } else {                     \
+      return (x) < (y) ? -1 : 1; \
+    }                            \
+  } while (0)
+
+#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
+
+// align to 8bytes
+#define ALIGN8(n) ALIGN_NUM(n, 8)
+
+#define POW2(x) ((x) * (x))
+
+#ifndef TAOS_OS_FUNC_MATH
+  #define SWAP(a, b, c)        \
+    do {                       \
+        typeof(a) __tmp = (a); \
+        (a) = (b);             \
+        (b) = __tmp;           \
+    } while (0)
+
+  #define MAX(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a > __b) ? __a : __b; \
+    })
+
+  #define MIN(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a < __b) ? __a : __b; \
+    })
+#endif
+
+#ifndef TAOS_OS_DEF_TIME
+  #define MILLISECOND_PER_SECOND ((int64_t)1000L)
+#endif
+#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
+#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
+#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
+#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
+#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
+#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
+
+#ifndef TAOS_OS_FUNC_SEMPHONE
+  #define tsem_t sem_t
+  #define tsem_init sem_init
+  #define tsem_wait sem_wait
+  #define tsem_post sem_post
+  #define tsem_destroy sem_destroy
+#endif
+
+#ifndef TAOS_OS_FUNC_ATOMIC
+  #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+
+  #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
+
+  #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+#endif
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count);
+ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
+#ifndef TAOS_OS_FUNC_FILE_OP
+  #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
+  #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
+  #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
+  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
+#endif
+
+#ifndef TAOS_OS_FUNC_NETWORK
+  #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) read(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) write(fd, buf, len)
+  #define taosCloseSocket(x) \
+    {                        \
+      if (FD_VALID(x)) {     \
+        close(x);            \
+        x = FD_INITIALIZER;  \
+      }                      \
+    }
+#endif
+
+#ifndef TAOS_OS_FUNC_LZ4
+  #define BUILDIN_CLZL(val) __builtin_clzl(val)
+  #define BUILDIN_CTZL(val) __builtin_ctzl(val)
+  #define BUILDIN_CLZ(val) __builtin_clz(val)
+  #define BUILDIN_CTZ(val) __builtin_ctz(val)
+#endif
+
+#ifndef TAOS_OS_FUNC_WCHAR
+  #define twcslen wcslen
+#endif  
+
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
+void osInit();
+
+// TAOS_OS_FUNC_PTHREAD
+bool taosCheckPthreadValid(pthread_t thread);
+int64_t taosGetPthreadId();
+
+// TAOS_OS_FUNC_SOCKET
+int taosSetNonblocking(int sock, int on);
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
+void taosBlockSIGPIPE();
+
+// TAOS_OS_FUNC_SYSINFO
+void taosGetSystemInfo();
+void taosPrintOsInfo();
+void taosKillSystem();
+int tSystem(const char * cmd) ;
+
+// TAOS_OS_FUNC_CORE
+void taosSetCoreDump();
+
+// TAOS_OS_FUNC_UTIL
+int64_t tsosStr2int64(char *str);
+
+// TAOS_OS_FUNC_TIMER
+void taosMsleep(int mseconds);
+int taosInitTimer(void (*callback)(int), int ms);
+void taosUninitTimer();
+
+// TAOS_OS_FUNC_RAND
+uint32_t taosRand(void);
+void taosRandStr(char* str, int32_t size);
+uint32_t trand(void);
+
+// TAOS_OS_FUNC_FILE
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath);
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
+
+// USE_LIBICONV
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
+bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len);
+int     tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes);
+bool    taosValidateEncodec(const char *encodec);
+char *  taosCharsetReplace(char *charsetstr);
+
+// TAOS_OS_FUNC_MALLOC
+#define TAOS_ALLOC_MODE_DEFAULT 0
+#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
+#define TAOS_ALLOC_MODE_DETECT_LEAK 2
+void   taosSetAllocMode(int mode, const char *path, bool autoDump);
+void   taosDumpMemoryLeak();
+void * tmalloc(size_t size);
+void * tcalloc(size_t nmemb, size_t size);
+size_t tsizeof(void *ptr);
+void   tmemset(void *ptr, int c);
+void * trealloc(void *ptr, size_t size);
+void   tzfree(void *ptr);
+
+// TAOS_OS_FUNC_DIR
+void taosRemoveDir(char *rootDir);
+int  taosMkDir(const char *pathname, mode_t mode); 
+void taosMvDir(char* destDir, char *srcDir);
+
+
+
+#ifdef TAOS_RANDOM_FILE_FAIL
+  void taosSetRandomFileFailFactor(int factor);
+  void taosSetRandomFileFailOutput(const char *path);
+  ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
+  #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
+#endif
+
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+  ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
+  ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+  ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
+  ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+  #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len)
+#endif
+
+#ifdef TAOS_MEM_CHECK
+  void *  taos_malloc(size_t size, const char *file, uint32_t line);
+  void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
+  void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
+  void    taos_free(void *ptr, const char *file, uint32_t line);
+  char *  taos_strdup(const char *str, const char *file, uint32_t line);
+  char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
+  ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+  #define malloc(size) taos_malloc(size, __FILE__, __LINE__)
+  #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
+  #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
+  #define free(ptr) taos_free(ptr, __FILE__, __LINE__)
+  #define strdup(str) taos_strdup(str, __FILE__, __LINE__)
+  #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+  #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+#endif  // TAOS_MEM_CHECK
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/os/inc/osWindows32.h b/src/os/inc/osWindows32.h
new file mode 100644
index 0000000000..4744d4beb5
--- /dev/null
+++ b/src/os/inc/osWindows32.h
@@ -0,0 +1,54 @@
+/*
+* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+*
+* This program is free software: you can use, redistribute, and/or modify
+* it under the terms of the GNU Affero General Public License, version 3
+* or later ("AGPL"), as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TDENGINE_PLATFORM_WINDOWS32_H
+#define TDENGINE_PLATFORM_WINDOWS32_H
+
+#include <assert.h>
+#include <ctype.h>
+#include <direct.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <float.h>
+#include <locale.h>
+#include <intrin.h>
+#include <io.h>
+#include <math.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <inttypes.h>
+#include "winsock2.h"
+#include <WS2tcpip.h>
+#include <winbase.h>
+#include <Winsock2.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file
diff --git a/src/os/windows/inc/os.h b/src/os/inc/osWindows64.h
similarity index 99%
rename from src/os/windows/inc/os.h
rename to src/os/inc/osWindows64.h
index a1901aa6e7..b4687afb82 100644
--- a/src/os/windows/inc/os.h
+++ b/src/os/inc/osWindows64.h
@@ -48,6 +48,8 @@
 extern "C" {
 #endif
 
+#define TAOS_OS_FUNC_WCHAR
+
 // for function open in stat.h 
 #define S_IRWXU                  _S_IREAD
 #define S_IRWXG                  _S_IWRITE
@@ -78,10 +80,10 @@ extern "C" {
 #define wcsncasecmp _wcsnicmp
 #define strtok_r strtok_s
 #ifdef _TD_GO_DLL_
-  int64_t str2int64(char *str);
+  int64_t tsosStr2int64(char *str);
   uint64_t htonll(uint64_t val);
 #else
-  #define str2int64 _atoi64
+  #define tsosStr2int64 _atoi64
 #endif
 
 #define snprintf _snprintf
@@ -359,7 +361,6 @@ int taosSetNonblocking(int sock, int on);
 
 int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
 
-char *taosCharsetReplace(char *charsetstr);
 
 void taosPrintOsInfo();
 
diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h
deleted file mode 100644
index 00b9f33f1b..0000000000
--- a/src/os/linux/inc/os.h
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_PLATFORM_LINUX_H
-#define TDENGINE_PLATFORM_LINUX_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <argp.h>
-#include <arpa/inet.h>
-#include <assert.h>
-#include <ctype.h>
-#include <dirent.h>
-#include <endian.h>
-#include <errno.h>
-#include <float.h>
-#include <ifaddrs.h>
-#include <libgen.h>
-#include <limits.h>
-#include <locale.h>
-#include <math.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/ip.h>
-#include <netinet/tcp.h>
-#include <netinet/udp.h>
-#include <pthread.h>
-#include <pwd.h>
-#include <regex.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/epoll.h>
-#include <sys/eventfd.h>
-#include <sys/file.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/sendfile.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/syscall.h>
-#include <sys/statvfs.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <sys/un.h>
-#include <syslog.h>
-#include <termios.h>
-#include <unistd.h>
-#include <wchar.h>
-#include <wordexp.h>
-#include <wctype.h>
-#include <inttypes.h>
-#include <fcntl.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <error.h>
-
-#define taosCloseSocket(x) \
-  {                        \
-    if (FD_VALID(x)) {     \
-      close(x);            \
-      x = FD_INITIALIZER;  \
-    }                      \
-  }
-  
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags);
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen);
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count);
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count);
-
-#define send(sockfd, buf, len, flags) taos_send_random_fail(sockfd, buf, len, flags)
-#define sendto(sockfd, buf, len, flags, dest_addr, addrlen) \
-          taos_sendto_random_fail(sockfd, buf, len, flags, dest_addr, addrlen)
-#define taosWriteSocket(fd, buf, len) taos_write_random_fail(fd, buf, len)
-#define taosReadSocket(fd, buf, len) taos_read_random_fail(fd, buf, len)
-
-#else
-
-#define taosWriteSocket(fd, buf, len) write(fd, buf, len)
-#define taosReadSocket(fd, buf, len) read(fd, buf, len)
-
-#endif  /* TAOS_RANDOM_NETWORK_FAIL */
-
-#define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-
-#define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
-
-#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define SWAP(a, b, c)      \
-  do {                     \
-    typeof(a) __tmp = (a); \
-    (a) = (b);             \
-    (b) = __tmp;           \
-  } while (0)
-
-#define MAX(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a > __b) ? __a : __b; \
-  })
-
-#define MIN(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a < __b) ? __a : __b; \
-  })
-
-#define MILLISECOND_PER_SECOND ((int64_t)1000L)
-
-#define tsem_t sem_t
-#define tsem_init sem_init
-#define tsem_wait sem_wait
-#define tsem_post sem_post
-#define tsem_destroy sem_destroy
-
-void osInit();
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size);
-
-ssize_t twrite(int fd, void *buf, size_t n);
-
-ssize_t tread(int fd, void *buf, size_t count);
-
-bool taosCheckPthreadValid(pthread_t thread);
-
-void taosResetPthread(pthread_t *thread);
-
-int64_t taosGetPthreadId();
-
-int taosSetNonblocking(int sock, int on);
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
-
-void taosPrintOsInfo();
-
-char *taosCharsetReplace(char *charsetstr);
-
-void taosGetSystemInfo();
-
-void taosKillSystem();
-
-bool taosSkipSocketCheck();
-
-int64_t str2int64(char *str);
-
-void taosSetCoreDump();
-
-void taosBlockSIGPIPE();
-
-int tSystem(const char * cmd) ;
-
-
-#ifdef _ALPINE
-  typedef int(*__compar_fn_t)(const void *, const void *);
-  void  error (int, int, const char *);
-  #ifndef PTHREAD_MUTEX_RECURSIVE_NP
-    #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
-  #endif
-#endif
-
-#ifndef _TD_ARM_32_
-#define BUILDIN_CLZL(val) __builtin_clzl(val)
-#define BUILDIN_CTZL(val) __builtin_ctzl(val)
-#else
-#define BUILDIN_CLZL(val) __builtin_clzll(val)
-#define BUILDIN_CTZL(val) __builtin_ctzll(val)
-#endif
-#define BUILDIN_CLZ(val) __builtin_clz(val)
-#define BUILDIN_CTZ(val) __builtin_ctz(val)
-
-#undef threadlocal
-#ifdef _ISOC11_SOURCE
-  #define threadlocal _Thread_local
-#elif defined(__APPLE__)
-  #define threadlocal
-#elif defined(__GNUC__) && !defined(threadlocal)
-  #define threadlocal __thread
-#else
-  #define threadlocal
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c
deleted file mode 100644
index 216d8942bc..0000000000
--- a/src/os/linux/src/linuxPlatform.c
+++ /dev/null
@@ -1,318 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taosdef.h"
-#include "tglobal.h"
-#include "ttimer.h"
-#include "tulog.h"
-#include "tutil.h"
-
-int64_t str2int64(char *str) {
-  char *endptr = NULL;
-  return strtoll(str, &endptr, 10);
-}
-
-/*
-  to make taosMsleep work,
-   signal SIGALRM shall be blocked in the calling thread,
-
-  sigset_t set;
-  sigemptyset(&set);
-  sigaddset(&set, SIGALRM);
-  pthread_sigmask(SIG_BLOCK, &set, NULL);
-*/
-void taosMsleep(int mseconds) {
-  struct timeval timeout;
-  int            seconds, useconds;
-
-  seconds = mseconds / 1000;
-  useconds = (mseconds % 1000) * 1000;
-  timeout.tv_sec = seconds;
-  timeout.tv_usec = useconds;
-
-  /* sigset_t set; */
-  /* sigemptyset(&set); */
-  /* sigaddset(&set, SIGALRM); */
-  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
-
-  select(0, NULL, NULL, NULL, &timeout);
-
-  /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
-}
-
-bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
-
-int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
-
-int taosSetNonblocking(int sock, int on) {
-  int flags = 0;
-  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
-    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  if (on)
-    flags |= O_NONBLOCK;
-  else
-    flags &= ~O_NONBLOCK;
-
-  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
-    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  return 0;
-}
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
-  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
-}
-static void taosDeleteTimer(void *tharg) {
-  timer_t *pTimer = tharg;
-  timer_delete(*pTimer);
-}
-
-static pthread_t timerThread;
-static timer_t         timerId;
-static volatile bool stopTimer = false;
-
-void *taosProcessAlarmSignal(void *tharg) {
-  // Block the signal
-  sigset_t sigset;
-  sigemptyset(&sigset);
-  sigaddset(&sigset, SIGALRM);
-  sigprocmask(SIG_BLOCK, &sigset, NULL);
-  void (*callback)(int) = tharg;
-
-  struct sigevent sevent = {{0}};
-
-  #ifdef _ALPINE
-    sevent.sigev_notify = SIGEV_THREAD;
-    sevent.sigev_value.sival_int = syscall(__NR_gettid);
-  #else
-    sevent.sigev_notify = SIGEV_THREAD_ID;
-    sevent._sigev_un._tid = syscall(__NR_gettid);
-  #endif
-  
-  sevent.sigev_signo = SIGALRM;
-
-  if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) {
-    uError("Failed to create timer");
-  }
-
-  pthread_cleanup_push(taosDeleteTimer, &timerId);
-
-  struct itimerspec ts;
-  ts.it_value.tv_sec = 0;
-  ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK;
-  ts.it_interval.tv_sec = 0;
-  ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK;
-
-  if (timer_settime(timerId, 0, &ts, NULL)) {
-    uError("Failed to init timer");
-    return NULL;
-  }
-
-  int signo;
-  while (!stopTimer) {
-    if (sigwait(&sigset, &signo)) {
-      uError("Failed to wait signal: number %d", signo);
-      continue;
-    }
-    /* printf("Signal handling: number %d ......\n", signo); */
-
-    callback(0);
-  }
-  
-  pthread_cleanup_pop(1);
-
-  return NULL;
-}
-
-
-int taosInitTimer(void (*callback)(int), int ms) {
-  pthread_attr_t tattr;
-  pthread_attr_init(&tattr);
-  int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback);
-  pthread_attr_destroy(&tattr);
-  if (code != 0) {
-    uError("failed to create timer thread");
-    return -1;
-  }
-  return 0;
-}
-
-void taosUninitTimer() {
-  stopTimer = true;
-  pthread_join(timerThread, NULL);
-}
-
-ssize_t tread(int fd, void *buf, size_t count) {
-  size_t  leftbytes = count;
-  ssize_t readbytes;
-  char *  tbuf = (char *)buf;
-
-  while (leftbytes > 0) {
-    readbytes = read(fd, (void *)tbuf, leftbytes);
-    if (readbytes < 0) {
-      if (errno == EINTR) {
-        continue;
-      } else {
-        return -1;
-      }
-    } else if (readbytes == 0) {
-      return (ssize_t)(count - leftbytes);
-    }
-
-    leftbytes -= readbytes;
-    tbuf += readbytes;
-  }
-
-  return (ssize_t)count;
-}
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size) {
-  size_t  leftbytes = size;
-  ssize_t sentbytes;
-
-  while (leftbytes > 0) {
-    /*
-     * TODO : Think to check if file is larger than 1GB
-     */
-    //if (leftbytes > 1000000000) leftbytes = 1000000000;
-    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
-    if (sentbytes == -1) {
-      if (errno == EINTR) {
-        continue;
-      }
-      else {
-        return -1;
-      }
-    } else if (sentbytes == 0) {
-      return (ssize_t)(size - leftbytes);
-    }
-
-    leftbytes -= sentbytes;
-  }
-
-  return size;
-}
-
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
-void taosBlockSIGPIPE() {
-  sigset_t signal_mask;
-  sigemptyset(&signal_mask);
-  sigaddset(&signal_mask, SIGPIPE);
-  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
-  if (rc != 0) {
-    uError("failed to block SIGPIPE");
-  }
-}
-
-int tSystem(const char * cmd) 
-{ 
-  FILE * fp; 
-  int res; 
-  char buf[1024]; 
-  if (cmd == NULL) { 
-    uError("tSystem cmd is NULL!\n");
-    return -1;
-  } 
-  
-  if ((fp = popen(cmd, "r") ) == NULL) { 
-    uError("popen cmd:%s error: %s/n", cmd, strerror(errno)); 
-    return -1; 
-  } else {
-    while(fgets(buf, sizeof(buf), fp))  { 
-      uDebug("popen result:%s", buf); 
-    } 
-
-    if ((res = pclose(fp)) == -1) { 
-      uError("close popen file pointer fp error!\n");
-    } else { 
-      uDebug("popen res is :%d\n", res);
-    } 
-
-    return res;
-  }
-}
-
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-#define RANDOM_NETWORK_FAIL_FACTOR  20
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return send(sockfd, buf, len, flags);
-}
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
-}
-
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return read(fd, buf, count);
-}
-
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = EINTR;
-    return -1;
-  }
-
-  return write(fd, buf, count);
-}
-
-#endif /* TAOS_RANDOM_NETWORK_FAIL */
diff --git a/src/os/src/alpine/CMakeLists.txt b/src/os/src/alpine/CMakeLists.txt
new file mode 100644
index 0000000000..79cbf917e2
--- /dev/null
+++ b/src/os/src/alpine/CMakeLists.txt
@@ -0,0 +1,12 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+
diff --git a/src/os/linux/src/linuxEnv.c b/src/os/src/alpine/alpineEnv.c
similarity index 100%
rename from src/os/linux/src/linuxEnv.c
rename to src/os/src/alpine/alpineEnv.c
diff --git a/src/os/linux/CMakeLists.txt b/src/os/src/darwin64/CMakeLists.txt
similarity index 61%
rename from src/os/linux/CMakeLists.txt
rename to src/os/src/darwin64/CMakeLists.txt
index 7be4a13f25..71029d9291 100644
--- a/src/os/linux/CMakeLists.txt
+++ b/src/os/src/darwin64/CMakeLists.txt
@@ -1,16 +1,13 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+IF (TD_DARWIN_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
-
   ADD_LIBRARY(os ${SRC})
-  TARGET_LINK_LIBRARIES(os m rt)
 ENDIF ()
 
-SET_SOURCE_FILES_PROPERTIES(src/linuxSysPara.c PROPERTIES COMPILE_FLAGS -w)
diff --git a/src/os/darwin/src/darwinEnv.c b/src/os/src/darwin64/darwinEnv.c
similarity index 100%
rename from src/os/darwin/src/darwinEnv.c
rename to src/os/src/darwin64/darwinEnv.c
diff --git a/src/os/darwin/src/darwinPlatform.c b/src/os/src/darwin64/darwinPlatform.c
similarity index 85%
rename from src/os/darwin/src/darwinPlatform.c
rename to src/os/src/darwin64/darwinPlatform.c
index 44540cf69e..045e4a7099 100644
--- a/src/os/darwin/src/darwinPlatform.c
+++ b/src/os/src/darwin64/darwinPlatform.c
@@ -15,12 +15,15 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
+#include "os.h"
 #include "taosdef.h"
-#include "tutil.h"
+#include "tglobal.h"
+#include "tconfig.h"
+#include "ttimer.h"
 #include "tulog.h"
+#include "tutil.h"
 
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
@@ -159,12 +162,14 @@ int taosInitTimer(void (*callback)(int), int ms) {
   tv.it_interval.tv_usec = 1000 * ms;  // resolution is in msecond
   tv.it_value = tv.it_interval;
 
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
+
+  return 0;
 }
 
 void taosUninitTimer() {
   struct itimerval tv = { 0 };
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
 }
 
 void taosGetSystemTimezone() {
@@ -207,7 +212,7 @@ void taosGetSystemLocale() {
 void taosPrintOsInfo() {}
 
 void taosKillSystem() {
-  tError("function taosKillSystem, exit!");
+  uError("function taosKillSystem, exit!");
   exit(0);
 }
 
@@ -221,95 +226,75 @@ void taosGetSystemInfo() {
 }
 
 void *taosInitTcpClient(char *ip, uint16_t port, char *flabel, int num, void *fp, void *shandle) {
-  tError("function taosInitTcpClient is not implemented in darwin system, exit!");
+  uError("function taosInitTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpClientConnection(void *chandle) {
-  tError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
-  tError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpClientData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpClientData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpClientData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpClient(void *chandle) {
-  tError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpServerConnection(void *chandle) {
-  tError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpServer(void *handle) {
-  tError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
-  tError("function taosInitTcpServer is not implemented in darwin system, exit!");
+  uError("function taosInitTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpServerData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpServerData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpServerData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosFreeMsgHdr(void *hdr) {
-  tError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosMsgHdrSize(void *hdr) {
-  tError("function taosMsgHdrSize is not implemented in darwin system, exit!");
+  uError("function taosMsgHdrSize is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSendMsgHdr(void *hdr, int fd) {
-  tError("function taosSendMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosSendMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) {
-  tError("function taosInitMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosInitMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSetMsgHdrData(void *hdr, char *data, int dataLen) {
-  tError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
+  uError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
   exit(0);
 }
 
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
 bool taosSkipSocketCheck() {
   return true;
 }
diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt
new file mode 100644
index 0000000000..5902a9356b
--- /dev/null
+++ b/src/os/src/detail/CMakeLists.txt
@@ -0,0 +1,14 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(.)
+AUX_SOURCE_DIRECTORY(. SRC)
+SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w)
+SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w)
+
+ADD_LIBRARY(osdetail ${SRC})
+TARGET_LINK_LIBRARIES(osdetail os)
diff --git a/src/os/src/detail/osCoredump.c b/src/os/src/detail/osCoredump.c
new file mode 100644
index 0000000000..1ead88ab9e
--- /dev/null
+++ b/src/os/src/detail/osCoredump.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "tsystem.h"
+
+#ifndef TAOS_OS_FUNC_CORE
+
+int _sysctl(struct __sysctl_args *args );
+
+void taosSetCoreDump() {
+  if (0 == tsEnableCoreFile) {
+    return;
+  }
+  
+  // 1. set ulimit -c unlimited
+  struct rlimit rlim;
+  struct rlimit rlim_new;
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+    rlim_new.rlim_cur = RLIM_INFINITY;
+    rlim_new.rlim_max = RLIM_INFINITY;
+    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
+      uInfo("set unlimited fail, error: %s", strerror(errno));
+      rlim_new.rlim_cur = rlim.rlim_max;
+      rlim_new.rlim_max = rlim.rlim_max;
+      (void)setrlimit(RLIMIT_CORE, &rlim_new);
+    }
+  }
+
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+  }
+
+#ifndef _TD_ARM_
+  // 2. set the path for saving core file
+  struct __sysctl_args args;
+  int     old_usespid = 0;
+  size_t  old_len     = 0;
+  int     new_usespid = 1;
+  size_t  new_len     = sizeof(new_usespid);
+  
+  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
+  
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  args.newval  = &new_usespid;
+  args.newlen  = new_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
+  }
+  
+  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+
+
+  old_usespid = 0;
+  old_len     = 0;
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
+  }
+  
+  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+#endif
+
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c
new file mode 100644
index 0000000000..27555938c2
--- /dev/null
+++ b/src/os/src/detail/osDir.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_DIR
+
+void taosRemoveDir(char *rootDir) {
+  DIR *dir = opendir(rootDir);
+  if (dir == NULL) return;
+
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+     
+    char filename[1024];
+    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
+    if (de->d_type & DT_DIR) {
+      taosRemoveDir(filename);
+    } else {
+      (void)remove(filename);
+      uInfo("file:%s is removed", filename);
+    }
+  }
+
+  closedir(dir);
+  rmdir(rootDir);
+
+  uInfo("dir:%s is removed", rootDir);
+}
+
+int taosMkDir(const char *path, mode_t mode) {
+  int code = mkdir(path, 0755);
+  if (code < 0 && errno == EEXIST) code = 0;
+  return code;
+}
+
+void taosMvDir(char* destDir, char *srcDir) {
+  if (0 == tsEnableVnodeBak) {
+    uInfo("vnode backup not enabled");
+    return;
+  }
+
+  char shellCmd[1024+1] = {0}; 
+  
+  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
+  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
+  tSystem(shellCmd);
+  uInfo("shell cmd:%s is executed", shellCmd);
+}
+
+
+#endif
\ No newline at end of file
diff --git a/src/util/src/tfile.c b/src/os/src/detail/osFail.c
similarity index 55%
rename from src/util/src/tfile.c
rename to src/os/src/detail/osFail.c
index d1a9c24615..e0eb200851 100644
--- a/src/util/src/tfile.c
+++ b/src/os/src/detail/osFail.c
@@ -13,31 +13,62 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <error.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <unistd.h>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
+#define _DEFAULT_SOURCE
 #include "os.h"
 
+#define RANDOM_NETWORK_FAIL_FACTOR  20
+
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+
+ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return send(sockfd, buf, len, flags);
+}
+
+ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
+                           socklen_t addrlen) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
+}
+
+ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return read(fd, buf, count);
+}
+
+ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = EINTR;
+    return -1;
+  }
+
+  return write(fd, buf, count);
+}
+
+#endif //TAOS_RANDOM_NETWORK_FAIL
+
 #ifdef TAOS_RANDOM_FILE_FAIL
 
 static int random_file_fail_factor = 20;
 static FILE *fpRandomFileFailOutput = NULL;
 
-void taosSetRandomFileFailFactor(int factor)
-{
+void taosSetRandomFileFailFactor(int factor) { 
   random_file_fail_factor = factor;
 }
 
-static void close_random_file_fail_output()
-{
+static void close_random_file_fail_output() {
   if (fpRandomFileFailOutput != NULL) {
     if (fpRandomFileFailOutput != stdout) {
       fclose(fpRandomFileFailOutput);
@@ -46,8 +77,7 @@ static void close_random_file_fail_output()
   }
 }
 
-static void random_file_fail_output_sig(int sig)
-{
+static void random_file_fail_output_sig(int sig) {
   fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
 
   struct sigaction act = {0};
@@ -58,8 +88,7 @@ static void random_file_fail_output_sig(int sig)
   exit(EXIT_FAILURE);
 }
 
-void taosSetRandomFileFailOutput(const char *path)
-{
+void taosSetRandomFileFailOutput(const char *path) {
   if (path == NULL) {
     fpRandomFileFailOutput = stdout;
   } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) {
@@ -75,43 +104,38 @@ void taosSetRandomFileFailOutput(const char *path)
   sigaction(SIGSEGV, &act, NULL);
   sigaction(SIGILL, &act, NULL);
 }
-#endif
 
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return tread(fd, buf, count);
+
+  return taosTReadImp(fd, buf, count);
 }
 
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return twrite(fd, buf, count);
+
+  return taosTWriteImp(fd, buf, count);
 }
 
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
+
   return lseek(fd, offset, whence);
 }
+
+#endif //TAOS_RANDOM_FILE_FAIL
diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c
new file mode 100644
index 0000000000..5502dfb4a6
--- /dev/null
+++ b/src/os/src/detail/osFile.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttime.h"
+
+#ifndef TAOS_OS_FUNC_FILE
+
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+  
+  char tmpPath[PATH_MAX];
+  char *tmpDir = "/tmp/";
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}
+
+// rename file name
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+  int32_t ts = taosGetTimestampSec();
+
+  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
+
+  char *delimiterPos = strrchr(fullPath, delimiter);
+  if (delimiterPos == NULL) return -1;
+
+  int32_t fileNameLen = 0;
+  if (suffix)
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
+  else
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
+
+  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+  if (*dstPath == NULL) {
+    *dstPath = calloc(1, len + 1);
+    if (*dstPath == NULL) return -1;
+  }
+
+  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
+  strncat(*dstPath, fname, (size_t)fileNameLen);
+  (*dstPath)[len] = 0;
+
+  return rename(fullPath, *dstPath);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osFileOp.c b/src/os/src/detail/osFileOp.c
new file mode 100644
index 0000000000..897b6c3f03
--- /dev/null
+++ b/src/os/src/detail/osFileOp.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count) {
+  size_t  leftbytes = count;
+  ssize_t readbytes;
+  char *  tbuf = (char *)buf;
+
+  while (leftbytes > 0) {
+    readbytes = read(fd, (void *)tbuf, leftbytes);
+    if (readbytes < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (readbytes == 0) {
+      return (ssize_t)(count - leftbytes);
+    }
+
+    leftbytes -= readbytes;
+    tbuf += readbytes;
+  }
+
+  return (ssize_t)count;
+}
+
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
+  size_t  leftbytes = size;
+  ssize_t sentbytes;
+
+  while (leftbytes > 0) {
+    /*
+     * TODO : Think to check if file is larger than 1GB
+     */
+    //if (leftbytes > 1000000000) leftbytes = 1000000000;
+    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
+    if (sentbytes == -1) {
+      if (errno == EINTR) {
+        continue;
+      }
+      else {
+        return -1;
+      }
+    } else if (sentbytes == 0) {
+      return (ssize_t)(size - leftbytes);
+    }
+
+    leftbytes -= sentbytes;
+  }
+
+  return size;
+}
+
+ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
+  size_t nleft = n; 
+  ssize_t nwritten = 0;
+  char *tbuf = (char *)buf;
+
+  while (nleft > 0) {
+    nwritten = write(fd, (void *)tbuf, nleft);
+    if (nwritten < 0) {
+      if (errno == EINTR) {
+        continue;
+      }
+      return -1;
+    }
+    nleft -= nwritten;
+    tbuf += nwritten;
+  }
+
+  return n;
+}
diff --git a/src/os/src/detail/osMalloc.c b/src/os/src/detail/osMalloc.c
new file mode 100644
index 0000000000..66377827da
--- /dev/null
+++ b/src/os/src/detail/osMalloc.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_MALLOC
+
+void *tmalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = malloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+void *tcalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = tmalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  tmemset(ret, 0);
+  return ret;
+}
+
+size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
+
+void * trealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return tmalloc(size);
+
+  if (size <= tsizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  tptr = realloc(tptr, tsize);
+  if (tptr == NULL) return NULL;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+void tzfree(void *ptr) {
+  if (ptr) {
+    free((void *)((char *)ptr - sizeof(size_t)));
+  }
+}
+
+#endif
\ No newline at end of file
diff --git a/src/util/src/tmem.c b/src/os/src/detail/osMem.c
similarity index 99%
rename from src/util/src/tmem.c
rename to src/os/src/detail/osMem.c
index 9c512ad8dc..93943c6543 100644
--- a/src/util/src/tmem.c
+++ b/src/os/src/detail/osMem.c
@@ -13,13 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tulog.h"
 
-#define TAOS_MEM_CHECK_IMPL
-#include "tutil.h"
-
-
 #ifdef TAOS_MEM_CHECK
 
 static int allocMode = TAOS_ALLOC_MODE_DEFAULT;
diff --git a/src/os/src/detail/osPThread.c b/src/os/src/detail/osPThread.c
new file mode 100644
index 0000000000..18207dc12e
--- /dev/null
+++ b/src/os/src/detail/osPThread.c
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_PTHREAD
+
+bool    taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
+int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c
new file mode 100644
index 0000000000..7e5f585634
--- /dev/null
+++ b/src/os/src/detail/osRand.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_RAND
+
+uint32_t taosRand(void) { return rand(); }
+
+uint32_t trand(void) {
+  int fd;
+  int seed;
+
+  fd = open("/dev/urandom", 0);
+  if (fd < 0) {
+    seed = time(0);
+  } else {
+    int len = read(fd, &seed, sizeof(seed));
+    if (len < 0) {
+      seed = time(0);
+    }
+    close(fd);
+  }
+
+  return (uint32_t)seed;
+}
+
+#endif
+
+void taosRandStr(char* str, int32_t size) {
+  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
+  int32_t     len = 39;
+
+  for (int32_t i = 0; i < size; ++i) {
+    str[i] = set[taosRand() % len];
+  }
+}
\ No newline at end of file
diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c
new file mode 100644
index 0000000000..ea0b92de5f
--- /dev/null
+++ b/src/os/src/detail/osSocket.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_SOCKET
+
+int taosSetNonblocking(int sock, int on) {
+  int flags = 0;
+  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
+    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  if (on)
+    flags |= O_NONBLOCK;
+  else
+    flags &= ~O_NONBLOCK;
+
+  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
+    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  return 0;
+}
+
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
+  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
+}
+
+void taosBlockSIGPIPE() {
+  sigset_t signal_mask;
+  sigemptyset(&signal_mask);
+  sigaddset(&signal_mask, SIGPIPE);
+  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+  if (rc != 0) {
+    uError("failed to block SIGPIPE");
+  }
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/src/detail/osSysinfo.c
similarity index 79%
rename from src/os/linux/src/linuxSysPara.c
rename to src/os/src/detail/osSysinfo.c
index 1331503619..7a395285af 100644
--- a/src/os/linux/src/linuxSysPara.c
+++ b/src/os/src/detail/osSysinfo.c
@@ -15,12 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tsysctl.h"
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tutil.h"
 #include "tsystem.h"
+#include "taosdef.h"
+
+#ifndef TAOS_OS_FUNC_SYSINFO
 
 #define PROCESS_ITEM 12
 
@@ -200,7 +201,7 @@ static void taosGetSystemTimezone() {
    * Asia/Shanghai   (CST, +0800)
    * Europe/London   (BST, +0100)
    */
-  sprintf(tsTimezone, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
+  snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
 
   // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
   uInfo("timezone not configured, set to system default:%s", tsTimezone);
@@ -234,7 +235,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       uError("can't get locale from system, set it to en_US.UTF-8");
       strcpy(tsLocale, "en_US.UTF-8");
     } else {
-      tstrncpy(tsLocale, locale, sizeof(tsLocale));
+      tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
       uError("locale not configured, set to system default:%s", tsLocale);
     }
   }
@@ -247,7 +248,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       str++;
 
       char *revisedCharset = taosCharsetReplace(str);
-      tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
+      tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
 
       free(revisedCharset);
       uWarn("charset not configured, set to system default:%s", tsCharset);
@@ -542,126 +543,31 @@ void taosKillSystem() {
   kill(tsProcId, 2);
 }
 
-int _sysctl(struct __sysctl_args *args );
-void taosSetCoreDump() {
-  if (0 == tsEnableCoreFile) {
-    return;
+int tSystem(const char *cmd) {
+  FILE *fp;
+  int   res;
+  char  buf[1024];
+  if (cmd == NULL) {
+    uError("tSystem cmd is NULL!\n");
+    return -1;
   }
-  
-  // 1. set ulimit -c unlimited
-  struct rlimit rlim;
-  struct rlimit rlim_new;
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
-    rlim_new.rlim_cur = RLIM_INFINITY;
-    rlim_new.rlim_max = RLIM_INFINITY;
-    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
-      uInfo("set unlimited fail, error: %s", strerror(errno));
-      rlim_new.rlim_cur = rlim.rlim_max;
-      rlim_new.rlim_max = rlim.rlim_max;
-      (void)setrlimit(RLIMIT_CORE, &rlim_new);
+
+  if ((fp = popen(cmd, "r")) == NULL) {
+    uError("popen cmd:%s error: %s/n", cmd, strerror(errno));
+    return -1;
+  } else {
+    while (fgets(buf, sizeof(buf), fp)) {
+      uDebug("popen result:%s", buf);
     }
-  }
 
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
-  }
-
-#ifndef _TD_ARM_
-  // 2. set the path for saving core file
-  struct __sysctl_args args;
-  int     old_usespid = 0;
-  size_t  old_len     = 0;
-  int     new_usespid = 1;
-  size_t  new_len     = sizeof(new_usespid);
-  
-  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
-  
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  args.newval  = &new_usespid;
-  args.newlen  = new_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
-  }
-  
-  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-
-
-  old_usespid = 0;
-  old_len     = 0;
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
-  }
-  
-  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-#endif
-  
-#if 0
-  // 3. create the path for saving core file
-  int status; 
-  char coredump_dir[32] = "/var/log/taosdump";
-  if (opendir(coredump_dir) == NULL) {
-    status = mkdir(coredump_dir, S_IRWXU | S_IRWXG | S_IRWXO); 
-    if (status) {
-      uInfo("mkdir fail, error: %s\n", strerror(errno));
+    if ((res = pclose(fp)) == -1) {
+      uError("close popen file pointer fp error!\n");
+    } else {
+      uDebug("popen res is :%d\n", res);
     }
+
+    return res;
   }
-
-  // 4. set kernel.core_pattern
-   struct __sysctl_args args;
-   char    old_corefile[128];
-   size_t  old_len;
-   char    new_corefile[128] = "/var/log/taosdump/core-%e-%p";
-   size_t  new_len = sizeof(new_corefile);
-   
-   int name[] = {CTL_KERN, KERN_CORE_PATTERN};
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   args.newval  = new_corefile;
-   args.newlen  = new_len;
-
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) set fail: %s", strerror(errno));
-   }
-   
-   uInfo("The old kern_core_pattern: %*s\n", old_len, old_corefile);
-
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) get fail: %s", strerror(errno));
-   }
-   
-   uInfo("The new kern_core_pattern: %*s\n", old_len, old_corefile);
-#endif
-  
 }
 
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c
new file mode 100644
index 0000000000..e0a2e90314
--- /dev/null
+++ b/src/os/src/detail/osTimer.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosdef.h"
+#include "tglobal.h"
+#include "ttimer.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_TIMER
+
+/*
+  to make taosMsleep work,
+   signal SIGALRM shall be blocked in the calling thread,
+
+  sigset_t set;
+  sigemptyset(&set);
+  sigaddset(&set, SIGALRM);
+  pthread_sigmask(SIG_BLOCK, &set, NULL);
+*/
+void taosMsleep(int mseconds) {
+  struct timeval timeout;
+  int            seconds, useconds;
+
+  seconds = mseconds / 1000;
+  useconds = (mseconds % 1000) * 1000;
+  timeout.tv_sec = seconds;
+  timeout.tv_usec = useconds;
+
+  /* sigset_t set; */
+  /* sigemptyset(&set); */
+  /* sigaddset(&set, SIGALRM); */
+  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
+
+  select(0, NULL, NULL, NULL, &timeout);
+
+  /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
+}
+
+
+static void taosDeleteTimer(void *tharg) {
+  timer_t *pTimer = tharg;
+  timer_delete(*pTimer);
+}
+
+static pthread_t timerThread;
+static timer_t         timerId;
+static volatile bool stopTimer = false;
+static void *taosProcessAlarmSignal(void *tharg) {
+  // Block the signal
+  sigset_t sigset;
+  sigemptyset(&sigset);
+  sigaddset(&sigset, SIGALRM);
+  sigprocmask(SIG_BLOCK, &sigset, NULL);
+  void (*callback)(int) = tharg;
+
+  struct sigevent sevent = {{0}};
+
+  #ifdef _ALPINE
+    sevent.sigev_notify = SIGEV_THREAD;
+    sevent.sigev_value.sival_int = syscall(__NR_gettid);
+  #else
+    sevent.sigev_notify = SIGEV_THREAD_ID;
+    sevent._sigev_un._tid = syscall(__NR_gettid);
+  #endif
+  
+  sevent.sigev_signo = SIGALRM;
+
+  if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) {
+    uError("Failed to create timer");
+  }
+
+  pthread_cleanup_push(taosDeleteTimer, &timerId);
+
+  struct itimerspec ts;
+  ts.it_value.tv_sec = 0;
+  ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK;
+  ts.it_interval.tv_sec = 0;
+  ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK;
+
+  if (timer_settime(timerId, 0, &ts, NULL)) {
+    uError("Failed to init timer");
+    return NULL;
+  }
+
+  int signo;
+  while (!stopTimer) {
+    if (sigwait(&sigset, &signo)) {
+      uError("Failed to wait signal: number %d", signo);
+      continue;
+    }
+    /* printf("Signal handling: number %d ......\n", signo); */
+
+    callback(0);
+  }
+  
+  pthread_cleanup_pop(1);
+
+  return NULL;
+}
+
+int taosInitTimer(void (*callback)(int), int ms) {
+  pthread_attr_t tattr;
+  pthread_attr_init(&tattr);
+  int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback);
+  pthread_attr_destroy(&tattr);
+  if (code != 0) {
+    uError("failed to create timer thread");
+    return -1;
+  }
+  return 0;
+}
+
+void taosUninitTimer() {
+  stopTimer = true;
+  pthread_join(timerThread, NULL);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osUtil.c b/src/os/src/detail/osUtil.c
new file mode 100644
index 0000000000..10576c9a0a
--- /dev/null
+++ b/src/os/src/detail/osUtil.c
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_UTIL
+
+int64_t tsosStr2int64(char *str) {
+  char *endptr = NULL;
+  return strtoll(str, &endptr, 10);
+}
+
+#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osWchar.c b/src/os/src/detail/osWchar.c
new file mode 100644
index 0000000000..4e02973e50
--- /dev/null
+++ b/src/os/src/detail/osWchar.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "taosdef.h"
+
+#ifndef TAOS_OS_FUNC_WCHAR
+
+int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
+  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
+}
+
+#endif
+
+#ifdef USE_LIBICONV
+#include "iconv.h"
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
+  size_t  ucs4_input_len = ucs4_max_len;
+  size_t  outLen = ucs4_max_len;
+  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
+    iconv_close(cd);
+    return -1;
+  }
+
+  iconv_close(cd);
+  return (int32_t)(ucs4_max_len - outLen);
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
+  size_t  ucs4_input_len = mbsLength;
+  size_t  outLeft = ucs4_max_len;
+  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
+    iconv_close(cd);
+    return false;
+  }
+
+  iconv_close(cd);
+  if (len != NULL) {
+    *len = ucs4_max_len - outLeft;
+  }
+
+  return true;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
+  if (cd == (iconv_t)(-1)) {
+    return false;
+  }
+
+  iconv_close(cd);
+  return true;
+}
+
+#else
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  mbstate_t state = {0};
+  int32_t   len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  memset(&state, 0, sizeof(state));
+  len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  return len;
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  mbstate_t state = {0};
+  int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
+  *len = retlen;
+
+  return retlen >= 0;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  return true;
+}
+
+#endif
+
+typedef struct CharsetPair {
+  char *oldCharset;
+  char *newCharset;
+} CharsetPair;
+
+char *taosCharsetReplace(char *charsetstr) {
+  CharsetPair charsetRep[] = {
+      { "utf8", "UTF-8" }, { "936", "CP936" },
+  };
+
+  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
+    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
+      return strdup(charsetRep[i].newCharset);
+    }
+  }
+
+  return strdup(charsetstr);
+}
\ No newline at end of file
diff --git a/src/os/src/linux64/CMakeLists.txt b/src/os/src/linux64/CMakeLists.txt
new file mode 100644
index 0000000000..79cbf917e2
--- /dev/null
+++ b/src/os/src/linux64/CMakeLists.txt
@@ -0,0 +1,12 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+
diff --git a/src/os/src/linux64/linuxEnv.c b/src/os/src/linux64/linuxEnv.c
new file mode 100644
index 0000000000..0a9d81311a
--- /dev/null
+++ b/src/os/src/linux64/linuxEnv.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void osInit() {
+  strcpy(configDir, "/etc/taos");
+  strcpy(tsVnodeDir, "");
+  strcpy(tsDnodeDir, "");
+  strcpy(tsMnodeDir, "");
+  strcpy(tsDataDir, "/var/lib/taos");
+  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsScriptDir, "/etc/taos");
+  strcpy(tsOsName, "Linux");
+}
\ No newline at end of file
diff --git a/src/os/windows/CMakeLists.txt b/src/os/src/windows64/CMakeLists.txt
similarity index 100%
rename from src/os/windows/CMakeLists.txt
rename to src/os/src/windows64/CMakeLists.txt
diff --git a/src/os/windows/src/twindows.c b/src/os/src/windows64/twindows.c
similarity index 99%
rename from src/os/windows/src/twindows.c
rename to src/os/src/windows64/twindows.c
index 295824846e..4284dcb14e 100644
--- a/src/os/windows/src/twindows.c
+++ b/src/os/src/windows64/twindows.c
@@ -398,7 +398,7 @@ char *strndup(const char *s, size_t n) {
 void taosSetCoreDump() {}
 
 #ifdef _TD_GO_DLL_
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
diff --git a/src/os/windows/src/twinenv.c b/src/os/src/windows64/twinenv.c
similarity index 100%
rename from src/os/windows/src/twinenv.c
rename to src/os/src/windows64/twinenv.c
diff --git a/src/os/windows/src/twingetline.c b/src/os/src/windows64/twingetline.c
similarity index 100%
rename from src/os/windows/src/twingetline.c
rename to src/os/src/windows64/twingetline.c
diff --git a/src/os/windows/src/twinmsghdr.c b/src/os/src/windows64/twinmsghdr.c
similarity index 100%
rename from src/os/windows/src/twinmsghdr.c
rename to src/os/src/windows64/twinmsghdr.c
diff --git a/src/os/windows/src/twinsocket.c b/src/os/src/windows64/twinsocket.c
similarity index 100%
rename from src/os/windows/src/twinsocket.c
rename to src/os/src/windows64/twinsocket.c
diff --git a/src/os/windows/src/twinstrptime.c b/src/os/src/windows64/twinstrptime.c
similarity index 100%
rename from src/os/windows/src/twinstrptime.c
rename to src/os/src/windows64/twinstrptime.c
diff --git a/src/os/windows/src/twintcpclient.c b/src/os/src/windows64/twintcpclient.c
similarity index 100%
rename from src/os/windows/src/twintcpclient.c
rename to src/os/src/windows64/twintcpclient.c
diff --git a/src/os/windows/src/twintcpserver.c b/src/os/src/windows64/twintcpserver.c
similarity index 100%
rename from src/os/windows/src/twintcpserver.c
rename to src/os/src/windows64/twintcpserver.c
diff --git a/src/os/windows/src/twintime.c b/src/os/src/windows64/twintime.c
similarity index 100%
rename from src/os/windows/src/twintime.c
rename to src/os/src/windows64/twintime.c
diff --git a/src/os/windows/src/twintimer.c b/src/os/src/windows64/twintimer.c
similarity index 100%
rename from src/os/windows/src/twintimer.c
rename to src/os/src/windows64/twintimer.c
diff --git a/src/os/src/windows64/w64File.c b/src/os/src/windows64/w64File.c
new file mode 100644
index 0000000000..140277f990
--- /dev/null
+++ b/src/os/src/windows64/w64File.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+    char tmpPath[PATH_MAX];
+
+  char *tmpDir = getenv("tmp");
+  if (tmpDir == NULL) {
+    tmpDir = "";
+  }
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}
diff --git a/src/os/src/windows64/w64Wchar.c b/src/os/src/windows64/w64Wchar.c
new file mode 100644
index 0000000000..d5930d1de9
--- /dev/null
+++ b/src/os/src/windows64/w64Wchar.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+size_t twcslen(const wchar_t *wcs) {
+  int *wstr = (int *)wcs;
+  if (NULL == wstr) {
+    return 0;
+  }
+
+  size_t n = 0;
+  while (1) {
+    if (0 == *wstr++) {
+      break;
+    }
+    n++;
+  }
+
+  return n;
+}
+
+int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) {
+  for (int i = 0; i < bytes; ++i) {
+    int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4);
+    int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4);
+
+    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
+      return f1 - f2;
+    } else if (f1 == 0 && f2 == 0) {
+      return 0;
+    }
+
+    if (f1 != f2) {
+      return f1 - f2;
+    }
+  }
+
+  return 0;
+
+#if 0
+  int32_t ucs4_max_len = bytes + 4;
+  char *f1_mbs = calloc(bytes, 1);
+  char *f2_mbs = calloc(bytes, 1);
+  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
+    return -1;
+  }
+  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
+    return -1;
+  }
+  int32_t ret = strcmp(f1_mbs, f2_mbs);
+  free(f1_mbs);
+  free(f2_mbs);
+  return ret;
+#endif
+}
diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt
index 27bcae943c..3280a37c94 100644
--- a/src/plugins/http/CMakeLists.txt
+++ b/src/plugins/http/CMakeLists.txt
@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(http ${SRC})
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index 225977abae..ca65f65608 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) {
 }
 
 bool httpInitContexts() {
-  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, false, httpDestroyContext, "restc");
+  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc");
   if (tsHttpServer.contextCache == NULL) {
     httpError("failed to init context cache");
     return false;
diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c
index 82666826bc..4748f03b66 100644
--- a/src/plugins/http/src/httpJson.c
+++ b/src/plugins/http/src/httpJson.c
@@ -45,7 +45,7 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) {
 
   do {
     if (pContext->fd > 2){
-      len = (int)send(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
+      len = (int)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
     }
     else {
       return sz;
diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt
index 0ea5507501..9a20286797 100644
--- a/src/plugins/monitor/CMakeLists.txt
+++ b/src/plugins/monitor/CMakeLists.txt
@@ -3,11 +3,11 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(./src SRC)
   ADD_LIBRARY(monitor ${SRC})
diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt
index cc88bd95a4..5fc8f9039f 100644
--- a/src/plugins/mqtt/CMakeLists.txt
+++ b/src/plugins/mqtt/CMakeLists.txt
@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(mqtt ${SRC})
diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt
index 620fd217de..625b85d908 100644
--- a/src/query/CMakeLists.txt
+++ b/src/query/CMakeLists.txt
@@ -1,9 +1,9 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h
index 00049b486d..01b4c16ac1 100644
--- a/src/query/inc/qAst.h
+++ b/src/query/inc/qAst.h
@@ -42,7 +42,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *);
 
 /**
  * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
- *
  */
 typedef struct tQueryInfo {
   uint8_t       optr;     // expression operator
diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index 127c38a6f8..bd2e0a4470 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -20,8 +20,8 @@
 #include "hash.h"
 #include "qFill.h"
 #include "qResultbuf.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "query.h"
 #include "taosdef.h"
 #include "tarray.h"
@@ -42,8 +42,8 @@ typedef struct SSqlGroupbyExpr {
 } SSqlGroupbyExpr;
 
 typedef struct SPosInfo {
-  int16_t pageId;
-  int16_t rowId;
+  int32_t pageId;
+  int32_t rowId;
 } SPosInfo;
 
 typedef struct SWindowStatus {
@@ -177,13 +177,18 @@ typedef struct SQueryRuntimeEnv {
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
 } SQueryRuntimeEnv;
 
+enum {
+  QUERY_RESULT_NOT_READY = 1,
+  QUERY_RESULT_READY     = 2,
+};
+
 typedef struct SQInfo {
   void*            signature;
   int32_t          pointsInterpo;
   int32_t          code;  // error code to returned to client
-  sem_t            dataReady;
+//  sem_t            dataReady;
+
   void*            tsdb;
-  void*            param;
   int32_t          vgId;
   STableGroupInfo  tableGroupInfo;       // table id list < only includes the STable list>
   STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
@@ -200,8 +205,11 @@ typedef struct SQInfo {
    */
   int32_t          tableIndex;
   int32_t          numOfGroupResultPages;
-  void*            pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
+  void*            pBuf;        // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
 
+  pthread_mutex_t  lock;        // used to synchronize the rsp/query threads
+  int32_t          dataReady;   // denote if query result is ready or not
+  void*            rspContext;  // response context
 } SQInfo;
 
 #endif  // TDENGINE_QUERYEXECUTOR_H
diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h
index b57c48933f..36fc0c9820 100644
--- a/src/query/inc/qExtbuffer.h
+++ b/src/query/inc/qExtbuffer.h
@@ -19,7 +19,6 @@
 extern "C" {
 #endif
 
-
 #include "os.h"
 #include "taosmsg.h"
 
@@ -28,9 +27,9 @@ extern "C" {
 #include "tdataformat.h"
 #include "talgo.h"
 
-#define DEFAULT_PAGE_SIZE (1024L*4)  // 16k larger than the SHistoInfo
-#define MAX_TMPFILE_PATH_LENGTH PATH_MAX
+#define MAX_TMPFILE_PATH_LENGTH        PATH_MAX
 #define INITIAL_ALLOCATION_BUFFER_SIZE 64
+#define DEFAULT_PAGE_SIZE              (4096L)  // 16k larger than the SHistoInfo
 
 typedef enum EXT_BUFFER_FLUSH_MODEL {
   /*
@@ -126,7 +125,7 @@ typedef struct tExtMemBuffer {
  * @param pModel
  * @return
  */
-tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel);
+tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel);
 
 /**
  *
diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h
index 8c8afb0957..5303251d98 100644
--- a/src/query/inc/qResultbuf.h
+++ b/src/query/inc/qResultbuf.h
@@ -13,50 +13,85 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODEQUERYUTIL_H
-#define TDENGINE_VNODEQUERYUTIL_H
+#ifndef TDENGINE_QRESULTBUF_H
+#define TDENGINE_QRESULTBUF_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <tlist.h>
 #include "hash.h"
 #include "os.h"
 #include "qExtbuffer.h"
+#include "tlockfree.h"
 
 typedef struct SArray* SIDList;
 
+typedef struct SPageDiskInfo {
+  int32_t offset;
+  int32_t length;
+} SPageDiskInfo;
+
+typedef struct SPageInfo {
+  SListNode*    pn;       // point to list node
+  int32_t       pageId;
+  SPageDiskInfo info;
+  void*         pData;
+  bool          used;     // set current page is in used
+} SPageInfo;
+
+typedef struct SFreeListItem {
+  int32_t offset;
+  int32_t len;
+} SFreeListItem;
+
+typedef struct SResultBufStatis {
+  int32_t flushBytes;
+  int32_t loadBytes;
+  int32_t getPages;
+  int32_t releasePages;
+  int32_t flushPages;
+} SResultBufStatis;
+
 typedef struct SDiskbasedResultBuf {
   int32_t   numOfRowsPerPage;
   int32_t   numOfPages;
   int64_t   totalBufSize;
-  int32_t   fd;                  // data file fd
+  int64_t   fileSize;            // disk file size
+  FILE*     file;
   int32_t   allocateId;          // allocated page id
-  int32_t   incStep;             // minimum allocated pages
-  void*     pBuf;                // mmap buffer pointer
   char*     path;                // file path
   int32_t   pageSize;            // current used page size
   int32_t   inMemPages;          // numOfPages that are allocated in memory
-  SHashObj* idsTable;            // id hash table
-  SIDList   list;                // for each id, there is a page id list
-
-  void*     iBuf;                // inmemory buf
-  void*     handle;              // for debug purpose
+  SHashObj* groupSet;            // id hash table
+  SHashObj* all;
+  SList*    lruList;
   void*     emptyDummyIdList;    // dummy id list
+  void*     assistBuf;           // assistant buffer for compress/decompress data
+  SArray*   pFree;               // free area in file
+  bool      comp;                // compressed before flushed to disk
+  int32_t   nextPos;             // next page flush position
+
+  const void*      handle;        // for debug purpose
+  SResultBufStatis statis;
 } SDiskbasedResultBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (4096L)
 #define DEFAULT_INMEM_BUF_PAGES       10
+#define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
 
 /**
  * create disk-based result buffer
  * @param pResultBuf
- * @param size
  * @param rowSize
+ * @param pagesize
+ * @param inMemPages
+ * @param handle
  * @return
  */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
-    int32_t inMemPages, void* handle);
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle);
 
 /**
  *
@@ -72,7 +107,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
@@ -88,42 +123,52 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
  * @param id
  * @return
  */
-static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
-  if (id < pResultBuf->inMemPages) {
-    return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize);
-  } else {
-    return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize);
-  }
-}
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
+
+/**
+ * release the referenced buf pages
+ * @param pResultBuf
+ * @param page
+ */
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
+
+/**
+ *
+ * @param pResultBuf
+ * @param pi
+ */
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
+
+
 /**
  * get the total buffer size in the format of disk file
  * @param pResultBuf
  * @return
  */
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf);
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * get the number of groups in the result buffer
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * destroy result buffer
  * @param pResultBuf
  */
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle);
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
  * @param pList
  * @return
  */
-int32_t getLastPageId(SIDList pList);
+SPageInfo* getLastPageInfo(SIDList pList);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_VNODEQUERYUTIL_H
+#endif  // TDENGINE_QRESULTBUF_H
diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qSqlparser.h
similarity index 99%
rename from src/query/inc/qsqlparser.h
rename to src/query/inc/qSqlparser.h
index 9e8706cf92..cd9618b223 100644
--- a/src/query/inc/qsqlparser.h
+++ b/src/query/inc/qSqlparser.h
@@ -119,6 +119,7 @@ typedef struct SCreateDBInfo {
   int32_t   fsyncPeriod;
   int64_t   commitTime;
   int32_t   walLevel;
+  int32_t   quorum;
   int32_t   compressionLevel;
   SSQLToken precision;
   bool      ignoreExists;
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 7119cb75fe..ed7c7e8845 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -45,13 +45,14 @@ bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
 int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
 
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) {
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+    tFilePage* page) {
   assert(pResult != NULL && pRuntimeEnv != NULL);
 
   SQuery    *pQuery = pRuntimeEnv->pQuery;
-  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
-  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
+//  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
 
+  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
   return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
       pQuery->pSelectExpr[columnIndex].bytes * realRowId;
 }
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index d709b4f838..15cfeee6b2 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -215,6 +215,7 @@ keep(Y)    ::= KEEP tagitemlist(X).           { Y = X; }
 tables(Y)  ::= MAXTABLES INTEGER(X).          { Y = X; }
 cache(Y)   ::= CACHE INTEGER(X).              { Y = X; }
 replica(Y) ::= REPLICA INTEGER(X).            { Y = X; }
+quorum(Y)  ::= QUORUM INTEGER(X).             { Y = X; }
 days(Y)    ::= DAYS INTEGER(X).               { Y = X; }
 minrows(Y) ::= MINROWS INTEGER(X).            { Y = X; }
 maxrows(Y) ::= MAXROWS INTEGER(X).            { Y = X; }
@@ -231,6 +232,7 @@ db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
 db_optr(Y) ::= db_optr(Z) tables(X).         { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) cache(X).          { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) replica(X).        { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
+db_optr(Y) ::= db_optr(Z) quorum(X).         { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) days(X).           { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) minrows(X).        { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); }
 db_optr(Y) ::= db_optr(Z) maxrows(X).        { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); }
@@ -246,6 +248,7 @@ db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
 alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
 
 alter_db_optr(Y) ::= alter_db_optr(Z) replica(X).     { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X).      { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) tables(X).      { Y = Z; Y.maxTablesPerVnode = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) keep(X).        { Y = Z; Y.keep = X; }
 alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X).      { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index 5ce9121cf1..58d2ff260e 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -22,7 +22,7 @@ extern "C" {
 
 #include "os.h"
 
-#include "../../common/inc/tname.h"
+#include "tname.h"
 #include "taosdef.h"
 #include "trpc.h"
 #include "tvariant.h"
diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c
index c2578c15c0..44051c7e3b 100644
--- a/src/query/src/qAst.c
+++ b/src/query/src/qAst.c
@@ -18,8 +18,8 @@
 
 #include "exception.h"
 #include "qAst.h"
+#include "qSqlparser.h"
 #include "qSyntaxtreefunction.h"
-#include "qsqlparser.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tarray.h"
@@ -878,28 +878,6 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
   free(pRightOutput);
 }
 
-//void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) {
-//  if (pExprs == NULL) {
-//    return;
-//  }
-//
-//  tExprNode *pLeft = pExprs->_node.pLeft;
-//  tExprNode *pRight = pExprs->_node.pRight;
-//
-//  // recursive traverse left child branch
-//  if (pLeft->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pLeft, res);
-//  } else if (pLeft->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pLeft->pSchema->colId);
-//  }
-//
-//  if (pRight->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pRight, res);
-//  } else if (pRight->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pRight->pSchema->colId);
-//  }
-//}
-
 static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
   tbufWriteUint8(bw, expr->nodeType);
   
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 8afc2fd87a..6efc8a827e 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -221,7 +221,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
 }
 
 static int32_t getGroupResultId(int32_t groupIndex) {
-  int32_t base = 200000;
+  int32_t base = 20000000;
   return base + (groupIndex * 10000);
 }
 
@@ -478,10 +478,14 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (taosArrayGetSize(list) == 0) {
     pData = getNewDataBuf(pResultBuf, sid, &pageId);
   } else {
-    pageId = getLastPageId(list);
-    pData = getResBufPage(pResultBuf, pageId);
+    SPageInfo* pi = getLastPageInfo(list);
+    pData = getResBufPage(pResultBuf, pi->pageId);
+    pageId = pi->pageId;
 
     if (pData->num >= numOfRowsPerPage) {
+      // release current page first, and prepare the next one
+      releaseResBufPageInfo(pResultBuf, pi);
+
       pData = getNewDataBuf(pResultBuf, sid, &pageId);
       if (pData != NULL) {
         assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
@@ -497,6 +501,8 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (pWindowRes->pos.pageId == -1) {  // not allocated yet, allocate new buffer
     pWindowRes->pos.pageId = pageId;
     pWindowRes->pos.rowId = pData->num++;
+
+    assert(pWindowRes->pos.pageId >= 0);
   }
 
   return 0;
@@ -1490,8 +1496,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     goto _clean;
   }
 
-  qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv));
-
   pRuntimeEnv->offset[0] = 0;
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base;
@@ -1536,8 +1540,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
       }
     }
 
-    qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv));
-
     // set the order information for top/bottom query
     int32_t functionId = pCtx->functionId;
 
@@ -1558,25 +1560,19 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     }
   }
 
-  qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv));
-
   char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
 
   // set the intermediate result output buffer
   setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
 
-  qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv));
-
   // if it is group by normal column, do not set output buffer, the output buffer is pResult
   if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) {
     resetCtxOutputBuf(pRuntimeEnv);
   }
 
-  qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv));
-
   setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx);
 
-  qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv));
+  qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv));
   return TSDB_CODE_SUCCESS;
 
 _clean:
@@ -1615,7 +1611,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
   pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
 
-  destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
+  destroyResultBuf(pRuntimeEnv->pResultBuf);
   tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
   tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
 
@@ -2111,9 +2107,6 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
     }
     
     if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) {
-#if defined(_DEBUG_VIEW)
-      qDebug("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv));
-#endif
       // current block has been discard due to filter applied
       pRuntimeEnv->summary.discardBlocks += 1;
       qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
@@ -2446,6 +2439,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (!mergeFlag) {
@@ -2458,7 +2453,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
 
     pCtx[i].hasNull = true;
     pCtx[i].nStartQueryTimestamp = timestamp;
-    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
 
     // in case of tag column, the tag information should be extracted from input buffer
     if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) {
@@ -2615,14 +2610,16 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
 
   SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
   SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos);
+  tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId);
 
-  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1);
+  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
   TSKEY leftTimestamp = GET_INT64_VAL(b1);
 
   SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
   SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos);
+  tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId);
 
-  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2);
+  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
   TSKEY rightTimestamp = GET_INT64_VAL(b2);
 
   if (leftTimestamp == rightTimestamp) {
@@ -2685,35 +2682,26 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   int32_t id = getGroupResultId(pQInfo->groupIndex - 1);
   SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id);
 
-  int32_t total = 0;
   int32_t size = taosArrayGetSize(list);
-  for (int32_t i = 0; i < size; ++i) {
-    int32_t* pgId = taosArrayGet(list, i);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
-    total += pData->num;
-  }
-
-  int32_t rows = total;
 
   int32_t offset = 0;
   for (int32_t j = 0; j < size; ++j) {
-    int32_t* pgId = taosArrayGet(list, j);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
+    SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
+    tFilePage *pData = getResBufPage(pResultBuf, pi->pageId);
 
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
       char *  pDest = pQuery->sdata[i]->data;
-
-      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num,
-             bytes * pData->num);
+      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num);
     }
 
+//    rows += pData->num;
     offset += pData->num;
   }
 
   assert(pQuery->rec.rows == 0);
 
-  pQuery->rec.rows += rows;
+  pQuery->rec.rows += offset;
   pQInfo->offset += 1;
 }
 
@@ -2777,7 +2765,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     assert(pQInfo->numOfGroupResultPages == 0);
     return 0;
   } else if (numOfTables == 1) { // no need to merge results since only one table in each group
-
   }
 
   SCompSupporter cs = {pTableList, posList, pQInfo};
@@ -2802,8 +2789,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
 
     SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
     SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
 
-    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes);
+    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
     TSKEY ts = GET_INT64_VAL(b);
 
     assert(ts == pWindowRes->window.skey);
@@ -3517,9 +3505,11 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page);
 
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
@@ -3542,6 +3532,8 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
 
@@ -3550,7 +3542,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
       continue;
     }
 
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage);
     pCtx->currentStage = 0;
 
     int32_t functionId = pCtx->functionId;
@@ -3713,11 +3705,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
       pQInfo->groupIndex += 1;
     }
 
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId);
+
     for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
       int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
 
       char *out = pQuery->sdata[j]->data + numOfResult * size;
-      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]);
+      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page);
       memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
     }
 
@@ -4238,10 +4232,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   int32_t ps = DEFAULT_PAGE_SIZE;
   int32_t rowsize = 0;
   getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
+  int32_t TWOMB = 1024*1024*2;
 
   if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) {
-    int32_t numOfPages = getInitialPageNum(pQInfo);
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -4269,8 +4263,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     int32_t numOfResultRows = getInitialPageNum(pQInfo);
     getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
-
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -5894,16 +5887,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   }
 
   pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+  pthread_mutex_init(&pQInfo->lock, NULL);
 
   pQuery->pos = -1;
   pQuery->window = pQueryMsg->window;
-
-  if (sem_init(&pQInfo->dataReady, 0, 0) != 0) {
-    int32_t code = TAOS_SYSTEM_ERROR(errno);
-    qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, tstrerror(code));
-    goto _cleanup;
-  }
-
   colIdCheck(pQuery);
 
   qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo);
@@ -5945,7 +5933,7 @@ static bool isValidQInfo(void *param) {
   return (sig == (uint64_t)pQInfo);
 }
 
-static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) {
+static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) {
   int32_t code = TSDB_CODE_SUCCESS;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
@@ -5965,18 +5953,12 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
            pQuery->window.ekey, pQuery->order.order);
     setQueryStatus(pQuery, QUERY_COMPLETED);
     pQInfo->tableqinfoGroupInfo.numOfTables = 0;
-
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
-  pQInfo->param = param;
-
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
     setQueryStatus(pQuery, QUERY_COMPLETED);
-  
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -6018,7 +6000,6 @@ static void freeQInfo(SQInfo *pQInfo) {
     tfree(pQuery->sdata[col]);
   }
 
-  sem_destroy(&(pQInfo->dataReady));
   teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
 
   for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
@@ -6170,7 +6151,7 @@ typedef struct SQueryMgmt {
   pthread_mutex_t lock;
 } SQueryMgmt;
 
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) {
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) {
   assert(pQueryMsg != NULL && tsdb != NULL);
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -6266,7 +6247,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo
     goto _over;
   }
 
-  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param);
+  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery);
 
 _over:
   free(tagCond);
@@ -6306,24 +6287,36 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
   freeQInfo(pQInfo);
 }
 
-void qTableQuery(qinfo_t qinfo) {
+static bool doBuildResCheck(SQInfo* pQInfo) {
+  bool buildRes = false;
+
+  pthread_mutex_lock(&pQInfo->lock);
+
+  pQInfo->dataReady = QUERY_RESULT_READY;
+  buildRes = (pQInfo->rspContext != NULL);
+
+  pthread_mutex_unlock(&pQInfo->lock);
+
+  return buildRes;
+}
+
+bool qTableQuery(qinfo_t qinfo) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || pQInfo->signature != pQInfo) {
     qDebug("QInfo:%p has been freed, no need to execute", pQInfo);
-    return;
+    return false;
   }
 
   if (IS_QUERY_KILLED(pQInfo)) {
     qDebug("QInfo:%p it is already killed, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    return doBuildResCheck(pQInfo);
   }
 
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table exists for query, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED);
+    return doBuildResCheck(pQInfo);
   }
 
   // error occurs, record the error code and return to client
@@ -6331,8 +6324,7 @@ void qTableQuery(qinfo_t qinfo) {
   if (ret != TSDB_CODE_SUCCESS) {
     pQInfo->code = ret;
     qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code));
-    sem_post(&pQInfo->dataReady);
-    return;
+    return doBuildResCheck(pQInfo);
   }
 
   qDebug("QInfo:%p query task is launched", pQInfo);
@@ -6357,10 +6349,10 @@ void qTableQuery(qinfo_t qinfo) {
            pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
   }
 
-  sem_post(&pQInfo->dataReady);
+  return doBuildResCheck(pQInfo);
 }
 
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6373,11 +6365,21 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
     return pQInfo->code;
   }
 
-  sem_wait(&pQInfo->dataReady);
-  qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
-         pQInfo->code);
+  int32_t code = TSDB_CODE_SUCCESS;
+  pthread_mutex_lock(&pQInfo->lock);
+  if (pQInfo->dataReady == QUERY_RESULT_READY) {
+    *buildRes = true;
+    qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
+           pQInfo->code);
+  } else {
+    *buildRes = false;
+    qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo);
+    pQInfo->rspContext = pRspContext;
+  }
 
-  return pQInfo->code;
+  code = pQInfo->code;
+  pthread_mutex_unlock(&pQInfo->lock);
+  return code;
 }
 
 bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
@@ -6389,6 +6391,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   }
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
   bool ret = false;
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     ret = false;
@@ -6407,7 +6410,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   return ret;
 }
 
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) {
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6417,8 +6420,10 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
+
   size += sizeof(int32_t);
   size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo);
+
   *contLen = size + sizeof(SRetrieveTableRsp);
 
   // todo proper handle failed to allocate memory,
@@ -6427,6 +6432,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   if (*pRsp == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
+
   (*pRsp)->numOfRows = htonl(pQuery->rec.rows);
 
   int32_t code = pQInfo->code;
@@ -6434,8 +6440,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->offset = htobe64(pQuery->limit.offset);
     (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime);
   } else {
-    (*pRsp)->offset = 0;
     (*pRsp)->useconds = 0;
+    (*pRsp)->offset = 0;
   }
   
   (*pRsp)->precision = htons(pQuery->precision);
@@ -6446,10 +6452,20 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     code = pQInfo->code;
   }
 
+  pQInfo->rspContext = NULL;
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     (*pRsp)->completed = 1;  // notify no more result to client
   }
 
+  if (qHasMoreResultsToRetrieve(pQInfo)) {
+    *continueExec = true;
+  } else { // failed to dump result, free qhandle immediately
+    *continueExec = false;
+    qKillQuery(pQInfo);
+  }
+
   return code;
 }
 
@@ -6460,7 +6476,6 @@ int32_t qKillQuery(qinfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  sem_post(&pQInfo->dataReady);
   setQueryKilled(pQInfo);
   return TSDB_CODE_SUCCESS;
 }
@@ -6607,6 +6622,13 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
   setQueryStatus(pQuery, QUERY_COMPLETED);
 }
 
+void* qGetResultRetrieveMsg(qinfo_t qinfo) {
+  SQInfo* pQInfo = (SQInfo*) qinfo;
+  assert(pQInfo != NULL);
+
+  return pQInfo->rspContext;
+}
+
 void freeqinfoFn(void *qhandle) {
   void** handle = qhandle;
   if (handle == NULL || *handle == NULL) {
@@ -6618,19 +6640,21 @@ void freeqinfoFn(void *qhandle) {
 }
 
 void* qOpenQueryMgmt(int32_t vgId) {
-  const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, refresh handle pool
+  const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool
 
   char cacheName[128] = {0};
   sprintf(cacheName, "qhandle_%d", vgId);
 
-  SQueryMgmt* pQueryHandle = calloc(1, sizeof(SQueryMgmt));
+  SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt));
 
-  pQueryHandle->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
-  pQueryHandle->closed    = false;
-  pthread_mutex_init(&pQueryHandle->lock, NULL);
+  pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
+  pQueryMgmt->closed    = false;
+  pQueryMgmt->vgId      = vgId;
+
+  pthread_mutex_init(&pQueryMgmt->lock, NULL);
 
   qDebug("vgId:%d, open querymgmt success", vgId);
-  return pQueryHandle;
+  return pQueryMgmt;
 }
 
 static void queryMgmtKillQueryFn(void* handle) {
@@ -6670,7 +6694,7 @@ void qCleanupQueryMgmt(void* pQMgmt) {
   pthread_mutex_destroy(&pQueryMgmt->lock);
   tfree(pQueryMgmt);
 
-  qDebug("vgId:%d querymgmt cleanup completed", vgId);
+  qDebug("vgId:%d queryMgmt cleanup completed", vgId);
 }
 
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
@@ -6716,14 +6740,15 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) {
   }
 }
 
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) {
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) {
   SQueryMgmt *pQueryMgmt = pMgmt;
 
   if (pQueryMgmt->qinfoPool == NULL) {
     return NULL;
   }
 
-  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, needFree);
+  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
   return 0;
 }
 
+
diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c
index 69c5f0e24f..fb57f71199 100644
--- a/src/query/src/qExtbuffer.c
+++ b/src/query/src/qExtbuffer.c
@@ -28,10 +28,10 @@
 /*
  * SColumnModel is deeply copy
  */
-tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel) {
+tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) {
   tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer));
 
-  pMemBuffer->pageSize = DEFAULT_PAGE_SIZE;
+  pMemBuffer->pageSize = pagesize;
   pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize;
   pMemBuffer->nElemSize = elemSize;
 
diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index ecc11f8f4d..7cf995193b 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -14,7 +14,7 @@
  */
 
 #include "os.h"
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "queryLog.h"
 #include "taosdef.h"
 #include "taosmsg.h"
@@ -907,6 +907,7 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
   pDBInfo->daysPerFile = -1;
 
   pDBInfo->replica = -1;
+  pDBInfo->quorum = -1;
   pDBInfo->keep = NULL;
 
   memset(&pDBInfo->precision, 0, sizeof(SSQLToken));
diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index c4490a01e7..85e45e46b3 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -535,7 +535,7 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) {
 
     if (pSeg->pBuffer[slotIdx] == NULL) {
       pSeg->pBuffer[slotIdx] = createExtMemBuffer(pBucket->numOfTotalPages * pBucket->pageSize, pBucket->nElemSize,
-                                                  pBucket->pOrderDesc->pColumnModel);
+                                                  pBucket->pageSize, pBucket->pOrderDesc->pColumnModel);
       pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL;
       pBucket->pOrderDesc->pColumnModel->capacity = pSeg->pBuffer[slotIdx]->numOfElemsPerPage;
     }
diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c
index de59676e59..93ccad8e27 100644
--- a/src/query/src/qResultbuf.c
+++ b/src/query/src/qResultbuf.c
@@ -1,221 +1,439 @@
 #include "qResultbuf.h"
+#include "stddef.h"
+#include "tscompression.h"
 #include "hash.h"
 #include "qExtbuffer.h"
 #include "queryLog.h"
 #include "taoserror.h"
 
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize,
-    int32_t pagesize, int32_t inMemPages, void* handle) {
+#define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES)
 
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle) {
   *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+
   SDiskbasedResultBuf* pResBuf = *pResultBuf;
   if (pResBuf == NULL) {
     return TSDB_CODE_COM_OUT_OF_MEMORY;  
   }
 
-  pResBuf->pageSize   = pagesize;
-  pResBuf->numOfPages = inMemPages;  // all pages are in buffer in the first place
-  pResBuf->inMemPages = inMemPages;
-  assert(inMemPages <= numOfPages);
+  pResBuf->pageSize     = pagesize;
+  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
+  pResBuf->totalBufSize = 0;
+  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
+  pResBuf->allocateId   = -1;
+  pResBuf->comp         = true;
+  pResBuf->file         = NULL;
+  pResBuf->handle       = handle;
+  pResBuf->fileSize = 0;
+
+  // at least more than 2 pages must be in memory
+  assert(inMemBufSize >= pagesize * 2);
 
   pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
-
-  pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
-  pResBuf->incStep = 4;
-  pResBuf->allocateId = -1;
-
-  pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize);
+  pResBuf->lruList = tdListNew(POINTER_BYTES);
 
   // init id hash table
-  pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
-  pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES);
+  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
+  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
 
   char path[PATH_MAX] = {0};
-  getTmpfilePath("tsdb_qbuf", path);
+  getTmpfilePath("qbuf", path);
   pResBuf->path = strdup(path);
 
-  pResBuf->fd = FD_INITIALIZER;
-  pResBuf->pBuf = NULL;
   pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
 
-  qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
-      pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize);
-  
+  qDebug("QInfo:%p create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", handle, pResBuf->pageSize,
+         pResBuf->inMemPages, pResBuf->path);
+
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); }
-
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
-
-#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages)
-#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
-
-static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
-  pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666);
-  if (!FD_VALID(pResultBuf->fd)) {
+static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
+  pResultBuf->file = fopen(pResultBuf->path, "wb+");
+  if (pResultBuf->file == NULL) {
     qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  assert(pResultBuf->numOfPages == pResultBuf->inMemPages);
-  pResultBuf->numOfPages += pResultBuf->incStep;
-
-  int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-  if (ret != TSDB_CODE_SUCCESS) {
-    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
-  }
-
-  pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
-  if (pResultBuf->pBuf == MAP_FAILED) {
-    qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
-  }
-
-  pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) {
-  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize);
-  int32_t ret = TSDB_CODE_SUCCESS;
+static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
 
-  if (pResultBuf->pBuf == NULL) {
-    assert(pResultBuf->fd == FD_INITIALIZER);
+  *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
 
-    if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
-      return ret;
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
+}
+
+static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
+
+  *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
+}
+
+static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
+  if (pResultBuf->pFree == NULL) {
+    return pResultBuf->nextPos;
+  } else {
+    int32_t offset = -1;
+
+    size_t num = taosArrayGetSize(pResultBuf->pFree);
+    for(int32_t i = 0; i < num; ++i) {
+      SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
+      if (pi->len >= size) {
+        offset = pi->offset;
+        pi->offset += size;
+        pi->len -= size;
+
+        return offset;
+      }
+    }
+
+    // no available recycle space, allocate new area in file
+    return pResultBuf->nextPos;
+  }
+}
+
+static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  assert(!pg->used && pg->pData != NULL);
+
+  int32_t size = -1;
+  char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
+
+  // this page is flushed to disk for the first time
+  if (pg->info.offset == -1) {
+    pg->info.offset = allocatePositionInFile(pResultBuf, size);
+    pResultBuf->nextPos += size;
+
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file);
+
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   } else {
-    ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->numOfPages += incNumOfPages;
+    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
+    if (pg->info.length < size) {
+      // 1. add current space to free list
+      taosArrayPush(pResultBuf->pFree, &pg->info);
 
-    /*
-     * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
-     * be insufficient
-     */
-    ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-    if (ret != TSDB_CODE_SUCCESS) {
-      //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
-      //           strerror(errno));
-      return TSDB_CODE_QRY_NO_DISKSPACE;
+      // 2. allocate new position, and update the info
+      pg->info.offset = allocatePositionInFile(pResultBuf, size);
+      pResultBuf->nextPos += size;
     }
 
-    pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
-    pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
+    //3. write to disk.
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    fwrite(t, size, 1, pResultBuf->file);
 
-    if (pResultBuf->pBuf == MAP_FAILED) {
-      //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
-      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  char* ret = pg->pData;
+  memset(ret, 0, pResultBuf->pageSize);
+
+  pg->pData = NULL;
+  pg->info.length = size;
+
+  pResultBuf->statis.flushBytes += pg->info.length;
+
+  return ret;
 }
 
-#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1)
+static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
 
-static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  assert(pResultBuf != NULL);
-
-  char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
-  if (p == NULL) {  // it is a new group id
-    return -1;
-  }
-
-  int32_t slot = GET_INT32_VAL(p);
-  assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable));
-
-  return slot;
-}
-
-static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
-  taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t));
-
-  SArray* pa = taosArrayInit(1, sizeof(int32_t));
-  taosArrayPush(pResultBuf->list, &pa);
-
-  assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable));
-  return num;
-}
-
-static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
-    slot = addNewGroupId(pResultBuf, groupId);
-  }
-
-  SIDList pList = taosArrayGetP(pResultBuf->list, slot);
-  taosArrayPush(pList, &pageId);
-}
-
-tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
-  if (NO_AVAILABLE_PAGES(pResultBuf)) {
-    if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
+  if (pResultBuf->file == NULL) {
+    if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
+      terrno = ret;
       return NULL;
     }
   }
 
-  // register new id in this group
-  *pageId = (++pResultBuf->allocateId);
-  registerPageId(pResultBuf, groupId, *pageId);
-
-  // clear memory for the new page
-  tFilePage* page = getResBufPage(pResultBuf, *pageId);
-  memset(page, 0, pResultBuf->pageSize);
-  
-  return page;
+  return doFlushPageToDisk(pResultBuf, pg);
 }
 
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+// load file block data in disk
+static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+  ret = fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
+  if (ret != pg->info.length) {
+    terrno = errno;
+    return NULL;
+  }
 
-SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
-    return pResultBuf->emptyDummyIdList;
+  pResultBuf->statis.loadBytes += pg->info.length;
+
+  int32_t fullSize = 0;
+  doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
+
+  return GET_DATA_PAYLOAD(pg);
+}
+
+#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages)
+
+static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
+
+  SArray* pa = taosArrayInit(1, POINTER_BYTES);
+  int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
+  assert(ret == 0);
+
+  return pa;
+}
+
+static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
+  SIDList list = NULL;
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    list = addNewGroup(pResultBuf, groupId);
   } else {
-    return taosArrayGetP(pResultBuf->list, slot);
+    list = (SIDList) (*p);
+  }
+
+  pResultBuf->numOfPages += 1;
+
+  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
+  ppi->info     = PAGE_INFO_INITIALIZER;
+  ppi->pageId   = pageId;
+  ppi->pData    = NULL;
+  ppi->pn       = NULL;
+  ppi->used     = true;
+
+  return *(SPageInfo**) taosArrayPush(list, &ppi);
+}
+
+static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
+  SListIter iter = {0};
+  tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
+
+  SListNode* pn = NULL;
+  while((pn = tdListNext(&iter)) != NULL) {
+    assert(pn != NULL);
+
+    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
+    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
+
+    if (!pageInfo->used) {
+      break;
+    }
+  }
+
+  return pn;
+}
+
+static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
+  char* bufPage = NULL;
+  SListNode* pn = getEldestUnrefedPage(pResultBuf);
+
+  // all pages are referenced by user, try to allocate new space
+  if (pn == NULL) {
+    int32_t prev = pResultBuf->inMemPages;
+    pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5;
+
+    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
+          pResultBuf->inMemPages, pResultBuf->pageSize);
+  } else {
+    pResultBuf->statis.flushPages += 1;
+    tdListPopNode(pResultBuf->lruList, pn);
+
+    SPageInfo* d = *(SPageInfo**) pn->data;
+    assert(d->pn == pn);
+
+    d->pn = NULL;
+    tfree(pn);
+
+    bufPage = flushPageToDisk(pResultBuf, d);
+  }
+
+  return bufPage;
+}
+
+static void lruListPushFront(SList *pList, SPageInfo* pi) {
+  tdListPrepend(pList, &pi);
+  SListNode* front = tdListGetHead(pList);
+  pi->pn = front;
+}
+
+static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
+  tdListPopNode(pList, pi->pn);
+  tdListPrependNode(pList, pi->pn);
+}
+
+tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
+  pResultBuf->statis.getPages += 1;
+
+  char* availablePage = NULL;
+  if (NO_AVAILABLE_PAGES(pResultBuf)) {
+    availablePage = evicOneDataPage(pResultBuf);
+  }
+
+  // register new id in this group
+  *pageId = (++pResultBuf->allocateId);
+
+  // register page id info
+  SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
+
+  // add to LRU list
+  assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
+
+  lruListPushFront(pResultBuf->lruList, pi);
+
+  // add to hash map
+  taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
+
+  // allocate buf
+  if (availablePage == NULL) {
+    pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+  } else {
+    pi->pData = availablePage;
+  }
+
+  pResultBuf->totalBufSize += pResultBuf->pageSize;
+
+  ((void**)pi->pData)[0] = pi;
+  pi->used = true;
+
+  return GET_DATA_PAYLOAD(pi);
+}
+
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
+  assert(pResultBuf != NULL && id >= 0);
+  pResultBuf->statis.getPages += 1;
+
+  SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
+  assert(pi != NULL && *pi != NULL);
+
+  if ((*pi)->pData != NULL) { // it is in memory
+    // no need to update the LRU list if only one page exists
+    if (pResultBuf->numOfPages == 1) {
+      (*pi)->used = true;
+      return GET_DATA_PAYLOAD(*pi);
+    }
+
+    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
+    assert(*pInfo == *pi);
+
+    lruListMoveToFront(pResultBuf->lruList, (*pi));
+    (*pi)->used = true;
+
+    return GET_DATA_PAYLOAD(*pi);
+
+  } else { // not in memory
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
+
+    char* availablePage = NULL;
+    if (NO_AVAILABLE_PAGES(pResultBuf)) {
+      availablePage = evicOneDataPage(pResultBuf);
+    }
+
+    if (availablePage == NULL) {
+      (*pi)->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+    } else {
+      (*pi)->pData = availablePage;
+    }
+
+    ((void**)((*pi)->pData))[0] = (*pi);
+
+    lruListPushFront(pResultBuf->lruList, *pi);
+    loadPageFromDisk(pResultBuf, *pi);
+    return GET_DATA_PAYLOAD(*pi);
   }
 }
 
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
+  assert(pResultBuf != NULL && page != NULL);
+  char* p = (char*) page - POINTER_BYTES;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  releaseResBufPageInfo(pResultBuf, ppi);
+}
+
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
+  assert(pi->pData != NULL && pi->used);
+
+  pi->used = false;
+  pResultBuf->statis.releasePages += 1;
+}
+
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
+
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
+
+SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(pResultBuf != NULL);
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    return pResultBuf->emptyDummyIdList;
+  } else {
+    return (SArray*) (*p);
+  }
+}
+
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   if (pResultBuf == NULL) {
     return;
   }
 
-  if (FD_VALID(pResultBuf->fd)) {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
-        pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
+  if (pResultBuf->file != NULL) {
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes",
+        pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize);
 
-    close(pResultBuf->fd);
-    munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->pBuf = NULL;
+    fclose(pResultBuf->file);
   } else {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
            pResultBuf->totalBufSize);
   }
 
   unlink(pResultBuf->path);
   tfree(pResultBuf->path);
 
-  size_t size = taosArrayGetSize(pResultBuf->list);
-  for (int32_t i = 0; i < size; ++i) {
-    SArray* pa = taosArrayGetP(pResultBuf->list, i);
-    taosArrayDestroy(pa);
+  SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
+  while(taosHashIterNext(iter)) {
+    SArray** p = (SArray**) taosHashIterGet(iter);
+    size_t n = taosArrayGetSize(*p);
+    for(int32_t i = 0; i < n; ++i) {
+      SPageInfo* pi = taosArrayGetP(*p, i);
+      tfree(pi->pData);
+      tfree(pi);
+    }
+
+    taosArrayDestroy(*p);
   }
 
-  taosArrayDestroy(pResultBuf->list);
-  taosArrayDestroy(pResultBuf->emptyDummyIdList);
-  taosHashCleanup(pResultBuf->idsTable);
+  taosHashDestroyIter(iter);
 
-  tfree(pResultBuf->iBuf);
+  tdListFree(pResultBuf->lruList);
+  taosArrayDestroy(pResultBuf->emptyDummyIdList);
+  taosHashCleanup(pResultBuf->groupSet);
+  taosHashCleanup(pResultBuf->all);
+
+  tfree(pResultBuf->assistBuf);
   tfree(pResultBuf);
 }
 
-int32_t getLastPageId(SIDList pList) {
+SPageInfo* getLastPageInfo(SIDList pList) {
   size_t size = taosArrayGetSize(pList);
-  return *(int32_t*) taosArrayGet(pList, size - 1);
+  return (SPageInfo*) taosArrayGetP(pList, size - 1);
 }
 
diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c
index 0ea0ff7bf3..b54dc32fb3 100644
--- a/src/query/src/qTokenizer.c
+++ b/src/query/src/qTokenizer.c
@@ -116,6 +116,7 @@ static SKeyword keywordTable[] = {
     {"STATE",        TK_STATE},
     {"KEEP",         TK_KEEP},
     {"REPLICA",      TK_REPLICA},
+    {"QUORUM",       TK_QUORUM},
     {"DAYS",         TK_DAYS},
     {"MINROWS",      TK_MINROWS},
     {"MAXROWS",      TK_MAXROWS},
diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c
index 20b29107f5..90cf394035 100644
--- a/src/query/src/qTsbuf.c
+++ b/src/query/src/qTsbuf.c
@@ -740,7 +740,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
   int32_t size = pSrcBuf->fileSize - offset;
 
 #ifdef LINUX
-  ssize_t rc = tsendfile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
+  ssize_t rc = taosTSendFile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
 #else
   ssize_t rc = fsendfile(pDestBuf->f, pSrcBuf->f, &offset, size);
 #endif
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 7b03dd9d94..c9143b3a53 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -236,11 +236,13 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
   if (pWindowRes == NULL) {
     return;
   }
-  
+
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
     SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
     
-    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
     size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     memset(s, 0, size);
     
@@ -277,8 +279,11 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
     memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
     
     // copy the output buffer data from src to dst, the position info keep unchanged
-    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst);
-    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src);
+    tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId);
+    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);
+
+    tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId);
+    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
     size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     
     memcpy(dstBuf, srcBuf, s);
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index ac9952bb97..fe26f61725 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -30,7 +30,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stdbool.h>
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "tcmdtype.h"
 #include "tstoken.h"
 #include "ttokendef.h"
@@ -97,26 +97,26 @@
 #endif
 /************* Begin control #defines *****************************************/
 #define YYCODETYPE unsigned short int
-#define YYNOCODE 272
+#define YYNOCODE 274
 #define YYACTIONTYPE unsigned short int
 #define ParseTOKENTYPE SSQLToken
 typedef union {
   int yyinit;
   ParseTOKENTYPE yy0;
-  SSubclauseInfo* yy25;
-  tSQLExpr* yy66;
-  SCreateAcctSQL yy73;
-  int yy82;
-  SQuerySQL* yy150;
-  SCreateDBInfo yy158;
-  TAOS_FIELD yy181;
-  SLimitVal yy188;
-  tSQLExprList* yy224;
-  int64_t yy271;
-  tVariant yy312;
-  SCreateTableSQL* yy374;
-  tFieldList* yy449;
-  tVariantList* yy494;
+  int yy46;
+  tSQLExpr* yy64;
+  tVariant yy134;
+  SCreateAcctSQL yy149;
+  int64_t yy207;
+  SLimitVal yy216;
+  TAOS_FIELD yy223;
+  SSubclauseInfo* yy231;
+  SCreateDBInfo yy268;
+  tSQLExprList* yy290;
+  SQuerySQL* yy414;
+  SCreateTableSQL* yy470;
+  tVariantList* yy498;
+  tFieldList* yy523;
 } YYMINORTYPE;
 #ifndef YYSTACKDEPTH
 #define YYSTACKDEPTH 100
@@ -126,17 +126,17 @@ typedef union {
 #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo
 #define ParseARG_STORE yypParser->pInfo = pInfo
 #define YYFALLBACK 1
-#define YYNSTATE             242
-#define YYNRULE              223
-#define YYNTOKEN             206
-#define YY_MAX_SHIFT         241
-#define YY_MIN_SHIFTREDUCE   401
-#define YY_MAX_SHIFTREDUCE   623
-#define YY_ERROR_ACTION      624
-#define YY_ACCEPT_ACTION     625
-#define YY_NO_ACTION         626
-#define YY_MIN_REDUCE        627
-#define YY_MAX_REDUCE        849
+#define YYNSTATE             243
+#define YYNRULE              226
+#define YYNTOKEN             207
+#define YY_MAX_SHIFT         242
+#define YY_MIN_SHIFTREDUCE   405
+#define YY_MAX_SHIFTREDUCE   630
+#define YY_ERROR_ACTION      631
+#define YY_ACCEPT_ACTION     632
+#define YY_NO_ACTION         633
+#define YY_MIN_REDUCE        634
+#define YY_MAX_REDUCE        859
 /************* End control #defines *******************************************/
 
 /* Define the yytestcase() macro to be a no-op if is not already defined
@@ -202,217 +202,218 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (547)
+#define YY_ACTTAB_COUNT (552)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */   727,  442,  726,   11,  725,  625,  241,  508,  728,  443,
- /*    10 */   730,  731,  729,   41,   43,  524,   35,   36,  521,  134,
- /*    20 */   522,   29,  523,  442,  197,   39,   37,   40,   38,  153,
- /*    30 */   239,  443,  219,   34,   33,  217,  216,   32,   31,   30,
- /*    40 */    41,   43,  757,   35,   36,  139,  170,  171,   29,  135,
- /*    50 */    21,  197,   39,   37,   40,   38,  182,  833,  158,  837,
- /*    60 */    34,   33,  744,  768,   32,   31,   30,  402,  403,  404,
- /*    70 */   405,  406,  407,  408,  409,  410,  411,  412,  413,  240,
- /*    80 */    41,   43,  846,   35,   36,  742,   60,  135,   29,  135,
- /*    90 */    21,  197,   39,   37,   40,   38,  157,  837,   27,  836,
- /*   100 */    34,   33,   56,  228,   32,   31,   30,  103,   43,    8,
- /*   110 */    35,   36,   61,  113,  765,   29,  757,  525,  197,   39,
- /*   120 */    37,   40,   38,  166,  537,  743,  579,   34,   33,   18,
- /*   130 */   154,   32,   31,   30,   16,  234,  208,  233,  207,  206,
- /*   140 */   205,  232,  204,  231,  230,  229,  203,  723,  168,  711,
- /*   150 */   712,  713,  714,  715,  716,  717,  718,  719,  720,  721,
- /*   160 */   722,   35,   36,  792,  103,  192,   29,  175,  155,  197,
- /*   170 */    39,   37,   40,   38,  179,  178,   21,  581,   34,   33,
- /*   180 */   442,   12,   32,   31,   30,  162,  592,  746,  443,  583,
- /*   190 */    17,  586,   76,  589,  103,  162,  592,   26,  228,  583,
- /*   200 */   148,  586,   99,  589,  103,   21,   88,   87,  142,  167,
- /*   210 */   165,  743,  169,  582,  147,  214,  213,  159,  160,   50,
- /*   220 */   791,  196,   74,   78,   83,   86,   77,  159,  160,  746,
- /*   230 */   235,  540,   80,  162,  592,   17,   51,  583,  215,  586,
- /*   240 */   743,  589,   26,   39,   37,   40,   38,  832,  194,  746,
- /*   250 */    58,   34,   33,   47,  185,   32,   31,   30,  666,  831,
- /*   260 */    59,  126,  181,  560,  561,  159,  160,   16,  234,  150,
- /*   270 */   233,   21,   48,  585,  232,  588,  231,  230,  229,   34,
- /*   280 */    33,  745,   42,   32,   31,   30,  116,  117,   68,   64,
- /*   290 */    67,  151,   42,  591,   32,   31,   30,  130,  128,   91,
- /*   300 */    90,   89,  675,  591,  220,  126,  743,   98,  590,  238,
- /*   310 */   237,   95,  667,  152,   26,  126,  532,  584,  590,  587,
- /*   320 */   551,  161,  552,  184,   46,  609,   14,  593,  514,   13,
- /*   330 */    42,   13,   46,  513,  201,   73,   72,   22,   22,   10,
- /*   340 */     9,  591,  528,  526,  529,  527,   85,   84,  802,  140,
- /*   350 */   141,  143,  801,  144,  163,  145,  590,  146,  798,  137,
- /*   360 */     3,  133,  138,  136,  797,  164,  759,  737,  218,  767,
- /*   370 */   100,  784,  783,  114,  112,  115,  677,  202,  131,   24,
- /*   380 */   211,  674,  212,  845,   70,  844,   26,  842,  118,   93,
- /*   390 */   695,   25,  547,   23,  186,  132,  664,   79,  662,   81,
- /*   400 */    82,  660,  659,  172,  190,  127,  657,  656,  655,  654,
- /*   410 */   653,  645,  129,  651,  649,  647,   52,   49,  756,  771,
- /*   420 */   104,  772,   44,  785,  195,  193,  191,  189,  187,   28,
- /*   430 */   210,   75,  221,  222,  223,  224,  225,  226,  199,  227,
- /*   440 */   236,   53,  623,  174,  173,  622,   62,  177,  149,  621,
- /*   450 */   183,   65,  176,  614,  658,  180,   92,   94,  652,  121,
- /*   460 */   125,  534,  120,  696,  741,  119,  110,  107,  105,  106,
- /*   470 */   122,  108,  123,    2,  111,  109,  124,    1,  184,   57,
- /*   480 */    55,  548,  156,  101,  188,    5,  553,  102,    6,   63,
- /*   490 */   594,   19,    4,   20,   15,  198,    7,  200,  483,  479,
- /*   500 */   477,  476,  475,  473,  446,  209,   66,   45,   69,   71,
- /*   510 */    22,  510,  509,  507,   54,  467,  465,  457,  463,  459,
- /*   520 */   461,  455,  453,  482,  481,  480,  478,  474,  472,   46,
- /*   530 */   444,  417,  415,  627,  626,  626,  626,  626,  626,  626,
- /*   540 */   626,  626,  626,  626,  626,   96,   97,
+ /*     0 */   103,  446,  135,  673,  632,  242,  126,  515,  135,  447,
+ /*    10 */   135,  158,  847,   41,   43,   11,   35,   36,  846,  157,
+ /*    20 */   847,   29,  134,  446,  197,   39,   37,   40,   38,  155,
+ /*    30 */   103,  447,  139,   34,   33,  217,  216,   32,   31,   30,
+ /*    40 */    41,   43,  767,   35,   36,   32,   31,   30,   29,  756,
+ /*    50 */   446,  197,   39,   37,   40,   38,  182,  802,  447,  192,
+ /*    60 */    34,   33,   21,   21,   32,   31,   30,  406,  407,  408,
+ /*    70 */   409,  410,  411,  412,  413,  414,  415,  416,  417,  241,
+ /*    80 */    41,   43,  228,   35,   36,  194,  843,   58,   29,   21,
+ /*    90 */   842,  197,   39,   37,   40,   38,  166,  167,  753,  753,
+ /*   100 */    34,   33,  168,   56,   32,   31,   30,  778,  841,   16,
+ /*   110 */   235,  208,  234,  233,  207,  206,  205,  232,  204,  231,
+ /*   120 */   230,  229,  203,  215,  151,  753,  732,  586,  719,  720,
+ /*   130 */   721,  722,  723,  724,  725,  726,  727,  728,  729,  730,
+ /*   140 */   731,   43,    8,   35,   36,   61,  113,   21,   29,  153,
+ /*   150 */   240,  197,   39,   37,   40,   38,  239,  238,   95,  775,
+ /*   160 */    34,   33,  165,   99,   32,   31,   30,  169,   35,   36,
+ /*   170 */   214,  213,  592,   29,  595,  103,  197,   39,   37,   40,
+ /*   180 */    38,  220,  756,  753,  236,   34,   33,  175,   12,   32,
+ /*   190 */    31,   30,  162,  599,  179,  178,  590,  767,  593,  103,
+ /*   200 */   596,  161,  162,  599,  756,   17,  590,  148,  593,  152,
+ /*   210 */   596,  154,   26,   88,   87,  142,  185,  567,  568,   16,
+ /*   220 */   235,  147,  234,  233,  159,  160,  219,  232,  196,  231,
+ /*   230 */   230,  229,  801,   76,  159,  160,  162,  599,  547,  228,
+ /*   240 */   590,    3,  593,   17,  596,   74,   78,   83,   86,   77,
+ /*   250 */    26,   39,   37,   40,   38,   80,   59,  754,   21,   34,
+ /*   260 */    33,  544,   60,   32,   31,   30,   18,  140,  159,  160,
+ /*   270 */   181,  737,  539,  736,   27,  734,  735,  150,  682,  184,
+ /*   280 */   738,  126,  740,  741,  739,  674,  531,  141,  126,  528,
+ /*   290 */    42,  529,  558,  530,  752,  591,   46,  594,   34,   33,
+ /*   300 */    42,  598,   32,   31,   30,  116,  117,   68,   64,   67,
+ /*   310 */   588,  598,  143,   50,   73,   72,  597,  170,  171,  130,
+ /*   320 */   128,   91,   90,   89,   98,   47,  597,  144,  559,  616,
+ /*   330 */    51,   26,   14,   13,   42,  145,  600,  521,  520,  201,
+ /*   340 */    13,   46,   22,   22,   48,  598,  589,   10,    9,  535,
+ /*   350 */   533,  536,  534,   85,   84,  146,  137,  133,  856,  138,
+ /*   360 */   597,  136,  755,  812,  811,  163,  808,  807,  164,  777,
+ /*   370 */   747,  218,  794,  100,  793,  769,  114,  115,   26,  684,
+ /*   380 */   112,  202,  131,  183,   24,  211,  681,  212,  855,  532,
+ /*   390 */    70,  854,   93,  852,  118,  702,  554,   25,   23,  132,
+ /*   400 */   671,   79,   52,  186,  669,   81,   82,  667,  666,  172,
+ /*   410 */   127,  664,  190,  663,  662,  661,  660,  652,  129,  658,
+ /*   420 */    49,  656,  654,  766,  781,  782,  795,  104,  195,   44,
+ /*   430 */   193,  191,  189,  187,  210,  105,   75,   28,  221,  199,
+ /*   440 */   222,  223,   53,  225,  224,  149,  226,   62,   65,  703,
+ /*   450 */   227,  237,  630,  173,  174,  629,  177,  665,  628,  119,
+ /*   460 */   176,   92,  121,  125,  120,  751,  122,  123,  659,  124,
+ /*   470 */   108,  106,  107,  109,  110,  111,   94,    1,    2,  621,
+ /*   480 */   180,  184,  541,   55,   57,  555,  101,  156,  188,  198,
+ /*   490 */    19,   63,    5,  560,  102,    4,    6,  601,   20,   15,
+ /*   500 */     7,  488,  484,  200,  482,  481,  480,  477,  450,  209,
+ /*   510 */    66,   45,   22,   69,   71,  517,  516,  514,  471,   54,
+ /*   520 */   469,  461,  467,  463,  465,  459,  457,  487,  486,  485,
+ /*   530 */   483,  479,  478,  476,   46,  448,  421,  419,  634,  633,
+ /*   540 */   633,  633,  633,  633,  633,  633,  633,  633,  633,  633,
+ /*   550 */    96,   97,
 };
 static const YYCODETYPE yy_lookahead[] = {
- /*     0 */   226,    1,  228,  260,  230,  207,  208,    5,  234,    9,
- /*    10 */   236,  237,  238,   13,   14,    2,   16,   17,    5,  260,
- /*    20 */     7,   21,    9,    1,   24,   25,   26,   27,   28,  209,
- /*    30 */   210,    9,  210,   33,   34,   33,   34,   37,   38,   39,
- /*    40 */    13,   14,  244,   16,   17,  260,   33,   34,   21,  260,
- /*    50 */   210,   24,   25,   26,   27,   28,  258,  260,  269,  270,
- /*    60 */    33,   34,  240,  210,   37,   38,   39,   45,   46,   47,
+ /*     0 */   211,    1,  262,  215,  208,  209,  218,    5,  262,    9,
+ /*    10 */   262,  271,  272,   13,   14,  262,   16,   17,  272,  271,
+ /*    20 */   272,   21,  262,    1,   24,   25,   26,   27,   28,  228,
+ /*    30 */   211,    9,  262,   33,   34,   33,   34,   37,   38,   39,
+ /*    40 */    13,   14,  246,   16,   17,   37,   38,   39,   21,  248,
+ /*    50 */     1,   24,   25,   26,   27,   28,  260,  268,    9,  270,
+ /*    60 */    33,   34,  211,  211,   37,   38,   39,   45,   46,   47,
  /*    70 */    48,   49,   50,   51,   52,   53,   54,   55,   56,   57,
- /*    80 */    13,   14,  246,   16,   17,  245,  247,  260,   21,  260,
- /*    90 */   210,   24,   25,   26,   27,   28,  269,  270,  259,  270,
- /*   100 */    33,   34,  102,   78,   37,   38,   39,  210,   14,   98,
- /*   110 */    16,   17,  101,  102,  261,   21,  244,  104,   24,   25,
- /*   120 */    26,   27,   28,  243,  103,  245,   99,   33,   34,  108,
- /*   130 */   258,   37,   38,   39,   85,   86,   87,   88,   89,   90,
- /*   140 */    91,   92,   93,   94,   95,   96,   97,  226,   63,  228,
- /*   150 */   229,  230,  231,  232,  233,  234,  235,  236,  237,  238,
- /*   160 */   239,   16,   17,  266,  210,  268,   21,  126,  227,   24,
- /*   170 */    25,   26,   27,   28,  133,  134,  210,    1,   33,   34,
- /*   180 */     1,   44,   37,   38,   39,    1,    2,  246,    9,    5,
- /*   190 */    98,    7,   72,    9,  210,    1,    2,  105,   78,    5,
- /*   200 */    63,    7,  210,    9,  210,  210,   69,   70,   71,  243,
- /*   210 */   227,  245,  127,   37,   77,  130,  131,   33,   34,  103,
- /*   220 */   266,   37,   64,   65,   66,   67,   68,   33,   34,  246,
- /*   230 */   227,   37,   74,    1,    2,   98,  120,    5,  243,    7,
- /*   240 */   245,    9,  105,   25,   26,   27,   28,  260,  264,  246,
- /*   250 */   266,   33,   34,  103,  262,   37,   38,   39,  214,  260,
- /*   260 */   266,  217,  125,  115,  116,   33,   34,   85,   86,  132,
- /*   270 */    88,  210,  122,    5,   92,    7,   94,   95,   96,   33,
- /*   280 */    34,  246,   98,   37,   38,   39,   64,   65,   66,   67,
- /*   290 */    68,  260,   98,  109,   37,   38,   39,   64,   65,   66,
- /*   300 */    67,   68,  214,  109,  243,  217,  245,   98,  124,   60,
- /*   310 */    61,   62,  214,  260,  105,  217,   99,    5,  124,    7,
- /*   320 */    99,   59,   99,  106,  103,   99,  103,   99,   99,  103,
- /*   330 */    98,  103,  103,   99,   99,  128,  129,  103,  103,  128,
- /*   340 */   129,  109,    5,    5,    7,    7,   72,   73,  241,  260,
- /*   350 */   260,  260,  241,  260,  241,  260,  124,  260,  241,  260,
- /*   360 */    98,  260,  260,  260,  241,  241,  244,  242,  241,  210,
- /*   370 */   210,  267,  267,  210,  248,  210,  210,  210,  210,  210,
- /*   380 */   210,  210,  210,  210,  210,  210,  105,  210,  210,   59,
- /*   390 */   210,  210,  109,  210,  263,  210,  210,  210,  210,  210,
- /*   400 */   210,  210,  210,  210,  263,  210,  210,  210,  210,  210,
- /*   410 */   210,  210,  210,  210,  210,  210,  119,  121,  257,  211,
- /*   420 */   256,  211,  118,  211,  113,  117,  112,  111,  110,  123,
- /*   430 */    75,   84,   83,   49,   80,   82,   53,   81,  211,   79,
- /*   440 */    75,  211,    5,    5,  135,    5,  215,    5,  211,    5,
- /*   450 */   244,  215,  135,   87,  211,  126,  212,  212,  211,  219,
- /*   460 */   218,   99,  223,  225,  244,  224,  250,  253,  255,  254,
- /*   470 */   222,  252,  220,  213,  249,  251,  221,  216,  106,  103,
- /*   480 */   107,   99,    1,   98,   98,  114,   99,   98,  114,   72,
- /*   490 */    99,  103,   98,  103,   98,  100,   98,  100,    9,    5,
- /*   500 */     5,    5,    5,    5,   76,   15,   72,   16,  129,  129,
- /*   510 */   103,    5,    5,   99,   98,    5,    5,    5,    5,    5,
- /*   520 */     5,    5,    5,    5,    5,    5,    5,    5,    5,  103,
- /*   530 */    76,   59,   58,    0,  271,  271,  271,  271,  271,  271,
- /*   540 */   271,  271,  271,  271,  271,   21,   21,  271,  271,  271,
- /*   550 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   560 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   570 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   580 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   590 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   600 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   610 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   620 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   630 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   640 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   650 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   660 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   670 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   680 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   690 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   700 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   710 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   720 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   730 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   740 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   750 */   271,  271,  271,
+ /*    80 */    13,   14,   78,   16,   17,  266,  262,  268,   21,  211,
+ /*    90 */   262,   24,   25,   26,   27,   28,  245,  245,  247,  247,
+ /*   100 */    33,   34,   63,  103,   37,   38,   39,  211,  262,   85,
+ /*   110 */    86,   87,   88,   89,   90,   91,   92,   93,   94,   95,
+ /*   120 */    96,   97,   98,  245,  262,  247,  227,  100,  229,  230,
+ /*   130 */   231,  232,  233,  234,  235,  236,  237,  238,  239,  240,
+ /*   140 */   241,   14,   99,   16,   17,  102,  103,  211,   21,  210,
+ /*   150 */   211,   24,   25,   26,   27,   28,   60,   61,   62,  263,
+ /*   160 */    33,   34,  228,  211,   37,   38,   39,  128,   16,   17,
+ /*   170 */   131,  132,    5,   21,    7,  211,   24,   25,   26,   27,
+ /*   180 */    28,  245,  248,  247,  228,   33,   34,  127,   44,   37,
+ /*   190 */    38,   39,    1,    2,  134,  135,    5,  246,    7,  211,
+ /*   200 */     9,   59,    1,    2,  248,   99,    5,   63,    7,  262,
+ /*   210 */     9,  260,  106,   69,   70,   71,  264,  116,  117,   85,
+ /*   220 */    86,   77,   88,   89,   33,   34,  211,   93,   37,   95,
+ /*   230 */    96,   97,  268,   72,   33,   34,    1,    2,   37,   78,
+ /*   240 */     5,   99,    7,   99,    9,   64,   65,   66,   67,   68,
+ /*   250 */   106,   25,   26,   27,   28,   74,  268,  242,  211,   33,
+ /*   260 */    34,  104,  249,   37,   38,   39,  109,  262,   33,   34,
+ /*   270 */   126,  227,  100,  229,  261,  231,  232,  133,  215,  107,
+ /*   280 */   236,  218,  238,  239,  240,  215,    2,  262,  218,    5,
+ /*   290 */    99,    7,  100,    9,  247,    5,  104,    7,   33,   34,
+ /*   300 */    99,  110,   37,   38,   39,   64,   65,   66,   67,   68,
+ /*   310 */     1,  110,  262,  104,  129,  130,  125,   33,   34,   64,
+ /*   320 */    65,   66,   67,   68,   99,  104,  125,  262,  100,  100,
+ /*   330 */   121,  106,  104,  104,   99,  262,  100,  100,  100,  100,
+ /*   340 */   104,  104,  104,  104,  123,  110,   37,  129,  130,    5,
+ /*   350 */     5,    7,    7,   72,   73,  262,  262,  262,  248,  262,
+ /*   360 */   125,  262,  248,  243,  243,  243,  243,  243,  243,  211,
+ /*   370 */   244,  243,  269,  211,  269,  246,  211,  211,  106,  211,
+ /*   380 */   250,  211,  211,  246,  211,  211,  211,  211,  211,  105,
+ /*   390 */   211,  211,   59,  211,  211,  211,  110,  211,  211,  211,
+ /*   400 */   211,  211,  120,  265,  211,  211,  211,  211,  211,  211,
+ /*   410 */   211,  211,  265,  211,  211,  211,  211,  211,  211,  211,
+ /*   420 */   122,  211,  211,  259,  212,  212,  212,  258,  114,  119,
+ /*   430 */   118,  113,  112,  111,   75,  257,   84,  124,   83,  212,
+ /*   440 */    49,   80,  212,   53,   82,  212,   81,  216,  216,  226,
+ /*   450 */    79,   75,    5,  136,    5,    5,    5,  212,    5,  225,
+ /*   460 */   136,  213,  220,  219,  224,  246,  223,  221,  212,  222,
+ /*   470 */   254,  256,  255,  253,  252,  251,  213,  217,  214,   87,
+ /*   480 */   127,  107,  100,  108,  104,  100,   99,    1,   99,  101,
+ /*   490 */   104,   72,  115,  100,   99,   99,  115,  100,  104,   99,
+ /*   500 */    99,    9,    5,  101,    5,    5,    5,    5,   76,   15,
+ /*   510 */    72,   16,  104,  130,  130,    5,    5,  100,    5,   99,
+ /*   520 */     5,    5,    5,    5,    5,    5,    5,    5,    5,    5,
+ /*   530 */     5,    5,    5,    5,  104,   76,   59,   58,    0,  273,
+ /*   540 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   550 */    21,   21,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   560 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   570 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   580 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   590 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   600 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   610 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   620 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   630 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   640 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   650 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   660 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   670 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   680 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   690 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   700 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   710 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   720 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   730 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   740 */   273,  273,  273,  273,  273,  273,  273,  273,  273,  273,
+ /*   750 */   273,  273,  273,  273,  273,  273,  273,  273,  273,
 };
-#define YY_SHIFT_COUNT    (241)
+#define YY_SHIFT_COUNT    (242)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (533)
+#define YY_SHIFT_MAX      (538)
 static const unsigned short int yy_shift_ofst[] = {
- /*     0 */   137,   49,  182,  184,  232,  179,  179,  179,  179,  179,
- /*    10 */   179,    0,   22,  232,   13,   13,   13,   92,  179,  179,
- /*    20 */   179,  179,  179,  120,   25,   25,  547,  194,  232,  232,
- /*    30 */   232,  232,  232,  232,  232,  232,  232,  232,  232,  232,
- /*    40 */   232,  232,  232,  232,  232,   13,   13,    2,    2,    2,
- /*    50 */     2,    2,    2,   11,    2,  209,  179,  179,  148,  148,
- /*    60 */    21,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    70 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    80 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    90 */   179,  179,  179,  179,  179,  179,  179,  179,  281,  330,
- /*   100 */   330,  283,  283,  330,  297,  296,  304,  311,  308,  314,
- /*   110 */   316,  318,  306,  281,  330,  330,  355,  355,  330,  347,
- /*   120 */   349,  384,  354,  353,  383,  356,  360,  330,  365,  330,
- /*   130 */   365,  547,  547,   27,   67,   67,   67,   94,  145,  218,
- /*   140 */   218,  218,  158,  246,  246,  246,  246,  222,  233,   85,
- /*   150 */    41,  257,  257,  249,  217,  221,  223,  226,  228,  268,
- /*   160 */   312,  176,  262,  150,  116,  229,  234,  235,  207,  211,
- /*   170 */   337,  338,  274,  437,  309,  438,  440,  317,  442,  444,
- /*   180 */   366,  329,  372,  362,  373,  376,  382,  385,  481,  386,
- /*   190 */   387,  389,  388,  371,  390,  374,  391,  394,  396,  395,
- /*   200 */   398,  397,  417,  489,  494,  495,  496,  497,  498,  428,
- /*   210 */   490,  434,  491,  379,  380,  407,  506,  507,  414,  416,
- /*   220 */   407,  510,  511,  512,  513,  514,  515,  516,  517,  518,
- /*   230 */   519,  520,  521,  522,  523,  426,  454,  524,  525,  472,
- /*   240 */   474,  533,
+ /*     0 */   144,   24,  134,  191,  235,   49,   49,   49,   49,   49,
+ /*    10 */    49,    0,   22,  235,  284,  284,  284,  106,   49,   49,
+ /*    20 */    49,   49,   49,  161,    4,    4,  552,  201,  235,  235,
+ /*    30 */   235,  235,  235,  235,  235,  235,  235,  235,  235,  235,
+ /*    40 */   235,  235,  235,  235,  235,  284,  284,    2,    2,    2,
+ /*    50 */     2,    2,    2,   43,    2,  225,   49,   49,  101,  101,
+ /*    60 */   157,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    70 */    49,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    80 */    49,   49,   49,   49,   49,   49,   49,   49,   49,   49,
+ /*    90 */    49,   49,   49,   49,   49,   49,   49,   49,  272,  333,
+ /*   100 */   333,  286,  286,  333,  282,  298,  310,  314,  312,  318,
+ /*   110 */   320,  322,  313,  272,  333,  333,  359,  359,  333,  352,
+ /*   120 */   355,  391,  361,  362,  390,  365,  371,  333,  376,  333,
+ /*   130 */   376,  552,  552,   27,   67,   67,   67,  127,  152,  226,
+ /*   140 */   226,  226,  181,  265,  265,  265,  265,  241,  255,   39,
+ /*   150 */    60,    8,    8,   96,  172,  192,  228,  229,  236,  167,
+ /*   160 */   290,  309,  142,  221,  209,  237,  238,  239,  185,  218,
+ /*   170 */   344,  345,  281,  447,  317,  449,  450,  324,  451,  453,
+ /*   180 */   392,  353,  374,  382,  375,  380,  385,  387,  486,  389,
+ /*   190 */   393,  395,  386,  377,  394,  381,  397,  396,  400,  388,
+ /*   200 */   401,  402,  419,  492,  497,  499,  500,  501,  502,  432,
+ /*   210 */   494,  438,  495,  383,  384,  408,  510,  511,  417,  420,
+ /*   220 */   408,  513,  515,  516,  517,  518,  519,  520,  521,  522,
+ /*   230 */   523,  524,  525,  526,  527,  528,  430,  459,  529,  530,
+ /*   240 */   477,  479,  538,
 };
 #define YY_REDUCE_COUNT (132)
-#define YY_REDUCE_MIN   (-257)
-#define YY_REDUCE_MAX   (261)
+#define YY_REDUCE_MIN   (-260)
+#define YY_REDUCE_MAX   (264)
 static const short yy_reduce_ofst[] = {
- /*     0 */  -202,  -79, -226, -211, -173, -103,  -16, -120,  -34,   -5,
- /*    10 */    61, -147, -180, -171,  -59,  -17,    3, -128,   -8,  -46,
- /*    20 */    -6, -178, -160,   44,   88,   98, -161, -257, -241, -215,
- /*    30 */  -203,  -13,   -1,   31,   53,   89,   90,   91,   93,   95,
- /*    40 */    97,   99,  101,  102,  103, -164,   35,  107,  111,  113,
- /*    50 */   117,  123,  124,  125,  127,  122,  159,  160,  104,  105,
- /*    60 */   126,  163,  165,  166,  167,  168,  169,  170,  171,  172,
- /*    70 */   173,  174,  175,  177,  178,  180,  181,  183,  185,  186,
- /*    80 */   187,  188,  189,  190,  191,  192,  193,  195,  196,  197,
- /*    90 */   198,  199,  200,  201,  202,  203,  204,  205,  206,  208,
- /*   100 */   210,  131,  141,  212,  161,  164,  213,  215,  214,  219,
- /*   110 */   224,  216,  225,  220,  227,  230,  231,  236,  237,  238,
- /*   120 */   241,  239,  240,  248,  252,  255,  242,  243,  244,  247,
- /*   130 */   245,  261,  260,
+ /*     0 */  -204, -101,   44, -260, -252, -211, -181, -149, -148, -122,
+ /*    10 */   -64, -104,  -61, -254, -199,  -66,  -44,  -49,  -48,  -36,
+ /*    20 */   -12,   15,   47, -212,   63,   70,   13, -247, -240, -230,
+ /*    30 */  -176, -172, -154, -138,  -53,    5,   25,   50,   65,   73,
+ /*    40 */    93,   94,   95,   97,   99,  110,  114,  120,  121,  122,
+ /*    50 */   123,  124,  125,  126,  128,  129,  158,  162,  103,  105,
+ /*    60 */   130,  165,  166,  168,  170,  171,  173,  174,  175,  176,
+ /*    70 */   177,  179,  180,  182,  183,  184,  186,  187,  188,  189,
+ /*    80 */   190,  193,  194,  195,  196,  197,  198,  199,  200,  202,
+ /*    90 */   203,  204,  205,  206,  207,  208,  210,  211,  137,  212,
+ /*   100 */   213,  138,  147,  214,  164,  169,  178,  215,  217,  216,
+ /*   110 */   220,  222,  224,  219,  227,  230,  231,  232,  233,  223,
+ /*   120 */   234,  240,  242,  243,  246,  247,  244,  245,  248,  256,
+ /*   130 */   263,  260,  264,
 };
 static const YYACTIONTYPE yy_default[] = {
- /*     0 */   624,  676,  665,  839,  839,  624,  624,  624,  624,  624,
- /*    10 */   624,  769,  642,  839,  624,  624,  624,  624,  624,  624,
- /*    20 */   624,  624,  624,  678,  678,  678,  764,  624,  624,  624,
- /*    30 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    40 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    50 */   624,  624,  624,  624,  624,  624,  624,  624,  788,  788,
- /*    60 */   762,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    70 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  663,
- /*    80 */   624,  661,  624,  624,  624,  624,  624,  624,  624,  624,
- /*    90 */   624,  624,  624,  624,  624,  650,  624,  624,  624,  644,
- /*   100 */   644,  624,  624,  644,  795,  799,  793,  781,  789,  780,
- /*   110 */   776,  775,  803,  624,  644,  644,  673,  673,  644,  694,
- /*   120 */   692,  690,  682,  688,  684,  686,  680,  644,  671,  644,
- /*   130 */   671,  710,  724,  624,  804,  838,  794,  822,  821,  834,
- /*   140 */   828,  827,  624,  826,  825,  824,  823,  624,  624,  624,
- /*   150 */   624,  830,  829,  624,  624,  624,  624,  624,  624,  624,
- /*   160 */   624,  624,  806,  800,  796,  624,  624,  624,  624,  624,
- /*   170 */   624,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   180 */   624,  624,  761,  624,  624,  770,  624,  624,  624,  624,
- /*   190 */   624,  624,  790,  624,  782,  624,  624,  624,  624,  624,
- /*   200 */   624,  738,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   210 */   624,  624,  624,  624,  624,  843,  624,  624,  624,  732,
- /*   220 */   841,  624,  624,  624,  624,  624,  624,  624,  624,  624,
- /*   230 */   624,  624,  624,  624,  624,  697,  624,  648,  646,  624,
- /*   240 */   640,  624,
+ /*     0 */   631,  683,  672,  849,  849,  631,  631,  631,  631,  631,
+ /*    10 */   631,  779,  649,  849,  631,  631,  631,  631,  631,  631,
+ /*    20 */   631,  631,  631,  685,  685,  685,  774,  631,  631,  631,
+ /*    30 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    40 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    50 */   631,  631,  631,  631,  631,  631,  631,  631,  798,  798,
+ /*    60 */   772,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    70 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  670,
+ /*    80 */   631,  668,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*    90 */   631,  631,  631,  631,  631,  657,  631,  631,  631,  651,
+ /*   100 */   651,  631,  631,  651,  805,  809,  803,  791,  799,  790,
+ /*   110 */   786,  785,  813,  631,  651,  651,  680,  680,  651,  701,
+ /*   120 */   699,  697,  689,  695,  691,  693,  687,  651,  678,  651,
+ /*   130 */   678,  718,  733,  631,  814,  848,  804,  832,  831,  844,
+ /*   140 */   838,  837,  631,  836,  835,  834,  833,  631,  631,  631,
+ /*   150 */   631,  840,  839,  631,  631,  631,  631,  631,  631,  631,
+ /*   160 */   631,  631,  816,  810,  806,  631,  631,  631,  631,  631,
+ /*   170 */   631,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   180 */   631,  631,  771,  631,  631,  780,  631,  631,  631,  631,
+ /*   190 */   631,  631,  800,  631,  792,  631,  631,  631,  631,  631,
+ /*   200 */   631,  748,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   210 */   631,  631,  631,  631,  631,  853,  631,  631,  631,  742,
+ /*   220 */   851,  631,  631,  631,  631,  631,  631,  631,  631,  631,
+ /*   230 */   631,  631,  631,  631,  631,  631,  704,  631,  655,  653,
+ /*   240 */   631,  647,  631,
 };
 /********** End of lemon-generated parsing tables *****************************/
 
@@ -521,6 +522,7 @@ static const YYCODETYPE yyFallback[] = {
     0,  /*  MAXTABLES => nothing */
     0,  /*      CACHE => nothing */
     0,  /*    REPLICA => nothing */
+    0,  /*     QUORUM => nothing */
     0,  /*       DAYS => nothing */
     0,  /*    MINROWS => nothing */
     0,  /*    MAXROWS => nothing */
@@ -813,188 +815,190 @@ static const char *const yyTokenName[] = {
   /*   86 */ "MAXTABLES",
   /*   87 */ "CACHE",
   /*   88 */ "REPLICA",
-  /*   89 */ "DAYS",
-  /*   90 */ "MINROWS",
-  /*   91 */ "MAXROWS",
-  /*   92 */ "BLOCKS",
-  /*   93 */ "CTIME",
-  /*   94 */ "WAL",
-  /*   95 */ "FSYNC",
-  /*   96 */ "COMP",
-  /*   97 */ "PRECISION",
-  /*   98 */ "LP",
-  /*   99 */ "RP",
-  /*  100 */ "TAGS",
-  /*  101 */ "USING",
-  /*  102 */ "AS",
-  /*  103 */ "COMMA",
-  /*  104 */ "NULL",
-  /*  105 */ "SELECT",
-  /*  106 */ "UNION",
-  /*  107 */ "ALL",
-  /*  108 */ "FROM",
-  /*  109 */ "VARIABLE",
-  /*  110 */ "INTERVAL",
-  /*  111 */ "FILL",
-  /*  112 */ "SLIDING",
-  /*  113 */ "ORDER",
-  /*  114 */ "BY",
-  /*  115 */ "ASC",
-  /*  116 */ "DESC",
-  /*  117 */ "GROUP",
-  /*  118 */ "HAVING",
-  /*  119 */ "LIMIT",
-  /*  120 */ "OFFSET",
-  /*  121 */ "SLIMIT",
-  /*  122 */ "SOFFSET",
-  /*  123 */ "WHERE",
-  /*  124 */ "NOW",
-  /*  125 */ "RESET",
-  /*  126 */ "QUERY",
-  /*  127 */ "ADD",
-  /*  128 */ "COLUMN",
-  /*  129 */ "TAG",
-  /*  130 */ "CHANGE",
-  /*  131 */ "SET",
-  /*  132 */ "KILL",
-  /*  133 */ "CONNECTION",
-  /*  134 */ "STREAM",
-  /*  135 */ "COLON",
-  /*  136 */ "ABORT",
-  /*  137 */ "AFTER",
-  /*  138 */ "ATTACH",
-  /*  139 */ "BEFORE",
-  /*  140 */ "BEGIN",
-  /*  141 */ "CASCADE",
-  /*  142 */ "CLUSTER",
-  /*  143 */ "CONFLICT",
-  /*  144 */ "COPY",
-  /*  145 */ "DEFERRED",
-  /*  146 */ "DELIMITERS",
-  /*  147 */ "DETACH",
-  /*  148 */ "EACH",
-  /*  149 */ "END",
-  /*  150 */ "EXPLAIN",
-  /*  151 */ "FAIL",
-  /*  152 */ "FOR",
-  /*  153 */ "IGNORE",
-  /*  154 */ "IMMEDIATE",
-  /*  155 */ "INITIALLY",
-  /*  156 */ "INSTEAD",
-  /*  157 */ "MATCH",
-  /*  158 */ "KEY",
-  /*  159 */ "OF",
-  /*  160 */ "RAISE",
-  /*  161 */ "REPLACE",
-  /*  162 */ "RESTRICT",
-  /*  163 */ "ROW",
-  /*  164 */ "STATEMENT",
-  /*  165 */ "TRIGGER",
-  /*  166 */ "VIEW",
-  /*  167 */ "COUNT",
-  /*  168 */ "SUM",
-  /*  169 */ "AVG",
-  /*  170 */ "MIN",
-  /*  171 */ "MAX",
-  /*  172 */ "FIRST",
-  /*  173 */ "LAST",
-  /*  174 */ "TOP",
-  /*  175 */ "BOTTOM",
-  /*  176 */ "STDDEV",
-  /*  177 */ "PERCENTILE",
-  /*  178 */ "APERCENTILE",
-  /*  179 */ "LEASTSQUARES",
-  /*  180 */ "HISTOGRAM",
-  /*  181 */ "DIFF",
-  /*  182 */ "SPREAD",
-  /*  183 */ "TWA",
-  /*  184 */ "INTERP",
-  /*  185 */ "LAST_ROW",
-  /*  186 */ "RATE",
-  /*  187 */ "IRATE",
-  /*  188 */ "SUM_RATE",
-  /*  189 */ "SUM_IRATE",
-  /*  190 */ "AVG_RATE",
-  /*  191 */ "AVG_IRATE",
-  /*  192 */ "TBID",
-  /*  193 */ "SEMI",
-  /*  194 */ "NONE",
-  /*  195 */ "PREV",
-  /*  196 */ "LINEAR",
-  /*  197 */ "IMPORT",
-  /*  198 */ "METRIC",
-  /*  199 */ "TBNAME",
-  /*  200 */ "JOIN",
-  /*  201 */ "METRICS",
-  /*  202 */ "STABLE",
-  /*  203 */ "INSERT",
-  /*  204 */ "INTO",
-  /*  205 */ "VALUES",
-  /*  206 */ "error",
-  /*  207 */ "program",
-  /*  208 */ "cmd",
-  /*  209 */ "dbPrefix",
-  /*  210 */ "ids",
-  /*  211 */ "cpxName",
-  /*  212 */ "ifexists",
-  /*  213 */ "alter_db_optr",
-  /*  214 */ "acct_optr",
-  /*  215 */ "ifnotexists",
-  /*  216 */ "db_optr",
-  /*  217 */ "pps",
-  /*  218 */ "tseries",
-  /*  219 */ "dbs",
-  /*  220 */ "streams",
-  /*  221 */ "storage",
-  /*  222 */ "qtime",
-  /*  223 */ "users",
-  /*  224 */ "conns",
-  /*  225 */ "state",
-  /*  226 */ "keep",
-  /*  227 */ "tagitemlist",
-  /*  228 */ "tables",
-  /*  229 */ "cache",
-  /*  230 */ "replica",
-  /*  231 */ "days",
-  /*  232 */ "minrows",
-  /*  233 */ "maxrows",
-  /*  234 */ "blocks",
-  /*  235 */ "ctime",
-  /*  236 */ "wal",
-  /*  237 */ "fsync",
-  /*  238 */ "comp",
-  /*  239 */ "prec",
-  /*  240 */ "typename",
-  /*  241 */ "signed",
-  /*  242 */ "create_table_args",
-  /*  243 */ "columnlist",
-  /*  244 */ "select",
-  /*  245 */ "column",
-  /*  246 */ "tagitem",
-  /*  247 */ "selcollist",
-  /*  248 */ "from",
-  /*  249 */ "where_opt",
-  /*  250 */ "interval_opt",
-  /*  251 */ "fill_opt",
-  /*  252 */ "sliding_opt",
-  /*  253 */ "groupby_opt",
-  /*  254 */ "orderby_opt",
-  /*  255 */ "having_opt",
-  /*  256 */ "slimit_opt",
-  /*  257 */ "limit_opt",
-  /*  258 */ "union",
-  /*  259 */ "sclp",
-  /*  260 */ "expr",
-  /*  261 */ "as",
-  /*  262 */ "tablelist",
-  /*  263 */ "tmvar",
-  /*  264 */ "sortlist",
-  /*  265 */ "sortitem",
-  /*  266 */ "item",
-  /*  267 */ "sortorder",
-  /*  268 */ "grouplist",
-  /*  269 */ "exprlist",
-  /*  270 */ "expritem",
+  /*   89 */ "QUORUM",
+  /*   90 */ "DAYS",
+  /*   91 */ "MINROWS",
+  /*   92 */ "MAXROWS",
+  /*   93 */ "BLOCKS",
+  /*   94 */ "CTIME",
+  /*   95 */ "WAL",
+  /*   96 */ "FSYNC",
+  /*   97 */ "COMP",
+  /*   98 */ "PRECISION",
+  /*   99 */ "LP",
+  /*  100 */ "RP",
+  /*  101 */ "TAGS",
+  /*  102 */ "USING",
+  /*  103 */ "AS",
+  /*  104 */ "COMMA",
+  /*  105 */ "NULL",
+  /*  106 */ "SELECT",
+  /*  107 */ "UNION",
+  /*  108 */ "ALL",
+  /*  109 */ "FROM",
+  /*  110 */ "VARIABLE",
+  /*  111 */ "INTERVAL",
+  /*  112 */ "FILL",
+  /*  113 */ "SLIDING",
+  /*  114 */ "ORDER",
+  /*  115 */ "BY",
+  /*  116 */ "ASC",
+  /*  117 */ "DESC",
+  /*  118 */ "GROUP",
+  /*  119 */ "HAVING",
+  /*  120 */ "LIMIT",
+  /*  121 */ "OFFSET",
+  /*  122 */ "SLIMIT",
+  /*  123 */ "SOFFSET",
+  /*  124 */ "WHERE",
+  /*  125 */ "NOW",
+  /*  126 */ "RESET",
+  /*  127 */ "QUERY",
+  /*  128 */ "ADD",
+  /*  129 */ "COLUMN",
+  /*  130 */ "TAG",
+  /*  131 */ "CHANGE",
+  /*  132 */ "SET",
+  /*  133 */ "KILL",
+  /*  134 */ "CONNECTION",
+  /*  135 */ "STREAM",
+  /*  136 */ "COLON",
+  /*  137 */ "ABORT",
+  /*  138 */ "AFTER",
+  /*  139 */ "ATTACH",
+  /*  140 */ "BEFORE",
+  /*  141 */ "BEGIN",
+  /*  142 */ "CASCADE",
+  /*  143 */ "CLUSTER",
+  /*  144 */ "CONFLICT",
+  /*  145 */ "COPY",
+  /*  146 */ "DEFERRED",
+  /*  147 */ "DELIMITERS",
+  /*  148 */ "DETACH",
+  /*  149 */ "EACH",
+  /*  150 */ "END",
+  /*  151 */ "EXPLAIN",
+  /*  152 */ "FAIL",
+  /*  153 */ "FOR",
+  /*  154 */ "IGNORE",
+  /*  155 */ "IMMEDIATE",
+  /*  156 */ "INITIALLY",
+  /*  157 */ "INSTEAD",
+  /*  158 */ "MATCH",
+  /*  159 */ "KEY",
+  /*  160 */ "OF",
+  /*  161 */ "RAISE",
+  /*  162 */ "REPLACE",
+  /*  163 */ "RESTRICT",
+  /*  164 */ "ROW",
+  /*  165 */ "STATEMENT",
+  /*  166 */ "TRIGGER",
+  /*  167 */ "VIEW",
+  /*  168 */ "COUNT",
+  /*  169 */ "SUM",
+  /*  170 */ "AVG",
+  /*  171 */ "MIN",
+  /*  172 */ "MAX",
+  /*  173 */ "FIRST",
+  /*  174 */ "LAST",
+  /*  175 */ "TOP",
+  /*  176 */ "BOTTOM",
+  /*  177 */ "STDDEV",
+  /*  178 */ "PERCENTILE",
+  /*  179 */ "APERCENTILE",
+  /*  180 */ "LEASTSQUARES",
+  /*  181 */ "HISTOGRAM",
+  /*  182 */ "DIFF",
+  /*  183 */ "SPREAD",
+  /*  184 */ "TWA",
+  /*  185 */ "INTERP",
+  /*  186 */ "LAST_ROW",
+  /*  187 */ "RATE",
+  /*  188 */ "IRATE",
+  /*  189 */ "SUM_RATE",
+  /*  190 */ "SUM_IRATE",
+  /*  191 */ "AVG_RATE",
+  /*  192 */ "AVG_IRATE",
+  /*  193 */ "TBID",
+  /*  194 */ "SEMI",
+  /*  195 */ "NONE",
+  /*  196 */ "PREV",
+  /*  197 */ "LINEAR",
+  /*  198 */ "IMPORT",
+  /*  199 */ "METRIC",
+  /*  200 */ "TBNAME",
+  /*  201 */ "JOIN",
+  /*  202 */ "METRICS",
+  /*  203 */ "STABLE",
+  /*  204 */ "INSERT",
+  /*  205 */ "INTO",
+  /*  206 */ "VALUES",
+  /*  207 */ "error",
+  /*  208 */ "program",
+  /*  209 */ "cmd",
+  /*  210 */ "dbPrefix",
+  /*  211 */ "ids",
+  /*  212 */ "cpxName",
+  /*  213 */ "ifexists",
+  /*  214 */ "alter_db_optr",
+  /*  215 */ "acct_optr",
+  /*  216 */ "ifnotexists",
+  /*  217 */ "db_optr",
+  /*  218 */ "pps",
+  /*  219 */ "tseries",
+  /*  220 */ "dbs",
+  /*  221 */ "streams",
+  /*  222 */ "storage",
+  /*  223 */ "qtime",
+  /*  224 */ "users",
+  /*  225 */ "conns",
+  /*  226 */ "state",
+  /*  227 */ "keep",
+  /*  228 */ "tagitemlist",
+  /*  229 */ "tables",
+  /*  230 */ "cache",
+  /*  231 */ "replica",
+  /*  232 */ "quorum",
+  /*  233 */ "days",
+  /*  234 */ "minrows",
+  /*  235 */ "maxrows",
+  /*  236 */ "blocks",
+  /*  237 */ "ctime",
+  /*  238 */ "wal",
+  /*  239 */ "fsync",
+  /*  240 */ "comp",
+  /*  241 */ "prec",
+  /*  242 */ "typename",
+  /*  243 */ "signed",
+  /*  244 */ "create_table_args",
+  /*  245 */ "columnlist",
+  /*  246 */ "select",
+  /*  247 */ "column",
+  /*  248 */ "tagitem",
+  /*  249 */ "selcollist",
+  /*  250 */ "from",
+  /*  251 */ "where_opt",
+  /*  252 */ "interval_opt",
+  /*  253 */ "fill_opt",
+  /*  254 */ "sliding_opt",
+  /*  255 */ "groupby_opt",
+  /*  256 */ "orderby_opt",
+  /*  257 */ "having_opt",
+  /*  258 */ "slimit_opt",
+  /*  259 */ "limit_opt",
+  /*  260 */ "union",
+  /*  261 */ "sclp",
+  /*  262 */ "expr",
+  /*  263 */ "as",
+  /*  264 */ "tablelist",
+  /*  265 */ "tmvar",
+  /*  266 */ "sortlist",
+  /*  267 */ "sortitem",
+  /*  268 */ "item",
+  /*  269 */ "sortorder",
+  /*  270 */ "grouplist",
+  /*  271 */ "exprlist",
+  /*  272 */ "expritem",
 };
 #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
 
@@ -1076,155 +1080,158 @@ static const char *const yyRuleName[] = {
  /*  71 */ "tables ::= MAXTABLES INTEGER",
  /*  72 */ "cache ::= CACHE INTEGER",
  /*  73 */ "replica ::= REPLICA INTEGER",
- /*  74 */ "days ::= DAYS INTEGER",
- /*  75 */ "minrows ::= MINROWS INTEGER",
- /*  76 */ "maxrows ::= MAXROWS INTEGER",
- /*  77 */ "blocks ::= BLOCKS INTEGER",
- /*  78 */ "ctime ::= CTIME INTEGER",
- /*  79 */ "wal ::= WAL INTEGER",
- /*  80 */ "fsync ::= FSYNC INTEGER",
- /*  81 */ "comp ::= COMP INTEGER",
- /*  82 */ "prec ::= PRECISION STRING",
- /*  83 */ "db_optr ::=",
- /*  84 */ "db_optr ::= db_optr tables",
- /*  85 */ "db_optr ::= db_optr cache",
- /*  86 */ "db_optr ::= db_optr replica",
- /*  87 */ "db_optr ::= db_optr days",
- /*  88 */ "db_optr ::= db_optr minrows",
- /*  89 */ "db_optr ::= db_optr maxrows",
- /*  90 */ "db_optr ::= db_optr blocks",
- /*  91 */ "db_optr ::= db_optr ctime",
- /*  92 */ "db_optr ::= db_optr wal",
- /*  93 */ "db_optr ::= db_optr fsync",
- /*  94 */ "db_optr ::= db_optr comp",
- /*  95 */ "db_optr ::= db_optr prec",
- /*  96 */ "db_optr ::= db_optr keep",
- /*  97 */ "alter_db_optr ::=",
- /*  98 */ "alter_db_optr ::= alter_db_optr replica",
- /*  99 */ "alter_db_optr ::= alter_db_optr tables",
- /* 100 */ "alter_db_optr ::= alter_db_optr keep",
- /* 101 */ "alter_db_optr ::= alter_db_optr blocks",
- /* 102 */ "alter_db_optr ::= alter_db_optr comp",
- /* 103 */ "alter_db_optr ::= alter_db_optr wal",
- /* 104 */ "alter_db_optr ::= alter_db_optr fsync",
- /* 105 */ "typename ::= ids",
- /* 106 */ "typename ::= ids LP signed RP",
- /* 107 */ "signed ::= INTEGER",
- /* 108 */ "signed ::= PLUS INTEGER",
- /* 109 */ "signed ::= MINUS INTEGER",
- /* 110 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
- /* 111 */ "create_table_args ::= LP columnlist RP",
- /* 112 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
- /* 113 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
- /* 114 */ "create_table_args ::= AS select",
- /* 115 */ "columnlist ::= columnlist COMMA column",
- /* 116 */ "columnlist ::= column",
- /* 117 */ "column ::= ids typename",
- /* 118 */ "tagitemlist ::= tagitemlist COMMA tagitem",
- /* 119 */ "tagitemlist ::= tagitem",
- /* 120 */ "tagitem ::= INTEGER",
- /* 121 */ "tagitem ::= FLOAT",
- /* 122 */ "tagitem ::= STRING",
- /* 123 */ "tagitem ::= BOOL",
- /* 124 */ "tagitem ::= NULL",
- /* 125 */ "tagitem ::= MINUS INTEGER",
- /* 126 */ "tagitem ::= MINUS FLOAT",
- /* 127 */ "tagitem ::= PLUS INTEGER",
- /* 128 */ "tagitem ::= PLUS FLOAT",
- /* 129 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
- /* 130 */ "union ::= select",
- /* 131 */ "union ::= LP union RP",
- /* 132 */ "union ::= union UNION ALL select",
- /* 133 */ "union ::= union UNION ALL LP select RP",
- /* 134 */ "cmd ::= union",
- /* 135 */ "select ::= SELECT selcollist",
- /* 136 */ "sclp ::= selcollist COMMA",
- /* 137 */ "sclp ::=",
- /* 138 */ "selcollist ::= sclp expr as",
- /* 139 */ "selcollist ::= sclp STAR",
- /* 140 */ "as ::= AS ids",
- /* 141 */ "as ::= ids",
- /* 142 */ "as ::=",
- /* 143 */ "from ::= FROM tablelist",
- /* 144 */ "tablelist ::= ids cpxName",
- /* 145 */ "tablelist ::= tablelist COMMA ids cpxName",
- /* 146 */ "tmvar ::= VARIABLE",
- /* 147 */ "interval_opt ::= INTERVAL LP tmvar RP",
- /* 148 */ "interval_opt ::=",
- /* 149 */ "fill_opt ::=",
- /* 150 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
- /* 151 */ "fill_opt ::= FILL LP ID RP",
- /* 152 */ "sliding_opt ::= SLIDING LP tmvar RP",
- /* 153 */ "sliding_opt ::=",
- /* 154 */ "orderby_opt ::=",
- /* 155 */ "orderby_opt ::= ORDER BY sortlist",
- /* 156 */ "sortlist ::= sortlist COMMA item sortorder",
- /* 157 */ "sortlist ::= item sortorder",
- /* 158 */ "item ::= ids cpxName",
- /* 159 */ "sortorder ::= ASC",
- /* 160 */ "sortorder ::= DESC",
- /* 161 */ "sortorder ::=",
- /* 162 */ "groupby_opt ::=",
- /* 163 */ "groupby_opt ::= GROUP BY grouplist",
- /* 164 */ "grouplist ::= grouplist COMMA item",
- /* 165 */ "grouplist ::= item",
- /* 166 */ "having_opt ::=",
- /* 167 */ "having_opt ::= HAVING expr",
- /* 168 */ "limit_opt ::=",
- /* 169 */ "limit_opt ::= LIMIT signed",
- /* 170 */ "limit_opt ::= LIMIT signed OFFSET signed",
- /* 171 */ "limit_opt ::= LIMIT signed COMMA signed",
- /* 172 */ "slimit_opt ::=",
- /* 173 */ "slimit_opt ::= SLIMIT signed",
- /* 174 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
- /* 175 */ "slimit_opt ::= SLIMIT signed COMMA signed",
- /* 176 */ "where_opt ::=",
- /* 177 */ "where_opt ::= WHERE expr",
- /* 178 */ "expr ::= LP expr RP",
- /* 179 */ "expr ::= ID",
- /* 180 */ "expr ::= ID DOT ID",
- /* 181 */ "expr ::= ID DOT STAR",
- /* 182 */ "expr ::= INTEGER",
- /* 183 */ "expr ::= MINUS INTEGER",
- /* 184 */ "expr ::= PLUS INTEGER",
- /* 185 */ "expr ::= FLOAT",
- /* 186 */ "expr ::= MINUS FLOAT",
- /* 187 */ "expr ::= PLUS FLOAT",
- /* 188 */ "expr ::= STRING",
- /* 189 */ "expr ::= NOW",
- /* 190 */ "expr ::= VARIABLE",
- /* 191 */ "expr ::= BOOL",
- /* 192 */ "expr ::= ID LP exprlist RP",
- /* 193 */ "expr ::= ID LP STAR RP",
- /* 194 */ "expr ::= expr AND expr",
- /* 195 */ "expr ::= expr OR expr",
- /* 196 */ "expr ::= expr LT expr",
- /* 197 */ "expr ::= expr GT expr",
- /* 198 */ "expr ::= expr LE expr",
- /* 199 */ "expr ::= expr GE expr",
- /* 200 */ "expr ::= expr NE expr",
- /* 201 */ "expr ::= expr EQ expr",
- /* 202 */ "expr ::= expr PLUS expr",
- /* 203 */ "expr ::= expr MINUS expr",
- /* 204 */ "expr ::= expr STAR expr",
- /* 205 */ "expr ::= expr SLASH expr",
- /* 206 */ "expr ::= expr REM expr",
- /* 207 */ "expr ::= expr LIKE expr",
- /* 208 */ "expr ::= expr IN LP exprlist RP",
- /* 209 */ "exprlist ::= exprlist COMMA expritem",
- /* 210 */ "exprlist ::= expritem",
- /* 211 */ "expritem ::= expr",
- /* 212 */ "expritem ::=",
- /* 213 */ "cmd ::= RESET QUERY CACHE",
- /* 214 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
- /* 215 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
- /* 216 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
- /* 217 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
- /* 218 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
- /* 219 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
- /* 220 */ "cmd ::= KILL CONNECTION INTEGER",
- /* 221 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
- /* 222 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
+ /*  74 */ "quorum ::= QUORUM INTEGER",
+ /*  75 */ "days ::= DAYS INTEGER",
+ /*  76 */ "minrows ::= MINROWS INTEGER",
+ /*  77 */ "maxrows ::= MAXROWS INTEGER",
+ /*  78 */ "blocks ::= BLOCKS INTEGER",
+ /*  79 */ "ctime ::= CTIME INTEGER",
+ /*  80 */ "wal ::= WAL INTEGER",
+ /*  81 */ "fsync ::= FSYNC INTEGER",
+ /*  82 */ "comp ::= COMP INTEGER",
+ /*  83 */ "prec ::= PRECISION STRING",
+ /*  84 */ "db_optr ::=",
+ /*  85 */ "db_optr ::= db_optr tables",
+ /*  86 */ "db_optr ::= db_optr cache",
+ /*  87 */ "db_optr ::= db_optr replica",
+ /*  88 */ "db_optr ::= db_optr quorum",
+ /*  89 */ "db_optr ::= db_optr days",
+ /*  90 */ "db_optr ::= db_optr minrows",
+ /*  91 */ "db_optr ::= db_optr maxrows",
+ /*  92 */ "db_optr ::= db_optr blocks",
+ /*  93 */ "db_optr ::= db_optr ctime",
+ /*  94 */ "db_optr ::= db_optr wal",
+ /*  95 */ "db_optr ::= db_optr fsync",
+ /*  96 */ "db_optr ::= db_optr comp",
+ /*  97 */ "db_optr ::= db_optr prec",
+ /*  98 */ "db_optr ::= db_optr keep",
+ /*  99 */ "alter_db_optr ::=",
+ /* 100 */ "alter_db_optr ::= alter_db_optr replica",
+ /* 101 */ "alter_db_optr ::= alter_db_optr quorum",
+ /* 102 */ "alter_db_optr ::= alter_db_optr tables",
+ /* 103 */ "alter_db_optr ::= alter_db_optr keep",
+ /* 104 */ "alter_db_optr ::= alter_db_optr blocks",
+ /* 105 */ "alter_db_optr ::= alter_db_optr comp",
+ /* 106 */ "alter_db_optr ::= alter_db_optr wal",
+ /* 107 */ "alter_db_optr ::= alter_db_optr fsync",
+ /* 108 */ "typename ::= ids",
+ /* 109 */ "typename ::= ids LP signed RP",
+ /* 110 */ "signed ::= INTEGER",
+ /* 111 */ "signed ::= PLUS INTEGER",
+ /* 112 */ "signed ::= MINUS INTEGER",
+ /* 113 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
+ /* 114 */ "create_table_args ::= LP columnlist RP",
+ /* 115 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
+ /* 116 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
+ /* 117 */ "create_table_args ::= AS select",
+ /* 118 */ "columnlist ::= columnlist COMMA column",
+ /* 119 */ "columnlist ::= column",
+ /* 120 */ "column ::= ids typename",
+ /* 121 */ "tagitemlist ::= tagitemlist COMMA tagitem",
+ /* 122 */ "tagitemlist ::= tagitem",
+ /* 123 */ "tagitem ::= INTEGER",
+ /* 124 */ "tagitem ::= FLOAT",
+ /* 125 */ "tagitem ::= STRING",
+ /* 126 */ "tagitem ::= BOOL",
+ /* 127 */ "tagitem ::= NULL",
+ /* 128 */ "tagitem ::= MINUS INTEGER",
+ /* 129 */ "tagitem ::= MINUS FLOAT",
+ /* 130 */ "tagitem ::= PLUS INTEGER",
+ /* 131 */ "tagitem ::= PLUS FLOAT",
+ /* 132 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
+ /* 133 */ "union ::= select",
+ /* 134 */ "union ::= LP union RP",
+ /* 135 */ "union ::= union UNION ALL select",
+ /* 136 */ "union ::= union UNION ALL LP select RP",
+ /* 137 */ "cmd ::= union",
+ /* 138 */ "select ::= SELECT selcollist",
+ /* 139 */ "sclp ::= selcollist COMMA",
+ /* 140 */ "sclp ::=",
+ /* 141 */ "selcollist ::= sclp expr as",
+ /* 142 */ "selcollist ::= sclp STAR",
+ /* 143 */ "as ::= AS ids",
+ /* 144 */ "as ::= ids",
+ /* 145 */ "as ::=",
+ /* 146 */ "from ::= FROM tablelist",
+ /* 147 */ "tablelist ::= ids cpxName",
+ /* 148 */ "tablelist ::= tablelist COMMA ids cpxName",
+ /* 149 */ "tmvar ::= VARIABLE",
+ /* 150 */ "interval_opt ::= INTERVAL LP tmvar RP",
+ /* 151 */ "interval_opt ::=",
+ /* 152 */ "fill_opt ::=",
+ /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
+ /* 154 */ "fill_opt ::= FILL LP ID RP",
+ /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP",
+ /* 156 */ "sliding_opt ::=",
+ /* 157 */ "orderby_opt ::=",
+ /* 158 */ "orderby_opt ::= ORDER BY sortlist",
+ /* 159 */ "sortlist ::= sortlist COMMA item sortorder",
+ /* 160 */ "sortlist ::= item sortorder",
+ /* 161 */ "item ::= ids cpxName",
+ /* 162 */ "sortorder ::= ASC",
+ /* 163 */ "sortorder ::= DESC",
+ /* 164 */ "sortorder ::=",
+ /* 165 */ "groupby_opt ::=",
+ /* 166 */ "groupby_opt ::= GROUP BY grouplist",
+ /* 167 */ "grouplist ::= grouplist COMMA item",
+ /* 168 */ "grouplist ::= item",
+ /* 169 */ "having_opt ::=",
+ /* 170 */ "having_opt ::= HAVING expr",
+ /* 171 */ "limit_opt ::=",
+ /* 172 */ "limit_opt ::= LIMIT signed",
+ /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed",
+ /* 174 */ "limit_opt ::= LIMIT signed COMMA signed",
+ /* 175 */ "slimit_opt ::=",
+ /* 176 */ "slimit_opt ::= SLIMIT signed",
+ /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
+ /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed",
+ /* 179 */ "where_opt ::=",
+ /* 180 */ "where_opt ::= WHERE expr",
+ /* 181 */ "expr ::= LP expr RP",
+ /* 182 */ "expr ::= ID",
+ /* 183 */ "expr ::= ID DOT ID",
+ /* 184 */ "expr ::= ID DOT STAR",
+ /* 185 */ "expr ::= INTEGER",
+ /* 186 */ "expr ::= MINUS INTEGER",
+ /* 187 */ "expr ::= PLUS INTEGER",
+ /* 188 */ "expr ::= FLOAT",
+ /* 189 */ "expr ::= MINUS FLOAT",
+ /* 190 */ "expr ::= PLUS FLOAT",
+ /* 191 */ "expr ::= STRING",
+ /* 192 */ "expr ::= NOW",
+ /* 193 */ "expr ::= VARIABLE",
+ /* 194 */ "expr ::= BOOL",
+ /* 195 */ "expr ::= ID LP exprlist RP",
+ /* 196 */ "expr ::= ID LP STAR RP",
+ /* 197 */ "expr ::= expr AND expr",
+ /* 198 */ "expr ::= expr OR expr",
+ /* 199 */ "expr ::= expr LT expr",
+ /* 200 */ "expr ::= expr GT expr",
+ /* 201 */ "expr ::= expr LE expr",
+ /* 202 */ "expr ::= expr GE expr",
+ /* 203 */ "expr ::= expr NE expr",
+ /* 204 */ "expr ::= expr EQ expr",
+ /* 205 */ "expr ::= expr PLUS expr",
+ /* 206 */ "expr ::= expr MINUS expr",
+ /* 207 */ "expr ::= expr STAR expr",
+ /* 208 */ "expr ::= expr SLASH expr",
+ /* 209 */ "expr ::= expr REM expr",
+ /* 210 */ "expr ::= expr LIKE expr",
+ /* 211 */ "expr ::= expr IN LP exprlist RP",
+ /* 212 */ "exprlist ::= exprlist COMMA expritem",
+ /* 213 */ "exprlist ::= expritem",
+ /* 214 */ "expritem ::= expr",
+ /* 215 */ "expritem ::=",
+ /* 216 */ "cmd ::= RESET QUERY CACHE",
+ /* 217 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
+ /* 218 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
+ /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
+ /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
+ /* 221 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
+ /* 222 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
+ /* 223 */ "cmd ::= KILL CONNECTION INTEGER",
+ /* 224 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
+ /* 225 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
 };
 #endif /* NDEBUG */
 
@@ -1345,50 +1352,50 @@ static void yy_destructor(
     ** inside the C code.
     */
 /********* Begin destructor definitions ***************************************/
-    case 226: /* keep */
-    case 227: /* tagitemlist */
-    case 251: /* fill_opt */
-    case 253: /* groupby_opt */
-    case 254: /* orderby_opt */
-    case 264: /* sortlist */
-    case 268: /* grouplist */
+    case 227: /* keep */
+    case 228: /* tagitemlist */
+    case 253: /* fill_opt */
+    case 255: /* groupby_opt */
+    case 256: /* orderby_opt */
+    case 266: /* sortlist */
+    case 270: /* grouplist */
 {
-tVariantListDestroy((yypminor->yy494));
+tVariantListDestroy((yypminor->yy498));
 }
       break;
-    case 243: /* columnlist */
+    case 245: /* columnlist */
 {
-tFieldListDestroy((yypminor->yy449));
+tFieldListDestroy((yypminor->yy523));
 }
       break;
-    case 244: /* select */
+    case 246: /* select */
 {
-doDestroyQuerySql((yypminor->yy150));
+doDestroyQuerySql((yypminor->yy414));
 }
       break;
-    case 247: /* selcollist */
-    case 259: /* sclp */
-    case 269: /* exprlist */
+    case 249: /* selcollist */
+    case 261: /* sclp */
+    case 271: /* exprlist */
 {
-tSQLExprListDestroy((yypminor->yy224));
+tSQLExprListDestroy((yypminor->yy290));
 }
       break;
-    case 249: /* where_opt */
-    case 255: /* having_opt */
-    case 260: /* expr */
-    case 270: /* expritem */
+    case 251: /* where_opt */
+    case 257: /* having_opt */
+    case 262: /* expr */
+    case 272: /* expritem */
 {
-tSQLExprDestroy((yypminor->yy66));
+tSQLExprDestroy((yypminor->yy64));
 }
       break;
-    case 258: /* union */
+    case 260: /* union */
 {
-destroyAllSelectClause((yypminor->yy25));
+destroyAllSelectClause((yypminor->yy231));
 }
       break;
-    case 265: /* sortitem */
+    case 267: /* sortitem */
 {
-tVariantDestroy(&(yypminor->yy312));
+tVariantDestroy(&(yypminor->yy134));
 }
       break;
 /********* End destructor definitions *****************************************/
@@ -1682,229 +1689,232 @@ static const struct {
   YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
 } yyRuleInfo[] = {
-  {  207,   -1 }, /* (0) program ::= cmd */
-  {  208,   -2 }, /* (1) cmd ::= SHOW DATABASES */
-  {  208,   -2 }, /* (2) cmd ::= SHOW MNODES */
-  {  208,   -2 }, /* (3) cmd ::= SHOW DNODES */
-  {  208,   -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
-  {  208,   -2 }, /* (5) cmd ::= SHOW USERS */
-  {  208,   -2 }, /* (6) cmd ::= SHOW MODULES */
-  {  208,   -2 }, /* (7) cmd ::= SHOW QUERIES */
-  {  208,   -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
-  {  208,   -2 }, /* (9) cmd ::= SHOW STREAMS */
-  {  208,   -2 }, /* (10) cmd ::= SHOW CONFIGS */
-  {  208,   -2 }, /* (11) cmd ::= SHOW SCORES */
-  {  208,   -2 }, /* (12) cmd ::= SHOW GRANTS */
-  {  208,   -2 }, /* (13) cmd ::= SHOW VNODES */
-  {  208,   -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
-  {  209,    0 }, /* (15) dbPrefix ::= */
-  {  209,   -2 }, /* (16) dbPrefix ::= ids DOT */
-  {  211,    0 }, /* (17) cpxName ::= */
-  {  211,   -2 }, /* (18) cpxName ::= DOT ids */
-  {  208,   -3 }, /* (19) cmd ::= SHOW dbPrefix TABLES */
-  {  208,   -5 }, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */
-  {  208,   -3 }, /* (21) cmd ::= SHOW dbPrefix STABLES */
-  {  208,   -5 }, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */
-  {  208,   -3 }, /* (23) cmd ::= SHOW dbPrefix VGROUPS */
-  {  208,   -4 }, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */
-  {  208,   -5 }, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */
-  {  208,   -4 }, /* (26) cmd ::= DROP DATABASE ifexists ids */
-  {  208,   -3 }, /* (27) cmd ::= DROP DNODE ids */
-  {  208,   -3 }, /* (28) cmd ::= DROP USER ids */
-  {  208,   -3 }, /* (29) cmd ::= DROP ACCOUNT ids */
-  {  208,   -2 }, /* (30) cmd ::= USE ids */
-  {  208,   -3 }, /* (31) cmd ::= DESCRIBE ids cpxName */
-  {  208,   -5 }, /* (32) cmd ::= ALTER USER ids PASS ids */
-  {  208,   -5 }, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */
-  {  208,   -4 }, /* (34) cmd ::= ALTER DNODE ids ids */
-  {  208,   -5 }, /* (35) cmd ::= ALTER DNODE ids ids ids */
-  {  208,   -3 }, /* (36) cmd ::= ALTER LOCAL ids */
-  {  208,   -4 }, /* (37) cmd ::= ALTER LOCAL ids ids */
-  {  208,   -4 }, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */
-  {  208,   -4 }, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */
-  {  208,   -6 }, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-  {  210,   -1 }, /* (41) ids ::= ID */
-  {  210,   -1 }, /* (42) ids ::= STRING */
-  {  212,   -2 }, /* (43) ifexists ::= IF EXISTS */
-  {  212,    0 }, /* (44) ifexists ::= */
-  {  215,   -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */
-  {  215,    0 }, /* (46) ifnotexists ::= */
-  {  208,   -3 }, /* (47) cmd ::= CREATE DNODE ids */
-  {  208,   -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-  {  208,   -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-  {  208,   -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */
-  {  217,    0 }, /* (51) pps ::= */
-  {  217,   -2 }, /* (52) pps ::= PPS INTEGER */
-  {  218,    0 }, /* (53) tseries ::= */
-  {  218,   -2 }, /* (54) tseries ::= TSERIES INTEGER */
-  {  219,    0 }, /* (55) dbs ::= */
-  {  219,   -2 }, /* (56) dbs ::= DBS INTEGER */
-  {  220,    0 }, /* (57) streams ::= */
-  {  220,   -2 }, /* (58) streams ::= STREAMS INTEGER */
-  {  221,    0 }, /* (59) storage ::= */
-  {  221,   -2 }, /* (60) storage ::= STORAGE INTEGER */
-  {  222,    0 }, /* (61) qtime ::= */
-  {  222,   -2 }, /* (62) qtime ::= QTIME INTEGER */
-  {  223,    0 }, /* (63) users ::= */
-  {  223,   -2 }, /* (64) users ::= USERS INTEGER */
-  {  224,    0 }, /* (65) conns ::= */
-  {  224,   -2 }, /* (66) conns ::= CONNS INTEGER */
-  {  225,    0 }, /* (67) state ::= */
-  {  225,   -2 }, /* (68) state ::= STATE ids */
-  {  214,   -9 }, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
-  {  226,   -2 }, /* (70) keep ::= KEEP tagitemlist */
-  {  228,   -2 }, /* (71) tables ::= MAXTABLES INTEGER */
-  {  229,   -2 }, /* (72) cache ::= CACHE INTEGER */
-  {  230,   -2 }, /* (73) replica ::= REPLICA INTEGER */
-  {  231,   -2 }, /* (74) days ::= DAYS INTEGER */
-  {  232,   -2 }, /* (75) minrows ::= MINROWS INTEGER */
-  {  233,   -2 }, /* (76) maxrows ::= MAXROWS INTEGER */
-  {  234,   -2 }, /* (77) blocks ::= BLOCKS INTEGER */
-  {  235,   -2 }, /* (78) ctime ::= CTIME INTEGER */
-  {  236,   -2 }, /* (79) wal ::= WAL INTEGER */
-  {  237,   -2 }, /* (80) fsync ::= FSYNC INTEGER */
-  {  238,   -2 }, /* (81) comp ::= COMP INTEGER */
-  {  239,   -2 }, /* (82) prec ::= PRECISION STRING */
-  {  216,    0 }, /* (83) db_optr ::= */
-  {  216,   -2 }, /* (84) db_optr ::= db_optr tables */
-  {  216,   -2 }, /* (85) db_optr ::= db_optr cache */
-  {  216,   -2 }, /* (86) db_optr ::= db_optr replica */
-  {  216,   -2 }, /* (87) db_optr ::= db_optr days */
-  {  216,   -2 }, /* (88) db_optr ::= db_optr minrows */
-  {  216,   -2 }, /* (89) db_optr ::= db_optr maxrows */
-  {  216,   -2 }, /* (90) db_optr ::= db_optr blocks */
-  {  216,   -2 }, /* (91) db_optr ::= db_optr ctime */
-  {  216,   -2 }, /* (92) db_optr ::= db_optr wal */
-  {  216,   -2 }, /* (93) db_optr ::= db_optr fsync */
-  {  216,   -2 }, /* (94) db_optr ::= db_optr comp */
-  {  216,   -2 }, /* (95) db_optr ::= db_optr prec */
-  {  216,   -2 }, /* (96) db_optr ::= db_optr keep */
-  {  213,    0 }, /* (97) alter_db_optr ::= */
-  {  213,   -2 }, /* (98) alter_db_optr ::= alter_db_optr replica */
-  {  213,   -2 }, /* (99) alter_db_optr ::= alter_db_optr tables */
-  {  213,   -2 }, /* (100) alter_db_optr ::= alter_db_optr keep */
-  {  213,   -2 }, /* (101) alter_db_optr ::= alter_db_optr blocks */
-  {  213,   -2 }, /* (102) alter_db_optr ::= alter_db_optr comp */
-  {  213,   -2 }, /* (103) alter_db_optr ::= alter_db_optr wal */
-  {  213,   -2 }, /* (104) alter_db_optr ::= alter_db_optr fsync */
-  {  240,   -1 }, /* (105) typename ::= ids */
-  {  240,   -4 }, /* (106) typename ::= ids LP signed RP */
-  {  241,   -1 }, /* (107) signed ::= INTEGER */
-  {  241,   -2 }, /* (108) signed ::= PLUS INTEGER */
-  {  241,   -2 }, /* (109) signed ::= MINUS INTEGER */
-  {  208,   -6 }, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
-  {  242,   -3 }, /* (111) create_table_args ::= LP columnlist RP */
-  {  242,   -7 }, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
-  {  242,   -7 }, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
-  {  242,   -2 }, /* (114) create_table_args ::= AS select */
-  {  243,   -3 }, /* (115) columnlist ::= columnlist COMMA column */
-  {  243,   -1 }, /* (116) columnlist ::= column */
-  {  245,   -2 }, /* (117) column ::= ids typename */
-  {  227,   -3 }, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */
-  {  227,   -1 }, /* (119) tagitemlist ::= tagitem */
-  {  246,   -1 }, /* (120) tagitem ::= INTEGER */
-  {  246,   -1 }, /* (121) tagitem ::= FLOAT */
-  {  246,   -1 }, /* (122) tagitem ::= STRING */
-  {  246,   -1 }, /* (123) tagitem ::= BOOL */
-  {  246,   -1 }, /* (124) tagitem ::= NULL */
-  {  246,   -2 }, /* (125) tagitem ::= MINUS INTEGER */
-  {  246,   -2 }, /* (126) tagitem ::= MINUS FLOAT */
-  {  246,   -2 }, /* (127) tagitem ::= PLUS INTEGER */
-  {  246,   -2 }, /* (128) tagitem ::= PLUS FLOAT */
-  {  244,  -12 }, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
-  {  258,   -1 }, /* (130) union ::= select */
-  {  258,   -3 }, /* (131) union ::= LP union RP */
-  {  258,   -4 }, /* (132) union ::= union UNION ALL select */
-  {  258,   -6 }, /* (133) union ::= union UNION ALL LP select RP */
-  {  208,   -1 }, /* (134) cmd ::= union */
-  {  244,   -2 }, /* (135) select ::= SELECT selcollist */
-  {  259,   -2 }, /* (136) sclp ::= selcollist COMMA */
-  {  259,    0 }, /* (137) sclp ::= */
-  {  247,   -3 }, /* (138) selcollist ::= sclp expr as */
-  {  247,   -2 }, /* (139) selcollist ::= sclp STAR */
-  {  261,   -2 }, /* (140) as ::= AS ids */
-  {  261,   -1 }, /* (141) as ::= ids */
-  {  261,    0 }, /* (142) as ::= */
-  {  248,   -2 }, /* (143) from ::= FROM tablelist */
-  {  262,   -2 }, /* (144) tablelist ::= ids cpxName */
-  {  262,   -4 }, /* (145) tablelist ::= tablelist COMMA ids cpxName */
-  {  263,   -1 }, /* (146) tmvar ::= VARIABLE */
-  {  250,   -4 }, /* (147) interval_opt ::= INTERVAL LP tmvar RP */
-  {  250,    0 }, /* (148) interval_opt ::= */
-  {  251,    0 }, /* (149) fill_opt ::= */
-  {  251,   -6 }, /* (150) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
-  {  251,   -4 }, /* (151) fill_opt ::= FILL LP ID RP */
-  {  252,   -4 }, /* (152) sliding_opt ::= SLIDING LP tmvar RP */
-  {  252,    0 }, /* (153) sliding_opt ::= */
-  {  254,    0 }, /* (154) orderby_opt ::= */
-  {  254,   -3 }, /* (155) orderby_opt ::= ORDER BY sortlist */
-  {  264,   -4 }, /* (156) sortlist ::= sortlist COMMA item sortorder */
-  {  264,   -2 }, /* (157) sortlist ::= item sortorder */
-  {  266,   -2 }, /* (158) item ::= ids cpxName */
-  {  267,   -1 }, /* (159) sortorder ::= ASC */
-  {  267,   -1 }, /* (160) sortorder ::= DESC */
-  {  267,    0 }, /* (161) sortorder ::= */
-  {  253,    0 }, /* (162) groupby_opt ::= */
-  {  253,   -3 }, /* (163) groupby_opt ::= GROUP BY grouplist */
-  {  268,   -3 }, /* (164) grouplist ::= grouplist COMMA item */
-  {  268,   -1 }, /* (165) grouplist ::= item */
-  {  255,    0 }, /* (166) having_opt ::= */
-  {  255,   -2 }, /* (167) having_opt ::= HAVING expr */
-  {  257,    0 }, /* (168) limit_opt ::= */
-  {  257,   -2 }, /* (169) limit_opt ::= LIMIT signed */
-  {  257,   -4 }, /* (170) limit_opt ::= LIMIT signed OFFSET signed */
-  {  257,   -4 }, /* (171) limit_opt ::= LIMIT signed COMMA signed */
-  {  256,    0 }, /* (172) slimit_opt ::= */
-  {  256,   -2 }, /* (173) slimit_opt ::= SLIMIT signed */
-  {  256,   -4 }, /* (174) slimit_opt ::= SLIMIT signed SOFFSET signed */
-  {  256,   -4 }, /* (175) slimit_opt ::= SLIMIT signed COMMA signed */
-  {  249,    0 }, /* (176) where_opt ::= */
-  {  249,   -2 }, /* (177) where_opt ::= WHERE expr */
-  {  260,   -3 }, /* (178) expr ::= LP expr RP */
-  {  260,   -1 }, /* (179) expr ::= ID */
-  {  260,   -3 }, /* (180) expr ::= ID DOT ID */
-  {  260,   -3 }, /* (181) expr ::= ID DOT STAR */
-  {  260,   -1 }, /* (182) expr ::= INTEGER */
-  {  260,   -2 }, /* (183) expr ::= MINUS INTEGER */
-  {  260,   -2 }, /* (184) expr ::= PLUS INTEGER */
-  {  260,   -1 }, /* (185) expr ::= FLOAT */
-  {  260,   -2 }, /* (186) expr ::= MINUS FLOAT */
-  {  260,   -2 }, /* (187) expr ::= PLUS FLOAT */
-  {  260,   -1 }, /* (188) expr ::= STRING */
-  {  260,   -1 }, /* (189) expr ::= NOW */
-  {  260,   -1 }, /* (190) expr ::= VARIABLE */
-  {  260,   -1 }, /* (191) expr ::= BOOL */
-  {  260,   -4 }, /* (192) expr ::= ID LP exprlist RP */
-  {  260,   -4 }, /* (193) expr ::= ID LP STAR RP */
-  {  260,   -3 }, /* (194) expr ::= expr AND expr */
-  {  260,   -3 }, /* (195) expr ::= expr OR expr */
-  {  260,   -3 }, /* (196) expr ::= expr LT expr */
-  {  260,   -3 }, /* (197) expr ::= expr GT expr */
-  {  260,   -3 }, /* (198) expr ::= expr LE expr */
-  {  260,   -3 }, /* (199) expr ::= expr GE expr */
-  {  260,   -3 }, /* (200) expr ::= expr NE expr */
-  {  260,   -3 }, /* (201) expr ::= expr EQ expr */
-  {  260,   -3 }, /* (202) expr ::= expr PLUS expr */
-  {  260,   -3 }, /* (203) expr ::= expr MINUS expr */
-  {  260,   -3 }, /* (204) expr ::= expr STAR expr */
-  {  260,   -3 }, /* (205) expr ::= expr SLASH expr */
-  {  260,   -3 }, /* (206) expr ::= expr REM expr */
-  {  260,   -3 }, /* (207) expr ::= expr LIKE expr */
-  {  260,   -5 }, /* (208) expr ::= expr IN LP exprlist RP */
-  {  269,   -3 }, /* (209) exprlist ::= exprlist COMMA expritem */
-  {  269,   -1 }, /* (210) exprlist ::= expritem */
-  {  270,   -1 }, /* (211) expritem ::= expr */
-  {  270,    0 }, /* (212) expritem ::= */
-  {  208,   -3 }, /* (213) cmd ::= RESET QUERY CACHE */
-  {  208,   -7 }, /* (214) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
-  {  208,   -7 }, /* (215) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
-  {  208,   -7 }, /* (216) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
-  {  208,   -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
-  {  208,   -8 }, /* (218) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
-  {  208,   -9 }, /* (219) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
-  {  208,   -3 }, /* (220) cmd ::= KILL CONNECTION INTEGER */
-  {  208,   -5 }, /* (221) cmd ::= KILL STREAM INTEGER COLON INTEGER */
-  {  208,   -5 }, /* (222) cmd ::= KILL QUERY INTEGER COLON INTEGER */
+  {  208,   -1 }, /* (0) program ::= cmd */
+  {  209,   -2 }, /* (1) cmd ::= SHOW DATABASES */
+  {  209,   -2 }, /* (2) cmd ::= SHOW MNODES */
+  {  209,   -2 }, /* (3) cmd ::= SHOW DNODES */
+  {  209,   -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
+  {  209,   -2 }, /* (5) cmd ::= SHOW USERS */
+  {  209,   -2 }, /* (6) cmd ::= SHOW MODULES */
+  {  209,   -2 }, /* (7) cmd ::= SHOW QUERIES */
+  {  209,   -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
+  {  209,   -2 }, /* (9) cmd ::= SHOW STREAMS */
+  {  209,   -2 }, /* (10) cmd ::= SHOW CONFIGS */
+  {  209,   -2 }, /* (11) cmd ::= SHOW SCORES */
+  {  209,   -2 }, /* (12) cmd ::= SHOW GRANTS */
+  {  209,   -2 }, /* (13) cmd ::= SHOW VNODES */
+  {  209,   -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
+  {  210,    0 }, /* (15) dbPrefix ::= */
+  {  210,   -2 }, /* (16) dbPrefix ::= ids DOT */
+  {  212,    0 }, /* (17) cpxName ::= */
+  {  212,   -2 }, /* (18) cpxName ::= DOT ids */
+  {  209,   -3 }, /* (19) cmd ::= SHOW dbPrefix TABLES */
+  {  209,   -5 }, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */
+  {  209,   -3 }, /* (21) cmd ::= SHOW dbPrefix STABLES */
+  {  209,   -5 }, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */
+  {  209,   -3 }, /* (23) cmd ::= SHOW dbPrefix VGROUPS */
+  {  209,   -4 }, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */
+  {  209,   -5 }, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */
+  {  209,   -4 }, /* (26) cmd ::= DROP DATABASE ifexists ids */
+  {  209,   -3 }, /* (27) cmd ::= DROP DNODE ids */
+  {  209,   -3 }, /* (28) cmd ::= DROP USER ids */
+  {  209,   -3 }, /* (29) cmd ::= DROP ACCOUNT ids */
+  {  209,   -2 }, /* (30) cmd ::= USE ids */
+  {  209,   -3 }, /* (31) cmd ::= DESCRIBE ids cpxName */
+  {  209,   -5 }, /* (32) cmd ::= ALTER USER ids PASS ids */
+  {  209,   -5 }, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */
+  {  209,   -4 }, /* (34) cmd ::= ALTER DNODE ids ids */
+  {  209,   -5 }, /* (35) cmd ::= ALTER DNODE ids ids ids */
+  {  209,   -3 }, /* (36) cmd ::= ALTER LOCAL ids */
+  {  209,   -4 }, /* (37) cmd ::= ALTER LOCAL ids ids */
+  {  209,   -4 }, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */
+  {  209,   -4 }, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */
+  {  209,   -6 }, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
+  {  211,   -1 }, /* (41) ids ::= ID */
+  {  211,   -1 }, /* (42) ids ::= STRING */
+  {  213,   -2 }, /* (43) ifexists ::= IF EXISTS */
+  {  213,    0 }, /* (44) ifexists ::= */
+  {  216,   -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */
+  {  216,    0 }, /* (46) ifnotexists ::= */
+  {  209,   -3 }, /* (47) cmd ::= CREATE DNODE ids */
+  {  209,   -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
+  {  209,   -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
+  {  209,   -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */
+  {  218,    0 }, /* (51) pps ::= */
+  {  218,   -2 }, /* (52) pps ::= PPS INTEGER */
+  {  219,    0 }, /* (53) tseries ::= */
+  {  219,   -2 }, /* (54) tseries ::= TSERIES INTEGER */
+  {  220,    0 }, /* (55) dbs ::= */
+  {  220,   -2 }, /* (56) dbs ::= DBS INTEGER */
+  {  221,    0 }, /* (57) streams ::= */
+  {  221,   -2 }, /* (58) streams ::= STREAMS INTEGER */
+  {  222,    0 }, /* (59) storage ::= */
+  {  222,   -2 }, /* (60) storage ::= STORAGE INTEGER */
+  {  223,    0 }, /* (61) qtime ::= */
+  {  223,   -2 }, /* (62) qtime ::= QTIME INTEGER */
+  {  224,    0 }, /* (63) users ::= */
+  {  224,   -2 }, /* (64) users ::= USERS INTEGER */
+  {  225,    0 }, /* (65) conns ::= */
+  {  225,   -2 }, /* (66) conns ::= CONNS INTEGER */
+  {  226,    0 }, /* (67) state ::= */
+  {  226,   -2 }, /* (68) state ::= STATE ids */
+  {  215,   -9 }, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
+  {  227,   -2 }, /* (70) keep ::= KEEP tagitemlist */
+  {  229,   -2 }, /* (71) tables ::= MAXTABLES INTEGER */
+  {  230,   -2 }, /* (72) cache ::= CACHE INTEGER */
+  {  231,   -2 }, /* (73) replica ::= REPLICA INTEGER */
+  {  232,   -2 }, /* (74) quorum ::= QUORUM INTEGER */
+  {  233,   -2 }, /* (75) days ::= DAYS INTEGER */
+  {  234,   -2 }, /* (76) minrows ::= MINROWS INTEGER */
+  {  235,   -2 }, /* (77) maxrows ::= MAXROWS INTEGER */
+  {  236,   -2 }, /* (78) blocks ::= BLOCKS INTEGER */
+  {  237,   -2 }, /* (79) ctime ::= CTIME INTEGER */
+  {  238,   -2 }, /* (80) wal ::= WAL INTEGER */
+  {  239,   -2 }, /* (81) fsync ::= FSYNC INTEGER */
+  {  240,   -2 }, /* (82) comp ::= COMP INTEGER */
+  {  241,   -2 }, /* (83) prec ::= PRECISION STRING */
+  {  217,    0 }, /* (84) db_optr ::= */
+  {  217,   -2 }, /* (85) db_optr ::= db_optr tables */
+  {  217,   -2 }, /* (86) db_optr ::= db_optr cache */
+  {  217,   -2 }, /* (87) db_optr ::= db_optr replica */
+  {  217,   -2 }, /* (88) db_optr ::= db_optr quorum */
+  {  217,   -2 }, /* (89) db_optr ::= db_optr days */
+  {  217,   -2 }, /* (90) db_optr ::= db_optr minrows */
+  {  217,   -2 }, /* (91) db_optr ::= db_optr maxrows */
+  {  217,   -2 }, /* (92) db_optr ::= db_optr blocks */
+  {  217,   -2 }, /* (93) db_optr ::= db_optr ctime */
+  {  217,   -2 }, /* (94) db_optr ::= db_optr wal */
+  {  217,   -2 }, /* (95) db_optr ::= db_optr fsync */
+  {  217,   -2 }, /* (96) db_optr ::= db_optr comp */
+  {  217,   -2 }, /* (97) db_optr ::= db_optr prec */
+  {  217,   -2 }, /* (98) db_optr ::= db_optr keep */
+  {  214,    0 }, /* (99) alter_db_optr ::= */
+  {  214,   -2 }, /* (100) alter_db_optr ::= alter_db_optr replica */
+  {  214,   -2 }, /* (101) alter_db_optr ::= alter_db_optr quorum */
+  {  214,   -2 }, /* (102) alter_db_optr ::= alter_db_optr tables */
+  {  214,   -2 }, /* (103) alter_db_optr ::= alter_db_optr keep */
+  {  214,   -2 }, /* (104) alter_db_optr ::= alter_db_optr blocks */
+  {  214,   -2 }, /* (105) alter_db_optr ::= alter_db_optr comp */
+  {  214,   -2 }, /* (106) alter_db_optr ::= alter_db_optr wal */
+  {  214,   -2 }, /* (107) alter_db_optr ::= alter_db_optr fsync */
+  {  242,   -1 }, /* (108) typename ::= ids */
+  {  242,   -4 }, /* (109) typename ::= ids LP signed RP */
+  {  243,   -1 }, /* (110) signed ::= INTEGER */
+  {  243,   -2 }, /* (111) signed ::= PLUS INTEGER */
+  {  243,   -2 }, /* (112) signed ::= MINUS INTEGER */
+  {  209,   -6 }, /* (113) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+  {  244,   -3 }, /* (114) create_table_args ::= LP columnlist RP */
+  {  244,   -7 }, /* (115) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+  {  244,   -7 }, /* (116) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+  {  244,   -2 }, /* (117) create_table_args ::= AS select */
+  {  245,   -3 }, /* (118) columnlist ::= columnlist COMMA column */
+  {  245,   -1 }, /* (119) columnlist ::= column */
+  {  247,   -2 }, /* (120) column ::= ids typename */
+  {  228,   -3 }, /* (121) tagitemlist ::= tagitemlist COMMA tagitem */
+  {  228,   -1 }, /* (122) tagitemlist ::= tagitem */
+  {  248,   -1 }, /* (123) tagitem ::= INTEGER */
+  {  248,   -1 }, /* (124) tagitem ::= FLOAT */
+  {  248,   -1 }, /* (125) tagitem ::= STRING */
+  {  248,   -1 }, /* (126) tagitem ::= BOOL */
+  {  248,   -1 }, /* (127) tagitem ::= NULL */
+  {  248,   -2 }, /* (128) tagitem ::= MINUS INTEGER */
+  {  248,   -2 }, /* (129) tagitem ::= MINUS FLOAT */
+  {  248,   -2 }, /* (130) tagitem ::= PLUS INTEGER */
+  {  248,   -2 }, /* (131) tagitem ::= PLUS FLOAT */
+  {  246,  -12 }, /* (132) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+  {  260,   -1 }, /* (133) union ::= select */
+  {  260,   -3 }, /* (134) union ::= LP union RP */
+  {  260,   -4 }, /* (135) union ::= union UNION ALL select */
+  {  260,   -6 }, /* (136) union ::= union UNION ALL LP select RP */
+  {  209,   -1 }, /* (137) cmd ::= union */
+  {  246,   -2 }, /* (138) select ::= SELECT selcollist */
+  {  261,   -2 }, /* (139) sclp ::= selcollist COMMA */
+  {  261,    0 }, /* (140) sclp ::= */
+  {  249,   -3 }, /* (141) selcollist ::= sclp expr as */
+  {  249,   -2 }, /* (142) selcollist ::= sclp STAR */
+  {  263,   -2 }, /* (143) as ::= AS ids */
+  {  263,   -1 }, /* (144) as ::= ids */
+  {  263,    0 }, /* (145) as ::= */
+  {  250,   -2 }, /* (146) from ::= FROM tablelist */
+  {  264,   -2 }, /* (147) tablelist ::= ids cpxName */
+  {  264,   -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */
+  {  265,   -1 }, /* (149) tmvar ::= VARIABLE */
+  {  252,   -4 }, /* (150) interval_opt ::= INTERVAL LP tmvar RP */
+  {  252,    0 }, /* (151) interval_opt ::= */
+  {  253,    0 }, /* (152) fill_opt ::= */
+  {  253,   -6 }, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+  {  253,   -4 }, /* (154) fill_opt ::= FILL LP ID RP */
+  {  254,   -4 }, /* (155) sliding_opt ::= SLIDING LP tmvar RP */
+  {  254,    0 }, /* (156) sliding_opt ::= */
+  {  256,    0 }, /* (157) orderby_opt ::= */
+  {  256,   -3 }, /* (158) orderby_opt ::= ORDER BY sortlist */
+  {  266,   -4 }, /* (159) sortlist ::= sortlist COMMA item sortorder */
+  {  266,   -2 }, /* (160) sortlist ::= item sortorder */
+  {  268,   -2 }, /* (161) item ::= ids cpxName */
+  {  269,   -1 }, /* (162) sortorder ::= ASC */
+  {  269,   -1 }, /* (163) sortorder ::= DESC */
+  {  269,    0 }, /* (164) sortorder ::= */
+  {  255,    0 }, /* (165) groupby_opt ::= */
+  {  255,   -3 }, /* (166) groupby_opt ::= GROUP BY grouplist */
+  {  270,   -3 }, /* (167) grouplist ::= grouplist COMMA item */
+  {  270,   -1 }, /* (168) grouplist ::= item */
+  {  257,    0 }, /* (169) having_opt ::= */
+  {  257,   -2 }, /* (170) having_opt ::= HAVING expr */
+  {  259,    0 }, /* (171) limit_opt ::= */
+  {  259,   -2 }, /* (172) limit_opt ::= LIMIT signed */
+  {  259,   -4 }, /* (173) limit_opt ::= LIMIT signed OFFSET signed */
+  {  259,   -4 }, /* (174) limit_opt ::= LIMIT signed COMMA signed */
+  {  258,    0 }, /* (175) slimit_opt ::= */
+  {  258,   -2 }, /* (176) slimit_opt ::= SLIMIT signed */
+  {  258,   -4 }, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */
+  {  258,   -4 }, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */
+  {  251,    0 }, /* (179) where_opt ::= */
+  {  251,   -2 }, /* (180) where_opt ::= WHERE expr */
+  {  262,   -3 }, /* (181) expr ::= LP expr RP */
+  {  262,   -1 }, /* (182) expr ::= ID */
+  {  262,   -3 }, /* (183) expr ::= ID DOT ID */
+  {  262,   -3 }, /* (184) expr ::= ID DOT STAR */
+  {  262,   -1 }, /* (185) expr ::= INTEGER */
+  {  262,   -2 }, /* (186) expr ::= MINUS INTEGER */
+  {  262,   -2 }, /* (187) expr ::= PLUS INTEGER */
+  {  262,   -1 }, /* (188) expr ::= FLOAT */
+  {  262,   -2 }, /* (189) expr ::= MINUS FLOAT */
+  {  262,   -2 }, /* (190) expr ::= PLUS FLOAT */
+  {  262,   -1 }, /* (191) expr ::= STRING */
+  {  262,   -1 }, /* (192) expr ::= NOW */
+  {  262,   -1 }, /* (193) expr ::= VARIABLE */
+  {  262,   -1 }, /* (194) expr ::= BOOL */
+  {  262,   -4 }, /* (195) expr ::= ID LP exprlist RP */
+  {  262,   -4 }, /* (196) expr ::= ID LP STAR RP */
+  {  262,   -3 }, /* (197) expr ::= expr AND expr */
+  {  262,   -3 }, /* (198) expr ::= expr OR expr */
+  {  262,   -3 }, /* (199) expr ::= expr LT expr */
+  {  262,   -3 }, /* (200) expr ::= expr GT expr */
+  {  262,   -3 }, /* (201) expr ::= expr LE expr */
+  {  262,   -3 }, /* (202) expr ::= expr GE expr */
+  {  262,   -3 }, /* (203) expr ::= expr NE expr */
+  {  262,   -3 }, /* (204) expr ::= expr EQ expr */
+  {  262,   -3 }, /* (205) expr ::= expr PLUS expr */
+  {  262,   -3 }, /* (206) expr ::= expr MINUS expr */
+  {  262,   -3 }, /* (207) expr ::= expr STAR expr */
+  {  262,   -3 }, /* (208) expr ::= expr SLASH expr */
+  {  262,   -3 }, /* (209) expr ::= expr REM expr */
+  {  262,   -3 }, /* (210) expr ::= expr LIKE expr */
+  {  262,   -5 }, /* (211) expr ::= expr IN LP exprlist RP */
+  {  271,   -3 }, /* (212) exprlist ::= exprlist COMMA expritem */
+  {  271,   -1 }, /* (213) exprlist ::= expritem */
+  {  272,   -1 }, /* (214) expritem ::= expr */
+  {  272,    0 }, /* (215) expritem ::= */
+  {  209,   -3 }, /* (216) cmd ::= RESET QUERY CACHE */
+  {  209,   -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+  {  209,   -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+  {  209,   -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+  {  209,   -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+  {  209,   -8 }, /* (221) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+  {  209,   -9 }, /* (222) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+  {  209,   -3 }, /* (223) cmd ::= KILL CONNECTION INTEGER */
+  {  209,   -5 }, /* (224) cmd ::= KILL STREAM INTEGER COLON INTEGER */
+  {  209,   -5 }, /* (225) cmd ::= KILL QUERY INTEGER COLON INTEGER */
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
@@ -2124,13 +2134,13 @@ static void yy_reduce(
 { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);          }
         break;
       case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */
-{ SSQLToken t = {0};  setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);}
+{ SSQLToken t = {0};  setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);}
         break;
       case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);}
         break;
       case 40: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
         break;
       case 41: /* ids ::= ID */
       case 42: /* ids ::= STRING */ yytestcase(yyruleno==42);
@@ -2151,10 +2161,10 @@ static void yy_reduce(
 { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);}
         break;
       case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
         break;
       case 49: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);}
+{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);}
         break;
       case 50: /* cmd ::= CREATE USER ids PASS ids */
 { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);}
@@ -2183,529 +2193,535 @@ static void yy_reduce(
         break;
       case 69: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */
 {
-    yylhsminor.yy73.maxUsers   = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
-    yylhsminor.yy73.maxDbs     = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
-    yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
-    yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
-    yylhsminor.yy73.maxPointsPerSecond     = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
-    yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
-    yylhsminor.yy73.maxQueryTime   = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
-    yylhsminor.yy73.maxConnections   = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
-    yylhsminor.yy73.stat    = yymsp[0].minor.yy0;
+    yylhsminor.yy149.maxUsers   = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
+    yylhsminor.yy149.maxDbs     = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
+    yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
+    yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
+    yylhsminor.yy149.maxPointsPerSecond     = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
+    yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
+    yylhsminor.yy149.maxQueryTime   = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
+    yylhsminor.yy149.maxConnections   = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
+    yylhsminor.yy149.stat    = yymsp[0].minor.yy0;
 }
-  yymsp[-8].minor.yy73 = yylhsminor.yy73;
+  yymsp[-8].minor.yy149 = yylhsminor.yy149;
         break;
       case 70: /* keep ::= KEEP tagitemlist */
-{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; }
+{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; }
         break;
       case 71: /* tables ::= MAXTABLES INTEGER */
       case 72: /* cache ::= CACHE INTEGER */ yytestcase(yyruleno==72);
       case 73: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==73);
-      case 74: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==74);
-      case 75: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==75);
-      case 76: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==76);
-      case 77: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==77);
-      case 78: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==78);
-      case 79: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==79);
-      case 80: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==80);
-      case 81: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==81);
-      case 82: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==82);
+      case 74: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==74);
+      case 75: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==75);
+      case 76: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==76);
+      case 77: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==77);
+      case 78: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==78);
+      case 79: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==79);
+      case 80: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==80);
+      case 81: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==81);
+      case 82: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==82);
+      case 83: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==83);
 { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
         break;
-      case 83: /* db_optr ::= */
-{setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
+      case 84: /* db_optr ::= */
+{setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
         break;
-      case 84: /* db_optr ::= db_optr tables */
-      case 99: /* alter_db_optr ::= alter_db_optr tables */ yytestcase(yyruleno==99);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxTablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 85: /* db_optr ::= db_optr tables */
+      case 102: /* alter_db_optr ::= alter_db_optr tables */ yytestcase(yyruleno==102);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxTablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 85: /* db_optr ::= db_optr cache */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 86: /* db_optr ::= db_optr cache */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 86: /* db_optr ::= db_optr replica */
-      case 98: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==98);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 87: /* db_optr ::= db_optr replica */
+      case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 87: /* db_optr ::= db_optr days */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 88: /* db_optr ::= db_optr quorum */
+      case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 88: /* db_optr ::= db_optr minrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 89: /* db_optr ::= db_optr days */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 89: /* db_optr ::= db_optr maxrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 90: /* db_optr ::= db_optr minrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 90: /* db_optr ::= db_optr blocks */
-      case 101: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==101);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 91: /* db_optr ::= db_optr maxrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 91: /* db_optr ::= db_optr ctime */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 92: /* db_optr ::= db_optr blocks */
+      case 104: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==104);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 92: /* db_optr ::= db_optr wal */
-      case 103: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==103);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 93: /* db_optr ::= db_optr ctime */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 93: /* db_optr ::= db_optr fsync */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 94: /* db_optr ::= db_optr wal */
+      case 106: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==106);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 94: /* db_optr ::= db_optr comp */
-      case 102: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==102);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 95: /* db_optr ::= db_optr fsync */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 95: /* db_optr ::= db_optr prec */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 96: /* db_optr ::= db_optr comp */
+      case 105: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==105);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 96: /* db_optr ::= db_optr keep */
-      case 100: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==100);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 97: /* db_optr ::= db_optr prec */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 97: /* alter_db_optr ::= */
-{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
+      case 98: /* db_optr ::= db_optr keep */
+      case 103: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==103);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
-      case 104: /* alter_db_optr ::= alter_db_optr fsync */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[-1].minor.yy158 = yylhsminor.yy158;
+      case 99: /* alter_db_optr ::= */
+{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
         break;
-      case 105: /* typename ::= ids */
+      case 107: /* alter_db_optr ::= alter_db_optr fsync */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[-1].minor.yy268 = yylhsminor.yy268;
+        break;
+      case 108: /* typename ::= ids */
 { 
   yymsp[0].minor.yy0.type = 0;
-  tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0); 
+  tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); 
 }
-  yymsp[0].minor.yy181 = yylhsminor.yy181;
+  yymsp[0].minor.yy223 = yylhsminor.yy223;
         break;
-      case 106: /* typename ::= ids LP signed RP */
+      case 109: /* typename ::= ids LP signed RP */
 {
-    if (yymsp[-1].minor.yy271 <= 0) {
+    if (yymsp[-1].minor.yy207 <= 0) {
       yymsp[-3].minor.yy0.type = 0;
-      tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+      tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
     } else {
-      yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271;          // negative value of name length
-      tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+      yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207;          // negative value of name length
+      tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
     }
 }
-  yymsp[-3].minor.yy181 = yylhsminor.yy181;
+  yymsp[-3].minor.yy223 = yylhsminor.yy223;
         break;
-      case 107: /* signed ::= INTEGER */
-{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
-  yymsp[0].minor.yy271 = yylhsminor.yy271;
+      case 110: /* signed ::= INTEGER */
+{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+  yymsp[0].minor.yy207 = yylhsminor.yy207;
         break;
-      case 108: /* signed ::= PLUS INTEGER */
-{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+      case 111: /* signed ::= PLUS INTEGER */
+{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
         break;
-      case 109: /* signed ::= MINUS INTEGER */
-{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
+      case 112: /* signed ::= MINUS INTEGER */
+{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
         break;
-      case 110: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+      case 113: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
 {
     yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
     setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0);
 }
         break;
-      case 111: /* create_table_args ::= LP columnlist RP */
+      case 114: /* create_table_args ::= LP columnlist RP */
 {
-    yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
-    setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
+    setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 112: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+      case 115: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
 {
-    yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE);
-    setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE);
+    setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 113: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+      case 116: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
-    yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
-    setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
+    setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 114: /* create_table_args ::= AS select */
+      case 117: /* create_table_args ::= AS select */
 {
-    yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM);
-    setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+    yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM);
+    setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
-      case 115: /* columnlist ::= columnlist COMMA column */
-{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181);   }
-  yymsp[-2].minor.yy449 = yylhsminor.yy449;
+      case 118: /* columnlist ::= columnlist COMMA column */
+{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223);   }
+  yymsp[-2].minor.yy523 = yylhsminor.yy523;
         break;
-      case 116: /* columnlist ::= column */
-{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);}
-  yymsp[0].minor.yy449 = yylhsminor.yy449;
+      case 119: /* columnlist ::= column */
+{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);}
+  yymsp[0].minor.yy523 = yylhsminor.yy523;
         break;
-      case 117: /* column ::= ids typename */
+      case 120: /* column ::= ids typename */
 {
-    tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181);
+    tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223);
 }
-  yymsp[-1].minor.yy181 = yylhsminor.yy181;
+  yymsp[-1].minor.yy223 = yylhsminor.yy223;
         break;
-      case 118: /* tagitemlist ::= tagitemlist COMMA tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);    }
-  yymsp[-2].minor.yy494 = yylhsminor.yy494;
+      case 121: /* tagitemlist ::= tagitemlist COMMA tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);    }
+  yymsp[-2].minor.yy498 = yylhsminor.yy498;
         break;
-      case 119: /* tagitemlist ::= tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); }
-  yymsp[0].minor.yy494 = yylhsminor.yy494;
+      case 122: /* tagitemlist ::= tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
+  yymsp[0].minor.yy498 = yylhsminor.yy498;
         break;
-      case 120: /* tagitem ::= INTEGER */
-      case 121: /* tagitem ::= FLOAT */ yytestcase(yyruleno==121);
-      case 122: /* tagitem ::= STRING */ yytestcase(yyruleno==122);
-      case 123: /* tagitem ::= BOOL */ yytestcase(yyruleno==123);
-{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
-  yymsp[0].minor.yy312 = yylhsminor.yy312;
+      case 123: /* tagitem ::= INTEGER */
+      case 124: /* tagitem ::= FLOAT */ yytestcase(yyruleno==124);
+      case 125: /* tagitem ::= STRING */ yytestcase(yyruleno==125);
+      case 126: /* tagitem ::= BOOL */ yytestcase(yyruleno==126);
+{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+  yymsp[0].minor.yy134 = yylhsminor.yy134;
         break;
-      case 124: /* tagitem ::= NULL */
-{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
-  yymsp[0].minor.yy312 = yylhsminor.yy312;
+      case 127: /* tagitem ::= NULL */
+{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+  yymsp[0].minor.yy134 = yylhsminor.yy134;
         break;
-      case 125: /* tagitem ::= MINUS INTEGER */
-      case 126: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==126);
-      case 127: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==127);
-      case 128: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==128);
+      case 128: /* tagitem ::= MINUS INTEGER */
+      case 129: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==129);
+      case 130: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==130);
+      case 131: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==131);
 {
     yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
     yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type;
     toTSDBType(yymsp[-1].minor.yy0.type);
-    tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+    tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
 }
-  yymsp[-1].minor.yy312 = yylhsminor.yy312;
+  yymsp[-1].minor.yy134 = yylhsminor.yy134;
         break;
-      case 129: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+      case 132: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
 {
-  yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188);
+  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
 }
-  yymsp[-11].minor.yy150 = yylhsminor.yy150;
+  yymsp[-11].minor.yy414 = yylhsminor.yy414;
         break;
-      case 130: /* union ::= select */
-{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); }
-  yymsp[0].minor.yy25 = yylhsminor.yy25;
+      case 133: /* union ::= select */
+{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); }
+  yymsp[0].minor.yy231 = yylhsminor.yy231;
         break;
-      case 131: /* union ::= LP union RP */
-{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; }
+      case 134: /* union ::= LP union RP */
+{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; }
         break;
-      case 132: /* union ::= union UNION ALL select */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); }
-  yymsp[-3].minor.yy25 = yylhsminor.yy25;
+      case 135: /* union ::= union UNION ALL select */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); }
+  yymsp[-3].minor.yy231 = yylhsminor.yy231;
         break;
-      case 133: /* union ::= union UNION ALL LP select RP */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); }
-  yymsp[-5].minor.yy25 = yylhsminor.yy25;
+      case 136: /* union ::= union UNION ALL LP select RP */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); }
+  yymsp[-5].minor.yy231 = yylhsminor.yy231;
         break;
-      case 134: /* cmd ::= union */
-{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); }
+      case 137: /* cmd ::= union */
+{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); }
         break;
-      case 135: /* select ::= SELECT selcollist */
+      case 138: /* select ::= SELECT selcollist */
 {
-  yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 }
-  yymsp[-1].minor.yy150 = yylhsminor.yy150;
+  yymsp[-1].minor.yy414 = yylhsminor.yy414;
         break;
-      case 136: /* sclp ::= selcollist COMMA */
-{yylhsminor.yy224 = yymsp[-1].minor.yy224;}
-  yymsp[-1].minor.yy224 = yylhsminor.yy224;
+      case 139: /* sclp ::= selcollist COMMA */
+{yylhsminor.yy290 = yymsp[-1].minor.yy290;}
+  yymsp[-1].minor.yy290 = yylhsminor.yy290;
         break;
-      case 137: /* sclp ::= */
-{yymsp[1].minor.yy224 = 0;}
+      case 140: /* sclp ::= */
+{yymsp[1].minor.yy290 = 0;}
         break;
-      case 138: /* selcollist ::= sclp expr as */
+      case 141: /* selcollist ::= sclp expr as */
 {
-   yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
+   yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
 }
-  yymsp[-2].minor.yy224 = yylhsminor.yy224;
+  yymsp[-2].minor.yy290 = yylhsminor.yy290;
         break;
-      case 139: /* selcollist ::= sclp STAR */
+      case 142: /* selcollist ::= sclp STAR */
 {
    tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL);
-   yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0);
+   yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0);
 }
-  yymsp[-1].minor.yy224 = yylhsminor.yy224;
+  yymsp[-1].minor.yy290 = yylhsminor.yy290;
         break;
-      case 140: /* as ::= AS ids */
+      case 143: /* as ::= AS ids */
 { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;    }
         break;
-      case 141: /* as ::= ids */
+      case 144: /* as ::= ids */
 { yylhsminor.yy0 = yymsp[0].minor.yy0;    }
   yymsp[0].minor.yy0 = yylhsminor.yy0;
         break;
-      case 142: /* as ::= */
+      case 145: /* as ::= */
 { yymsp[1].minor.yy0.n = 0;  }
         break;
-      case 143: /* from ::= FROM tablelist */
-{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;}
+      case 146: /* from ::= FROM tablelist */
+{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;}
         break;
-      case 144: /* tablelist ::= ids cpxName */
-{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);}
-  yymsp[-1].minor.yy494 = yylhsminor.yy494;
+      case 147: /* tablelist ::= ids cpxName */
+{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);}
+  yymsp[-1].minor.yy498 = yylhsminor.yy498;
         break;
-      case 145: /* tablelist ::= tablelist COMMA ids cpxName */
-{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1);   }
-  yymsp[-3].minor.yy494 = yylhsminor.yy494;
+      case 148: /* tablelist ::= tablelist COMMA ids cpxName */
+{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1);   }
+  yymsp[-3].minor.yy498 = yylhsminor.yy498;
         break;
-      case 146: /* tmvar ::= VARIABLE */
+      case 149: /* tmvar ::= VARIABLE */
 {yylhsminor.yy0 = yymsp[0].minor.yy0;}
   yymsp[0].minor.yy0 = yylhsminor.yy0;
         break;
-      case 147: /* interval_opt ::= INTERVAL LP tmvar RP */
-      case 152: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==152);
+      case 150: /* interval_opt ::= INTERVAL LP tmvar RP */
+      case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==155);
 {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0;     }
         break;
-      case 148: /* interval_opt ::= */
-      case 153: /* sliding_opt ::= */ yytestcase(yyruleno==153);
+      case 151: /* interval_opt ::= */
+      case 156: /* sliding_opt ::= */ yytestcase(yyruleno==156);
 {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0;   }
         break;
-      case 149: /* fill_opt ::= */
-{yymsp[1].minor.yy494 = 0;     }
+      case 152: /* fill_opt ::= */
+{yymsp[1].minor.yy498 = 0;     }
         break;
-      case 150: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+      case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
 {
     tVariant A = {0};
     toTSDBType(yymsp[-3].minor.yy0.type);
     tVariantCreate(&A, &yymsp[-3].minor.yy0);
 
-    tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0);
-    yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494;
+    tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0);
+    yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498;
 }
         break;
-      case 151: /* fill_opt ::= FILL LP ID RP */
+      case 154: /* fill_opt ::= FILL LP ID RP */
 {
     toTSDBType(yymsp[-1].minor.yy0.type);
-    yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+    yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
 }
         break;
-      case 154: /* orderby_opt ::= */
-      case 162: /* groupby_opt ::= */ yytestcase(yyruleno==162);
-{yymsp[1].minor.yy494 = 0;}
+      case 157: /* orderby_opt ::= */
+      case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165);
+{yymsp[1].minor.yy498 = 0;}
         break;
-      case 155: /* orderby_opt ::= ORDER BY sortlist */
-      case 163: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==163);
-{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;}
+      case 158: /* orderby_opt ::= ORDER BY sortlist */
+      case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166);
+{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;}
         break;
-      case 156: /* sortlist ::= sortlist COMMA item sortorder */
+      case 159: /* sortlist ::= sortlist COMMA item sortorder */
 {
-    yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+    yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-3].minor.yy494 = yylhsminor.yy494;
+  yymsp[-3].minor.yy498 = yylhsminor.yy498;
         break;
-      case 157: /* sortlist ::= item sortorder */
+      case 160: /* sortlist ::= item sortorder */
 {
-  yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-1].minor.yy494 = yylhsminor.yy494;
+  yymsp[-1].minor.yy498 = yylhsminor.yy498;
         break;
-      case 158: /* item ::= ids cpxName */
+      case 161: /* item ::= ids cpxName */
 {
   toTSDBType(yymsp[-1].minor.yy0.type);
   yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
 
-  tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+  tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
 }
-  yymsp[-1].minor.yy312 = yylhsminor.yy312;
+  yymsp[-1].minor.yy134 = yylhsminor.yy134;
         break;
-      case 159: /* sortorder ::= ASC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; }
+      case 162: /* sortorder ::= ASC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; }
         break;
-      case 160: /* sortorder ::= DESC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;}
+      case 163: /* sortorder ::= DESC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;}
         break;
-      case 161: /* sortorder ::= */
-{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;}
+      case 164: /* sortorder ::= */
+{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;}
         break;
-      case 164: /* grouplist ::= grouplist COMMA item */
+      case 167: /* grouplist ::= grouplist COMMA item */
 {
-  yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);
+  yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[-2].minor.yy494 = yylhsminor.yy494;
+  yymsp[-2].minor.yy498 = yylhsminor.yy498;
         break;
-      case 165: /* grouplist ::= item */
+      case 168: /* grouplist ::= item */
 {
-  yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1);
+  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[0].minor.yy494 = yylhsminor.yy494;
+  yymsp[0].minor.yy498 = yylhsminor.yy498;
         break;
-      case 166: /* having_opt ::= */
-      case 176: /* where_opt ::= */ yytestcase(yyruleno==176);
-      case 212: /* expritem ::= */ yytestcase(yyruleno==212);
-{yymsp[1].minor.yy66 = 0;}
+      case 169: /* having_opt ::= */
+      case 179: /* where_opt ::= */ yytestcase(yyruleno==179);
+      case 215: /* expritem ::= */ yytestcase(yyruleno==215);
+{yymsp[1].minor.yy64 = 0;}
         break;
-      case 167: /* having_opt ::= HAVING expr */
-      case 177: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==177);
-{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;}
+      case 170: /* having_opt ::= HAVING expr */
+      case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180);
+{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;}
         break;
-      case 168: /* limit_opt ::= */
-      case 172: /* slimit_opt ::= */ yytestcase(yyruleno==172);
-{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;}
+      case 171: /* limit_opt ::= */
+      case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175);
+{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
         break;
-      case 169: /* limit_opt ::= LIMIT signed */
-      case 173: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==173);
-{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271;  yymsp[-1].minor.yy188.offset = 0;}
+      case 172: /* limit_opt ::= LIMIT signed */
+      case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176);
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
         break;
-      case 170: /* limit_opt ::= LIMIT signed OFFSET signed */
-      case 174: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==174);
-{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271;  yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;}
+      case 173: /* limit_opt ::= LIMIT signed OFFSET signed */
+      case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177);
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
         break;
-      case 171: /* limit_opt ::= LIMIT signed COMMA signed */
-      case 175: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==175);
-{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271;  yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;}
+      case 174: /* limit_opt ::= LIMIT signed COMMA signed */
+      case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178);
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
         break;
-      case 178: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; }
+      case 181: /* expr ::= LP expr RP */
+{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; }
         break;
-      case 179: /* expr ::= ID */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 182: /* expr ::= ID */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 180: /* expr ::= ID DOT ID */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 183: /* expr ::= ID DOT ID */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 181: /* expr ::= ID DOT STAR */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 184: /* expr ::= ID DOT STAR */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 182: /* expr ::= INTEGER */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 185: /* expr ::= INTEGER */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 183: /* expr ::= MINUS INTEGER */
-      case 184: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==184);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
-  yymsp[-1].minor.yy66 = yylhsminor.yy66;
+      case 186: /* expr ::= MINUS INTEGER */
+      case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
+  yymsp[-1].minor.yy64 = yylhsminor.yy64;
         break;
-      case 185: /* expr ::= FLOAT */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 188: /* expr ::= FLOAT */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 186: /* expr ::= MINUS FLOAT */
-      case 187: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==187);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
-  yymsp[-1].minor.yy66 = yylhsminor.yy66;
+      case 189: /* expr ::= MINUS FLOAT */
+      case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
+  yymsp[-1].minor.yy64 = yylhsminor.yy64;
         break;
-      case 188: /* expr ::= STRING */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 191: /* expr ::= STRING */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 189: /* expr ::= NOW */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 192: /* expr ::= NOW */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 190: /* expr ::= VARIABLE */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 193: /* expr ::= VARIABLE */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 191: /* expr ::= BOOL */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 194: /* expr ::= BOOL */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 192: /* expr ::= ID LP exprlist RP */
+      case 195: /* expr ::= ID LP exprlist RP */
 {
-  yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
+  yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
 }
-  yymsp[-3].minor.yy66 = yylhsminor.yy66;
+  yymsp[-3].minor.yy64 = yylhsminor.yy64;
         break;
-      case 193: /* expr ::= ID LP STAR RP */
+      case 196: /* expr ::= ID LP STAR RP */
 {
-  yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
+  yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
 }
-  yymsp[-3].minor.yy66 = yylhsminor.yy66;
+  yymsp[-3].minor.yy64 = yylhsminor.yy64;
         break;
-      case 194: /* expr ::= expr AND expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 197: /* expr ::= expr AND expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 195: /* expr ::= expr OR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 198: /* expr ::= expr OR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 196: /* expr ::= expr LT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 199: /* expr ::= expr LT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 197: /* expr ::= expr GT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 200: /* expr ::= expr GT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 198: /* expr ::= expr LE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 201: /* expr ::= expr LE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 199: /* expr ::= expr GE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 202: /* expr ::= expr GE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 200: /* expr ::= expr NE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 203: /* expr ::= expr NE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 201: /* expr ::= expr EQ expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 204: /* expr ::= expr EQ expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 202: /* expr ::= expr PLUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 205: /* expr ::= expr PLUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 203: /* expr ::= expr MINUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 206: /* expr ::= expr MINUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 204: /* expr ::= expr STAR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 207: /* expr ::= expr STAR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 205: /* expr ::= expr SLASH expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);}
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 208: /* expr ::= expr SLASH expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 206: /* expr ::= expr REM expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM);   }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 209: /* expr ::= expr REM expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM);   }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 207: /* expr ::= expr LIKE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE);  }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 210: /* expr ::= expr LIKE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE);  }
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
-      case 208: /* expr ::= expr IN LP exprlist RP */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); }
-  yymsp[-4].minor.yy66 = yylhsminor.yy66;
+      case 211: /* expr ::= expr IN LP exprlist RP */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); }
+  yymsp[-4].minor.yy64 = yylhsminor.yy64;
         break;
-      case 209: /* exprlist ::= exprlist COMMA expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);}
-  yymsp[-2].minor.yy224 = yylhsminor.yy224;
+      case 212: /* exprlist ::= exprlist COMMA expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);}
+  yymsp[-2].minor.yy290 = yylhsminor.yy290;
         break;
-      case 210: /* exprlist ::= expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);}
-  yymsp[0].minor.yy224 = yylhsminor.yy224;
+      case 213: /* exprlist ::= expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);}
+  yymsp[0].minor.yy290 = yylhsminor.yy290;
         break;
-      case 211: /* expritem ::= expr */
-{yylhsminor.yy66 = yymsp[0].minor.yy66;}
-  yymsp[0].minor.yy66 = yylhsminor.yy66;
+      case 214: /* expritem ::= expr */
+{yylhsminor.yy64 = yymsp[0].minor.yy64;}
+  yymsp[0].minor.yy64 = yylhsminor.yy64;
         break;
-      case 213: /* cmd ::= RESET QUERY CACHE */
+      case 216: /* cmd ::= RESET QUERY CACHE */
 { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
         break;
-      case 214: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+      case 217: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 215: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+      case 218: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2716,14 +2732,14 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 216: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+      case 219: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 217: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+      case 220: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2734,7 +2750,7 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 218: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+      case 221: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
 
@@ -2748,25 +2764,25 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 219: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+      case 222: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
 {
     yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
 
     toTSDBType(yymsp[-2].minor.yy0.type);
     tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
-    A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1);
+    A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 220: /* cmd ::= KILL CONNECTION INTEGER */
+      case 223: /* cmd ::= KILL CONNECTION INTEGER */
 {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);}
         break;
-      case 221: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
+      case 224: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);}
         break;
-      case 222: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
+      case 225: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);}
         break;
       default:
diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp
index 63ed89ab9f..e9611a3232 100644
--- a/src/query/tests/resultBufferTest.cpp
+++ b/src/query/tests/resultBufferTest.cpp
@@ -10,7 +10,7 @@ namespace {
 // simple test
 void simpleTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -18,17 +18,144 @@ void simpleTest() {
   tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
   ASSERT_TRUE(pBufPage != NULL);
   
-  ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
-  ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
+  ASSERT_EQ(getResBufSize(pResultBuf), 1024);
   
   SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(list), 1);
   ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
-  
-  destroyResultBuf(pResultBuf, NULL);
+
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+
+  tFilePage* t = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t == pBufPage1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void writeDownTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+
+  *(int32_t*)(pBufPage->data) = nx;
+  writePageId = pageId;
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void recyclePageTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+  releaseResBufPage(pResultBuf, t4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t5 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t5 == pBufPage5);
+  ASSERT_TRUE(pageId == 5);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  *(int32_t*)(pBufPagex->data) = nx;
+  writePageId = pageId;   // update the data
+  releaseResBufPage(pResultBuf, pBufPagex);
+
+  tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 6);
+
+  destroyResultBuf(pResultBuf);
 }
 } // namespace
 
+
 TEST(testCase, resultBufferTest) {
+  srand(time(NULL));
   simpleTest();
+  writeDownTest();
+  recyclePageTest();
 }
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index 96c63b45ca..902c8b66e4 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
@@ -12,10 +12,11 @@ INCLUDE_DIRECTORIES(inc)
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(./src SRC)
 ELSEIF (TD_DARWIN_64)
-  LIST(APPEND SRC ./src/thaship.c)
-  LIST(APPEND SRC ./src/trpc.c)
-  LIST(APPEND SRC ./src/tstring.c)
-  LIST(APPEND SRC ./src/tudp.c)
+  #LIST(APPEND SRC ./src/thaship.c)
+  #LIST(APPEND SRC ./src/trpc.c)
+  #LIST(APPEND SRC ./src/tstring.c)
+  #LIST(APPEND SRC ./src/tudp.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
 ELSEIF (TD_WINDOWS_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   LIST(APPEND SRC ./src/thaship.c)
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 95840af1a2..7425558535 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -626,6 +626,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   pConn->pReqMsg = NULL;
   pConn->reqMsgLen = 0;
   pConn->pContext = NULL;
+  pConn->chandle = NULL;
 
   taosFreeId(pRpc->idPool, pConn->sid);
   tDebug("%s, rpc connection is released", pConn->info);
@@ -656,7 +657,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->sid = sid;
     pConn->tranId = (uint16_t)(random() & 0xFFFF);
     pConn->ownId = htonl(pConn->sid);
-    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid());
+    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid() + (int64_t)pConn->tranId);
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
@@ -913,7 +914,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
   }
 
   sid = pConn->sid;
-  pConn->chandle = pRecv->chandle;
+  if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle;
   pConn->peerIp = pRecv->ip; 
   pConn->peerPort = pRecv->port;
   if (pHead->port) pConn->peerPort = htons(pHead->port); 
@@ -1015,7 +1016,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     if (code != 0) { // parsing error
       if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
-        tDebug("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
+        tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 
       rpcProcessIncomingMsg(pConn, pHead);
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index decd7a8307..bdaae4c597 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -242,7 +242,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
   destAdd.sin_addr.s_addr = ip;
   destAdd.sin_port = htons(port);
 
-  int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
+  int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
 
   return ret;
 }
diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt
index 6787de56f6..ac7ea8e4c1 100644
--- a/src/rpc/test/CMakeLists.txt
+++ b/src/rpc/test/CMakeLists.txt
@@ -1,13 +1,13 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
 
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   LIST(APPEND CLIENT_SRC ./rclient.c)
   ADD_EXECUTABLE(rclient ${CLIENT_SRC})
   TARGET_LINK_LIBRARIES(rclient trpc)
diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt
new file mode 100644
index 0000000000..ca6a74f695
--- /dev/null
+++ b/src/sync/CMakeLists.txt
@@ -0,0 +1,23 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  LIST(REMOVE_ITEM SRC ./src/tarbitrator.c)
+
+  ADD_LIBRARY(sync ${SRC})
+  TARGET_LINK_LIBRARIES(sync tutil pthread common)
+
+  LIST(APPEND BIN_SRC ./src/tarbitrator.c)
+  LIST(APPEND BIN_SRC ./src/taosTcpPool.c)
+  ADD_EXECUTABLE(tarbitrator ${BIN_SRC})
+  TARGET_LINK_LIBRARIES(tarbitrator tutil sync common)
+
+  ADD_SUBDIRECTORY(test)
+ENDIF ()
diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h
new file mode 100644
index 0000000000..be1e01cb23
--- /dev/null
+++ b/src/sync/inc/syncInt.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_SYNCINT_H
+#define TDENGINE_SYNCINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }}
+#define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }}
+#define sWarn(...)  { if (sDebugFlag & DEBUG_WARN)  { taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); }}
+#define sInfo(...)  { if (sDebugFlag & DEBUG_INFO)  { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+
+#define TAOS_SMSG_SYNC_DATA    1
+#define TAOS_SMSG_FORWARD      2
+#define TAOS_SMSG_FORWARD_RSP  3
+#define TAOS_SMSG_SYNC_REQ     4 
+#define TAOS_SMSG_SYNC_RSP     5
+#define TAOS_SMSG_SYNC_MUST    6
+#define TAOS_SMSG_STATUS       7
+
+#define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
+#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
+#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
+
+#pragma pack(push, 1)
+
+typedef struct {
+  char     type;        // msg type
+  char     pversion;    // protocol version
+  char     reserved[6]; // not used
+  int32_t  vgId;        // vg ID
+  int32_t  len;         // content length, does not include head
+  // char     cont[];      // message content starts from here
+} SSyncHead;
+
+typedef struct {
+  SSyncHead syncHead;
+  uint16_t  port;
+  char      fqdn[TSDB_FQDN_LEN];
+  int32_t   sourceId;  // only for arbitrator
+} SFirstPkt;
+
+typedef struct {
+  int8_t    role;
+  uint64_t  version;
+} SPeerStatus;
+
+typedef struct {
+  int8_t      role;
+  int8_t      ack;
+  uint64_t    version;
+  SPeerStatus peersStatus[];
+} SPeersStatus;
+
+typedef struct {
+  char      name[TSDB_FILENAME_LEN];
+  uint32_t  magic;
+  uint32_t  index;
+  uint64_t  fversion;
+  int32_t   size;
+} SFileInfo;
+
+typedef struct {
+  int8_t    sync;
+} SFileAck;
+
+typedef struct {
+  uint64_t  version;
+  int32_t   code;
+} SFwdRsp;
+  
+#pragma pack(pop)
+
+typedef struct {
+  char  *buffer;
+  int    bufferSize;
+  char  *offset;
+  int    forwards;
+  int    code;
+} SRecvBuffer;
+
+typedef struct {
+  uint64_t  version;
+  void     *mhandle;
+  int8_t    acks;
+  int8_t    nacks;
+  int8_t    confirmed;
+  int32_t   code;
+  uint64_t  time;
+} SFwdInfo;
+
+typedef struct {
+  int       first;
+  int       last;
+  int       fwds;  // number of forwards
+  SFwdInfo  fwdInfo[];
+} SSyncFwds;
+
+typedef struct SsyncPeer {
+  int32_t     nodeId;
+  uint32_t    ip;
+  uint16_t    port;
+  char        fqdn[TSDB_FQDN_LEN];  // peer ip string
+  char        id[TSDB_EP_LEN+16];   // peer vgId + end point
+  int8_t      role;
+  int8_t      sstatus;    // sync status
+  uint64_t    version;
+  uint64_t    sversion;   // track the peer version in retrieve process 
+  int         syncFd;
+  int         peerFd;     // forward FD
+  int         numOfRetrieves; // number of retrieves tried 
+  int         fileChanged;    // a flag to indicate file is changed during retrieving process
+  void       *timer;
+  void       *pConn;
+  int         notifyFd;
+  int         watchNum;
+  int        *watchFd;
+  int8_t      refCount;   // reference count
+  struct SSyncNode *pSyncNode;
+} SSyncPeer;
+
+typedef struct SSyncNode {
+  char         path[TSDB_FILENAME_LEN];
+  int8_t       replica;
+  int8_t       quorum;
+  uint32_t     vgId;
+  void        *ahandle;
+  int8_t       selfIndex;
+  SSyncPeer   *peerInfo[TAOS_SYNC_MAX_REPLICA+1];  // extra one for arbitrator
+  SSyncPeer   *pMaster;
+  int8_t       refCount;
+  SRecvBuffer *pRecv;
+  SSyncFwds   *pSyncFwds;  // saved forward info if quorum >1
+  void        *pFwdTimer;
+  FGetFileInfo    getFileInfo;
+  FGetWalInfo     getWalInfo;
+  FWriteToCache   writeToCache;
+  FConfirmForward confirmForward;
+  FNotifyRole     notifyRole;
+  FNotifyFlowCtrl notifyFlowCtrl;
+  FNotifyFileSynced notifyFileSynced;
+  pthread_mutex_t mutex;
+} SSyncNode;
+
+// sync module global
+extern int  tsSyncNum;
+extern char tsNodeFqdn[TSDB_FQDN_LEN];
+
+void *syncRetrieveData(void *param);
+void *syncRestoreData(void *param);
+int   syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
+void  syncRestartConnection(SSyncPeer *pPeer);
+void  syncBroadcastStatus(SSyncNode *pNode);
+void  syncAddPeerRef(SSyncPeer *pPeer);
+int   syncDecPeerRef(SSyncPeer *pPeer);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_VNODEPEER_H
diff --git a/src/os/linux/inc/tsysctl.h b/src/sync/inc/syncMain.h
similarity index 80%
rename from src/os/linux/inc/tsysctl.h
rename to src/sync/inc/syncMain.h
index 559d5653ee..d4ddb12733 100644
--- a/src/os/linux/inc/tsysctl.h
+++ b/src/sync/inc/syncMain.h
@@ -13,18 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSCTL_H
-#define TDENGINE_TSYSCTL_H
+#ifndef TDENGINE_PLUGINS_SYNC_H
+#define TDENGINE_PLUGINS_SYNC_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef _ALPINE
-  #include <error.h>
-  #include <sys/sysctl.h>
-#else
-  #include <linux/sysctl.h>  
+#include <stdbool.h>
+#include <stdint.h>
+
+int32_t syncTest1();
+int32_t syncTest2();
+
+#ifdef __cplusplus
+}
 #endif
 
 #endif
diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h
new file mode 100644
index 0000000000..1e410acc26
--- /dev/null
+++ b/src/sync/inc/taosTcpPool.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TCP_POOL_H
+#define TDENGINE_TCP_POOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* ttpool_h;
+typedef void* tthread_h;
+
+typedef struct {
+  int       numOfThreads;
+  uint32_t  serverIp;
+  short     port;
+  int       bufferSize;
+  void     (*processBrokenLink)(void *ahandle);
+  int      (*processIncomingMsg)(void *ahandle, void *buffer);
+  void     (*processIncomingConn)(int fd, uint32_t ip);
+} SPoolInfo;
+
+ttpool_h   taosOpenTcpThreadPool(SPoolInfo *pInfo);
+void       taosCloseTcpThreadPool(ttpool_h);
+void      *taosAllocateTcpConn(void *, void *ahandle, int connFd);
+void       taosFreeTcpConn(void *);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TCP_POOL_H
+                                   
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
new file mode 100644
index 0000000000..5455529f62
--- /dev/null
+++ b/src/sync/src/syncMain.c
@@ -0,0 +1,1226 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#include <stdint.h>
+//#include <stdbool.h>
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+// global configurable
+int       tsMaxSyncNum = 4;
+int       tsSyncTcpThreads = 2;
+int       tsMaxWatchFiles = 100;
+int       tsMaxFwdInfo = 200;
+int       tsSyncTimer = 1;
+//int       sDebugFlag = 135;
+//char      tsArbitrator[TSDB_FQDN_LEN] = {0};
+
+// module global, not configurable
+int       tsSyncNum;    // number of sync in process in whole system
+char      tsNodeFqdn[TSDB_FQDN_LEN];
+
+static int            tsNodeNum;    // number of nodes in system
+static ttpool_h       tsTcpPool;
+static void          *syncTmrCtrl = NULL;
+static void          *vgIdHash;
+static pthread_once_t syncModuleInit = PTHREAD_ONCE_INIT;
+
+// local functions
+static void  syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
+static void  syncRecoverFromMaster(SSyncPeer *pPeer);
+static void  syncCheckPeerConnection(void *param, void *tmrId);
+static void  syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void  syncProcessBrokenLink(void *param);
+static int   syncProcessPeerMsg(void *param, void *buffer);
+static void  syncProcessIncommingConnection(int connFd, uint32_t sourceIp); 
+static void  syncRemovePeer(SSyncPeer *pPeer);
+static void  syncAddArbitrator(SSyncNode *pNode);
+static void  syncAddNodeRef(SSyncNode *pNode);
+static void  syncDecNodeRef(SSyncNode *pNode);
+static void  syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
+static void  syncMonitorFwdInfos(void *param, void *tmrId);
+static void  syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
+static void  syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); 
+static void  syncRestartPeer(SSyncPeer *pPeer);
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
+
+char* syncRole[] = {
+  "offline",
+  "unsynced",
+  "syncing",
+  "slave",
+  "master"
+};
+
+static void syncModuleInitFunc() {
+  SPoolInfo info;
+
+  info.numOfThreads = tsSyncTcpThreads;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = syncProcessBrokenLink;
+  info.processIncomingMsg = syncProcessPeerMsg;
+  info.processIncomingConn = syncProcessIncommingConnection;
+
+  tsTcpPool = taosOpenTcpThreadPool(&info);
+  if (tsTcpPool == NULL) return;
+
+  syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
+  if (syncTmrCtrl == NULL) {
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    return;
+  }
+    
+  vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); 
+  if (vgIdHash == NULL) {
+    taosTmrCleanUp(syncTmrCtrl);
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    syncTmrCtrl = NULL;
+    return;
+  } 
+
+  tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn));
+}
+
+void *syncStart(const SSyncInfo *pInfo) 
+{
+  const SSyncCfg *pCfg = &pInfo->syncCfg;
+
+  SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1);
+  if (pNode == NULL) {
+    sError("no memory to allocate syncNode");
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return NULL;
+  }
+
+  pthread_once(&syncModuleInit, syncModuleInitFunc); 
+  if (tsTcpPool == NULL) {
+    free(pNode);
+    syncModuleInit = PTHREAD_ONCE_INIT;
+    sError("failed to init sync module(%s)", tstrerror(errno));
+    return NULL;
+  }
+
+  atomic_add_fetch_32(&tsNodeNum, 1);
+  tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
+  pthread_mutex_init(&pNode->mutex, NULL);
+
+  pNode->ahandle = pInfo->ahandle;
+  pNode->getFileInfo = pInfo->getFileInfo;
+  pNode->getWalInfo = pInfo->getWalInfo;
+  pNode->writeToCache = pInfo->writeToCache;
+  pNode->notifyRole = pInfo->notifyRole;
+  pNode->confirmForward = pInfo->confirmForward;
+  pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
+  pNode->notifyFileSynced = pInfo->notifyFileSynced;
+ 
+  pNode->selfIndex = -1;
+  pNode->vgId = pInfo->vgId;
+  pNode->replica = pCfg->replica;
+  pNode->quorum = pCfg->quorum;
+  if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
+
+  for (int i = 0; i < pCfg->replica; ++i) {
+    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
+    pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
+    if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  if (pNode->selfIndex < 0) {
+    sInfo("vgId:%d, this node is not configured", pNode->vgId);
+    terrno = TSDB_CODE_SYN_INVALID_CONFIG;
+    syncStop(pNode);
+    return NULL;
+  }
+
+  nodeVersion = pInfo->version;    // set the initial version
+  nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER;
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+
+  pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo*sizeof(SFwdInfo), 1);
+  if (pNode->pSyncFwds == NULL) {
+    sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+  if (pNode->pFwdTimer == NULL) {
+    sError("vgId:%d, failed to allocate timer", pNode->vgId);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  syncAddArbitrator(pNode);
+  syncAddNodeRef(pNode);
+  taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+ 
+  if (pNode->notifyRole) 
+   (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+
+  return pNode;
+}
+
+void syncStop(void *param) 
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+
+  if (pNode == NULL) return;
+  sInfo("vgId:%d, cleanup sync", pNode->vgId);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer) syncRemovePeer(pPeer); 
+  }
+
+  pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pPeer) syncRemovePeer(pPeer);
+
+  if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+  if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  syncDecNodeRef(pNode);
+}
+
+int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) 
+{
+  SSyncNode  *pNode = param;
+  int         i, j;
+
+  if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], 
+         pNewCfg->replica, pNode->replica);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (i = 0; i < pNode->replica; ++i) {
+    for (j = 0; j < pNewCfg->replica; ++j) {
+      if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && 
+          (pNode->peerInfo[i]->port == pNewCfg->nodeInfo[j].nodePort)) 
+        break;
+    }
+
+    if (j >= pNewCfg->replica) {
+      syncRemovePeer(pNode->peerInfo[i]);
+      pNode->peerInfo[i] = NULL;
+    }
+  }
+
+  SSyncPeer *newPeers[TAOS_SYNC_MAX_REPLICA];
+  for (i = 0; i < pNewCfg->replica; ++i) {
+    const SNodeInfo *pNewNode = &pNewCfg->nodeInfo[i];
+
+    for (j = 0; j < pNode->replica; ++j) {
+      if (pNode->peerInfo[j] && (strcmp(pNode->peerInfo[j]->fqdn, pNewNode->nodeFqdn) == 0) && 
+         (pNode->peerInfo[j]->port == pNewNode->nodePort))
+        break;
+    }
+
+    if (j >= pNode->replica) {
+      newPeers[i] = syncAddPeer(pNode, pNewNode);
+    } else {
+      newPeers[i] = pNode->peerInfo[j];
+    }
+
+    if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  pNode->replica = pNewCfg->replica;
+  pNode->quorum = pNewCfg->quorum;
+  if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
+  memcpy(pNode->peerInfo, newPeers, sizeof(SSyncPeer *) * pNewCfg->replica);
+
+  for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i)
+    pNode->peerInfo[i] = NULL;
+
+  syncAddArbitrator(pNode);
+
+  if (pNewCfg->replica <= 1) {
+    sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
+    nodeRole = TAOS_SYNC_ROLE_MASTER;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+  syncBroadcastStatus(pNode);
+
+  return 0;
+}
+
+int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype)
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+  SSyncHead  *pSyncHead;
+  SWalHead   *pWalHead = data;
+  int         fwdLen;
+  int         code = 0;
+
+  if (pNode == NULL) return 0;
+
+  // always update version
+  nodeVersion = pWalHead->version;
+  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER ) return 0;
+
+  // only pkt from RPC or CQ can be forwarded
+  if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+  // a hacker way to improve the performance
+  pSyncHead = (SSyncHead *) ( ((char *)pWalHead) - sizeof(SSyncHead));
+  pSyncHead->type = TAOS_SMSG_FORWARD;
+  pSyncHead->pversion = 0;
+  pSyncHead->len = sizeof(SWalHead) + pWalHead->len; 
+  fwdLen = pSyncHead->len + sizeof(SSyncHead);  //include the WAL and SYNC head 
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  if (pNode->quorum > 1) {
+    syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+    code = 1;
+  }
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer == NULL || pPeer->peerFd <0) continue; 
+    if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; 
+  
+    int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+    if (retLen == fwdLen) {
+      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+    } else {
+      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+void syncConfirmForward(void *param, uint64_t version, int32_t code)
+{
+  SSyncNode  *pNode = param;
+  if (pNode == NULL) return;
+  if (pNode->quorum <= 1) return;
+
+  SSyncPeer  *pPeer = pNode->pMaster;
+  if (pPeer == NULL) return;
+
+  char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+
+  SSyncHead   *pHead = (SSyncHead *) msg;
+  pHead->type = TAOS_SMSG_FORWARD_RSP;
+  pHead->len = sizeof(SFwdRsp);
+
+  SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+  pFwdRsp->version = version;
+  pFwdRsp->code = code;
+
+  int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+  int retLen = write(pPeer->peerFd, msg, msgLen);
+
+  if (retLen == msgLen) {
+    sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
+  } else {
+    sDebug("%s, failed to send forward ack, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+}
+
+void syncRecover(void *param) {
+  SSyncNode *pNode = param;
+  SSyncPeer *pPeer;
+
+  // to do: add a few lines to check if recover is OK 
+  // if take this node to unsync state, the whole system may not work
+
+  nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  nodeVersion = 0;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = (SSyncPeer *) pNode->peerInfo[i];
+    if (pPeer->peerFd >= 0) {
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+int syncGetNodesRole(void *param, SNodesRole *pNodesRole)
+{
+  SSyncNode *pNode = param;
+  
+  pNodesRole->selfIndex = pNode->selfIndex;
+  for (int i=0; i<pNode->replica; ++i) {
+    pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
+    pNodesRole->role[i] = pNode->peerInfo[i]->role;
+  }
+
+  return 0;
+}
+
+static void syncAddArbitrator(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+
+  // if not configured, return right away
+  if (tsArbitrator[0] == 0) {
+    if (pPeer) syncRemovePeer(pPeer);
+    pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    return;
+  }
+
+  SNodeInfo nodeInfo;
+  nodeInfo.nodeId = 0;
+  taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
+  nodeInfo.nodePort += TSDB_PORT_SYNC;
+  
+  if (pPeer) {
+    if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) {
+      return;
+    } else {
+      syncRemovePeer(pPeer);
+      pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    }
+  }   
+
+  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
+}
+
+static void syncAddNodeRef(SSyncNode *pNode)
+{
+   atomic_add_fetch_8(&pNode->refCount, 1);
+}
+
+static void syncDecNodeRef(SSyncNode *pNode)
+{
+  if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
+    pthread_mutex_destroy(&pNode->mutex);
+    tfree(pNode->pRecv);
+    tfree(pNode->pSyncFwds);
+    tfree(pNode);
+
+    if (atomic_sub_fetch_32(&tsNodeNum, 1) == 0) { 
+      if (tsTcpPool) taosCloseTcpThreadPool(tsTcpPool);
+      if (syncTmrCtrl) taosTmrCleanUp(syncTmrCtrl);
+      if (vgIdHash) taosHashCleanup(vgIdHash);
+      syncTmrCtrl = NULL;
+      tsTcpPool = NULL;
+      vgIdHash = NULL;
+      syncModuleInit = PTHREAD_ONCE_INIT;
+      sDebug("sync module is cleaned up");
+    }
+  }
+}
+
+void syncAddPeerRef(SSyncPeer *pPeer)
+{
+   atomic_add_fetch_8(&pPeer->refCount, 1);
+}
+
+int syncDecPeerRef(SSyncPeer *pPeer)
+{
+  if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
+    syncDecNodeRef(pPeer->pSyncNode);
+
+    sDebug("%s, resource is freed", pPeer->id);
+    tfree(pPeer->watchFd);
+    tfree(pPeer);
+    return 0;
+  }
+
+  return 1;
+}
+
+static void syncClosePeerConn(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+  tclose(pPeer->syncFd);
+  if (pPeer->peerFd >=0) {
+    pPeer->peerFd = -1;
+    taosFreeTcpConn(pPeer->pConn);
+  }
+}
+
+static void syncRemovePeer(SSyncPeer *pPeer)
+{
+  sInfo("%s, it is removed", pPeer->id);
+
+  pPeer->ip = 0;
+  syncClosePeerConn(pPeer);
+  syncDecPeerRef(pPeer);
+}
+
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) 
+{
+  uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn);
+  if (ip == -1) return NULL;
+ 
+  SSyncPeer *pPeer = (SSyncPeer *) calloc(1, sizeof(SSyncPeer));
+  if (pPeer == NULL) return NULL;
+
+  pPeer->nodeId = pInfo->nodeId;
+  tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn));
+  pPeer->ip = ip;
+  pPeer->port = pInfo->nodePort;
+  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%d", pNode->vgId, pPeer->fqdn, pPeer->port);
+
+  pPeer->peerFd = -1;
+  pPeer->syncFd = -1;
+  pPeer->role = TAOS_SYNC_ROLE_OFFLINE;
+  pPeer->pSyncNode = pNode;
+  pPeer->refCount = 1;
+
+  sInfo("%s, it is configured", pPeer->id);
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
+    sDebug("%s, start to check peer connection", pPeer->id);
+    taosTmrReset(syncCheckPeerConnection, 100, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+  
+  syncAddNodeRef(pNode);
+  return pPeer;
+}
+
+void syncBroadcastStatus(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if ( i == pNode->selfIndex ) continue;
+    pPeer = pNode->peerInfo[i];
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+  }
+} 
+
+static void syncResetFlowCtrl(SSyncNode *pNode) {
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pNode->peerInfo[i]->numOfRetrieves = 0;
+  }
+
+  if (pNode->notifyFlowCtrl)
+    (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);   
+}
+
+static void syncChooseMaster(SSyncNode *pNode) {
+  SSyncPeer *pPeer;
+  int        onlineNum = 0;
+  int        index = -1;
+  int        replica = pNode->replica;
+
+  sDebug("vgId:%d, choose master", pNode->vgId);
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE)
+      onlineNum++;
+  }
+
+  if (onlineNum == pNode->replica) {
+    // if all peers are online, peer with highest version shall be master
+    index = 0;
+    for (int i = 1; i < pNode->replica; ++i) {
+      if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version)
+        index = i;
+    }
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (index < 0 && onlineNum > replica/2.0) {
+    // over half of nodes are online
+    for (int i = 0; i < pNode->replica; ++i) {
+      //slave with highest version shall be master
+      pPeer = pNode->peerInfo[i];
+      if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
+        if (index < 0 || pPeer->version > pNode->peerInfo[index]->version)
+          index = i;
+      }
+    }
+  }
+
+  if (index >= 0) {
+    if (index == pNode->selfIndex) {
+      sInfo("vgId:%d, start to work as master", pNode->vgId);
+      nodeRole = TAOS_SYNC_ROLE_MASTER;
+      syncResetFlowCtrl(pNode);
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    } else {
+      pPeer = pNode->peerInfo[index];
+      sInfo("%s, it shall work as master", pPeer->id);
+    }
+  } else {
+    sDebug("vgId:%d, failed to choose master", pNode->vgId);
+  }
+} 
+ 
+static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) {
+  int onlineNum = 0;
+  int index = -1;
+  int replica = pNode->replica;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) 
+      onlineNum++;
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (onlineNum <= replica*0.5) {
+    if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+    }
+  } else {
+    for (int i=0; i<pNode->replica; ++i) {
+      SSyncPeer *pTemp = pNode->peerInfo[i];
+      if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue;
+      if ( index < 0 ) {
+        index = i;
+      } else { // multiple masters, it shall not happen 
+        if ( i == pNode->selfIndex ) {
+          sError("%s, peer is master, work as slave instead", pTemp->id);
+          nodeRole = TAOS_SYNC_ROLE_SLAVE;
+          (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+        }
+      }
+    }
+  }
+
+  SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL;
+  return pMaster;
+}
+
+static int syncValidateMaster(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int code = 0;
+
+  if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
+    sDebug("%s, slave has higher version, restart all connections!!!",  pPeer->id);
+    nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    code = -1;
+
+    for (int i = 0; i < pNode->replica; ++i) {
+      if ( i == pNode->selfIndex ) continue;
+      syncRestartPeer(pNode->peerInfo[i]);
+    }
+  } 
+
+  return code;
+}
+
+static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int8_t     peerOldRole = pPeer->role;
+  int8_t     selfOldRole = nodeRole;
+  int8_t     i, syncRequired = 0;
+
+  pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
+  pPeer->role = newRole;
+
+  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, 
+          syncRole[nodeRole], syncRole[pPeer->role]);  
+
+  SSyncPeer *pMaster = syncCheckMaster(pNode);
+
+  if ( pMaster ) {
+    // master is there
+    pNode->pMaster = pMaster;
+    sDebug("%s, it is the master, ver:%" PRIu64, pMaster->id, pMaster->version);
+
+    if (syncValidateMaster(pPeer) < 0) return;
+
+    if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) {
+      if ( nodeVersion < pMaster->version) {
+        syncRequired = 1;
+      } else {
+        sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version);
+        nodeRole = TAOS_SYNC_ROLE_SLAVE;
+        (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      }
+    } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
+      // nodeVersion = pMaster->version;
+    }
+  } else {
+    // master not there, if all peer's state and version are consistent, choose the master
+    int consistent = 0;
+    if (peersStatus) {
+      for (i = 0; i < pNode->replica; ++i) {
+        SSyncPeer *pTemp = pNode->peerInfo[i];
+        if (pTemp->role != peersStatus[i].role) break;
+        if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[i].version)) break; 
+      }
+ 
+      if (i >= pNode->replica) consistent = 1;
+    } else {
+      if (pNode->replica == 2) consistent = 1;
+    }
+
+    if (consistent)
+      syncChooseMaster(pNode);
+  }
+
+  if (syncRequired) {
+    syncRecoverFromMaster(pMaster);
+  }
+
+  if (peerOldRole != newRole || nodeRole != selfOldRole)
+    syncBroadcastStatus(pNode);
+
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) 
+    syncResetFlowCtrl(pNode);
+}
+
+static void syncRestartPeer(SSyncPeer *pPeer) {
+  sDebug("%s, restart connection", pPeer->id);
+
+  syncClosePeerConn(pPeer);
+
+  pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
+
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) )
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+}
+
+void syncRestartConnection(SSyncPeer *pPeer)
+{
+  if (pPeer->ip == 0) return;
+
+  syncRestartPeer(pPeer);
+  syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE);
+}
+
+static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  sDebug("%s, sync-req is received", pPeer->id);
+
+  if (pPeer->ip == 0) return;
+
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) {
+    sError("%s, I am not master anymore", pPeer->id);
+    tclose(pPeer->syncFd);
+    return;
+  }
+
+  if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started", pPeer->id);
+    return; // already started
+  }
+
+  // start a new thread to retrieve the data
+  syncAddPeerRef(pPeer);
+  pthread_attr_t  thattr;
+  pthread_t       thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+  int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    sError("%s, failed to create sync thread(%s)", pPeer->id, strerror(errno));
+    syncDecPeerRef(pPeer);
+  } else {
+    pPeer->sstatus = TAOS_SYNC_STATUS_START;
+    sDebug("%s, thread is created to retrieve data", pPeer->id);
+  }
+}
+
+static void syncNotStarted(void *param, void *tmrId)
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  pPeer->timer = NULL;
+  sInfo("%s, sync connection is still not up, restart", pPeer->id);
+  syncRestartConnection(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncTryRecoverFromMaster(void *param, void *tmrId) {
+  SSyncPeer   *pPeer = param;
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  syncRecoverFromMaster(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncRecoverFromMaster(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus);
+    return;
+  } 
+
+  taosTmrStopA(&pPeer->timer);
+  if (tsSyncNum >= tsMaxSyncNum) {
+    sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
+    taosTmrReset(syncTryRecoverFromMaster, 500, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  sDebug("%s, try to sync", pPeer->id)
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+  taosTmrReset(syncNotStarted, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+
+  if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt) ) {
+    sError("%s, failed to send sync-req to peer", pPeer->id);
+  } else {
+    nodeSStatus = TAOS_SYNC_STATUS_START;
+    sInfo("%s, sync-req is sent", pPeer->id);
+  }
+
+  return;
+}
+
+static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFwdRsp    *pFwdRsp = (SFwdRsp *) cont;
+  SSyncFwds  *pSyncFwds = pNode->pSyncFwds;
+  SFwdInfo   *pFwdInfo;
+
+  sDebug("%s, forward-rsp is received, ver:%" PRIu64, pPeer->id, pFwdRsp->version);
+  SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
+
+  if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
+    // find the forwardInfo from first
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      pFwdInfo = pSyncFwds->fwdInfo + (i+pSyncFwds->first)%tsMaxFwdInfo;
+      if (pFwdRsp->version == pFwdInfo->version) break;
+    }
+ 
+    syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code);
+    syncRemoveConfirmedFwdInfo(pNode);
+  }
+}
+
+
+static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SWalHead    *pHead = (SWalHead *)cont;
+
+  sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
+
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
+    //nodeVersion = pHead->version;
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  } else { 
+    if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+      syncSaveIntoBuffer(pPeer, pHead);
+    } else {
+      sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    }
+  }
+
+  return;
+}
+
+static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode    *pNode = pPeer->pSyncNode;
+  SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
+
+  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+         syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
+
+  pPeer->version = pPeersStatus->version;
+  syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role);
+
+  if (pPeersStatus->ack)
+    syncSendPeersStatusMsgToPeer(pPeer, 0);
+}
+
+static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
+  if (pPeer->peerFd <0) return -1;
+
+  int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
+  if (hlen != sizeof(SSyncHead)) {
+    sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
+    return -1;
+  }
+
+  // head.len = htonl(head.len);
+  if (pHead->len <0) {
+    sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
+    return -1;
+  } 
+
+  int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
+  if (bytes != pHead->len) {
+    sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int syncProcessPeerMsg(void *param, void *buffer)
+{
+  SSyncPeer  *pPeer = param;
+  SSyncHead   head;
+  char       *cont = (char *)buffer;
+
+  SSyncNode *pNode = pPeer->pSyncNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  int code = syncReadPeerMsg(pPeer, &head, cont);
+
+  if (code == 0) {
+    if (head.type == TAOS_SMSG_FORWARD) {
+      syncProcessForwardFromPeer(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_FORWARD_RSP) {
+      syncProcessFwdResponse(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_SYNC_REQ) {
+      syncProcessSyncRequest(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_STATUS) {
+      syncProcessPeersStatusMsg(cont, pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+#define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA
+
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  char       msg[statusMsgLen] = {0};
+
+  if (pPeer->peerFd <0 || pPeer->ip ==0) return;
+
+  SSyncHead    *pHead = (SSyncHead *) msg;
+  SPeersStatus *pPeersStatus = (SPeersStatus *) (msg + sizeof(SSyncHead));
+
+  pHead->type = TAOS_SMSG_STATUS;
+  pHead->len = statusMsgLen - sizeof(SSyncHead);
+
+  pPeersStatus->version = nodeVersion;
+  pPeersStatus->role = nodeRole;
+  pPeersStatus->ack = ack;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
+    pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
+  }
+
+  int retLen = write(pPeer->peerFd, msg, statusMsgLen);
+  if (retLen == statusMsgLen) {
+    sDebug("%s, status msg is sent", pPeer->id);
+  } else {
+    sDebug("%s, failed to send status msg, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+
+  return;
+}
+
+static void syncSetupPeerConnection(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  taosTmrStopA(&pPeer->timer);
+  if (pPeer->peerFd >= 0) {
+    sDebug("%s, send role version to peer", pPeer->id);
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+    return;
+  }
+
+  int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (connFd < 0) {
+    sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.vgId = pPeer->nodeId ? pNode->vgId:0;
+  firstPkt.syncHead.type = TAOS_SMSG_STATUS;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); 
+  firstPkt.port = tsSyncPort;
+  firstPkt.sourceId = pNode->vgId;  // tell arbitrator its vgId
+
+  if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
+    sDebug("%s, connection to peer server is setup", pPeer->id);
+    pPeer->peerFd = connFd; 
+    pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
+    pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+    syncAddPeerRef(pPeer);
+  } else {
+    sDebug("try later");
+    close(connFd);
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+}
+
+static void syncCheckPeerConnection(void *param, void *tmrId) 
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, check peer connection", pPeer->id);
+  syncSetupPeerConnection(pPeer); 
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncCreateRestoreDataThread(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+
+  pthread_attr_t thattr;
+  pthread_t thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+
+  syncAddPeerRef(pPeer);
+  int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret < 0) {
+    sError("%s, failed to create sync thread", pPeer->id);
+    tclose(pPeer->syncFd);
+    syncDecPeerRef(pPeer);
+  } else { 
+    sInfo("%s, sync connection is up", pPeer->id);
+  }
+}
+
+static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) 
+{
+  char  ipstr[24];
+  int   i;
+   
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  int32_t vgId = firstPkt.syncHead.vgId;
+  SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
+  if (ppNode == NULL || *ppNode == NULL) {
+    sError("vgId:%d, vgId could not be found", vgId);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SSyncNode *pNode = *ppNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  SSyncPeer *pPeer;
+  for (i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port))
+      break;
+  }
+
+  pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL;
+  if (pPeer == NULL) {
+    sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn);
+    taosCloseSocket(connFd);
+    // syncSendVpeerCfgMsg(sync);
+  } else {
+    // first packet tells what kind of link
+    if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) {
+      pPeer->syncFd = connFd;
+      syncCreateRestoreDataThread(pPeer);
+    } else {
+      sDebug("%s, TCP connection is already up, close one", pPeer->id);
+      syncClosePeerConn(pPeer);
+      pPeer->peerFd = connFd;
+      pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+      syncAddPeerRef(pPeer);
+      sDebug("%s, ready to exchange data", pPeer->id);
+      syncSendPeersStatusMsgToPeer(pPeer, 1);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return;
+}
+
+static void syncProcessBrokenLink(void *param) {
+  if (param == NULL) return;  // the connection for arbitrator
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  syncAddNodeRef(pNode);
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
+  pPeer->peerFd = -1;
+
+  if (syncDecPeerRef(pPeer) != 0) {
+    syncRestartConnection(pPeer);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+  syncDecNodeRef(pNode);
+}
+
+static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) 
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds >= tsMaxFwdInfo) {
+    pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+  } 
+
+  if (pSyncFwds->fwds > 0) 
+    pSyncFwds->last = (pSyncFwds->last+1) % tsMaxFwdInfo;
+  SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
+  pFwdInfo->version = version;
+  pFwdInfo->mhandle = mhandle;
+  pFwdInfo->acks = 0;
+  pFwdInfo->confirmed = 0;
+  pFwdInfo->time = time;
+
+  pSyncFwds->fwds++;
+  sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds);
+}
+
+static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode)
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+
+  int fwds = pSyncFwds->fwds;
+  for (int i=0; i<fwds; ++i) {
+    SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; 
+    if (pFwdInfo->confirmed == 0) break;
+
+    pSyncFwds->first = (pSyncFwds->first+1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+    if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last;
+    //sDebug("vgId:%d, fwd info is removed, ver:%d, fwds:%d", 
+    //        pNode->vgId, pFwdInfo->version, pSyncFwds->fwds);
+    memset(pFwdInfo, 0, sizeof(SFwdInfo));
+  }
+}
+
+static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) 
+{
+  int confirm = 0;
+  if (pFwdInfo->code == 0) pFwdInfo->code = code;
+
+  if (code == 0) {
+    pFwdInfo->acks++;
+    if (pFwdInfo->acks >= pNode->quorum-1) 
+      confirm = 1;
+  } else {
+    pFwdInfo->nacks++;
+    if (pFwdInfo->nacks > pNode->replica-pNode->quorum) 
+      confirm = 1;
+  }
+
+  if (confirm && pFwdInfo->confirmed == 0) {
+    sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);
+    (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code);
+    pFwdInfo->confirmed = 1;
+  }
+}
+
+static void syncMonitorFwdInfos(void *param, void *tmrId)
+{
+  SSyncNode *pNode = param;
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds > 0) {
+    pthread_mutex_lock(&(pNode->mutex));
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first+i) % tsMaxFwdInfo; 
+      if (time - pFwdInfo->time < 2000) break;
+      syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+    }
+
+    syncRemoveConfirmedFwdInfo(pNode);
+    pthread_mutex_unlock(&(pNode->mutex));
+  } 
+  
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+}
+ 
+
+
diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c
new file mode 100644
index 0000000000..07d99b916e
--- /dev/null
+++ b/src/sync/src/syncRestore.c
@@ -0,0 +1,329 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eindex) {
+  char       name[TSDB_FILENAME_LEN*2] = {0};
+  char       fname[TSDB_FILENAME_LEN*3] = {0};
+  uint32_t   magic; 
+  uint64_t   fversion;
+  int32_t    size;
+  uint32_t   index = sindex;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  if (sindex < 0 || eindex < sindex) return;
+
+  while (1) {
+    name[0] = 0;
+    magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion);
+    if (magic == 0) break;
+
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
+    remove(fname);
+    sDebug("%s, %s is removed", pPeer->id, fname);
+
+    index++;
+    if (index > eindex) break;
+  }        
+}
+
+static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) 
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SFileInfo  minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */  // master file info
+  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */  // slave file info
+  SFileAck   fileAck; 
+  int        code = -1;
+  char       name[TSDB_FILENAME_LEN * 2] = {0};
+  uint32_t   pindex = 0;    // index in last restore
+
+  *fversion = 0;
+  sinfo.index = 0;
+  while (1) {
+    // read file info
+    int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
+    if (ret < 0 ) break;
+
+    // if no more file from master, break;
+    if (minfo.name[0] == 0 || minfo.magic == 0) {
+      sDebug("%s, no more files to restore", pPeer->id);
+
+      // remove extra files after the current index
+      syncRemoveExtraFile(pPeer, sinfo.index+1, TAOS_SYNC_MAX_INDEX);
+      code = 0; 
+      break;
+    }
+   
+    // remove extra files on slave between the current and last index
+    syncRemoveExtraFile(pPeer, pindex+1, minfo.index-1);
+    pindex = minfo.index;
+
+    // check the file info
+    sinfo = minfo;
+    sDebug("%s, get file info:%s", pPeer->id, minfo.name);
+    sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
+
+    // if file not there or magic is not the same, file shall be synced
+    memset(&fileAck, 0, sizeof(fileAck));
+    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1:0;
+
+    // send file ack
+    ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+ 
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      sDebug("%s, %s is the same", pPeer->id, minfo.name);
+      continue;
+    }
+
+    // if sync is required, open file, receive from master, and write to file
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
+
+    int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+    if ( dfd < 0 ) {
+      sError("%s, failed to open file:%s", pPeer->id, name);
+      break;
+    }
+
+    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
+    fsync(dfd);
+    close(dfd);
+    if (ret<0) break;
+
+    sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size);
+
+  }
+
+  if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+   // data file is changed, code shall be set to 1 
+    *fversion = minfo.fversion;
+    code = 1;  
+  }
+
+  if (code < 0) {
+    sError("%s, failed to restore %s(%s)", pPeer->id, name, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRestoreWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         ret, code = -1;
+
+  void *buffer = calloc(1024000, 1);  // size for one record
+  if (buffer == NULL) return -1;
+
+  SWalHead *pHead = (SWalHead *)buffer;
+
+  while (1) {
+    ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
+    if (ret <0)  break;
+
+    if (pHead->len == 0) {code = 0; break;}  // wal sync over
+    
+    ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
+    if (ret <0)  break;
+
+    sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
+  }
+
+  if (code<0) {
+    sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  free(buffer);
+  return code;
+}
+
+static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SWalHead  *pHead = (SWalHead *) offset;
+
+  (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  offset += pHead->len + sizeof(SWalHead);
+
+  return offset;
+}
+
+static int syncProcessBufferedFwd(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+  int          forwards = 0;
+
+  sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
+
+  char *offset = pRecv->buffer;
+  while (forwards < pRecv->forwards) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+  
+  pthread_mutex_lock(&pNode->mutex);
+
+  while (forwards < pRecv->forwards && pRecv->code == 0) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+
+  nodeRole = TAOS_SYNC_ROLE_SLAVE;
+  sDebug("%s, finish processing buffered fwds:%d", pPeer->id, forwards);
+
+  pthread_mutex_unlock(&pNode->mutex);
+
+  return pRecv->code;
+}
+
+int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead)
+{ 
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+
+  if (pRecv == NULL) return -1;
+  int len = pHead->len + sizeof(SWalHead);
+
+  if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
+    memcpy(pRecv->offset, pHead, len);
+    pRecv->offset += len;
+    pRecv->forwards++;
+    sDebug("%s, fwd is saved into queue, ver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
+  } else {
+    sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize);
+    pRecv->code = -1;  // set error code
+  }
+
+  return pRecv->code;
+}
+
+static void syncCloseRecvBuffer(SSyncNode *pNode)
+{
+  if (pNode->pRecv) {
+    tfree(pNode->pRecv->buffer);
+  }
+
+  tfree(pNode->pRecv);
+}
+
+static int syncOpenRecvBuffer(SSyncNode *pNode) 
+{
+  syncCloseRecvBuffer(pNode);
+
+  SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
+  if (pRecv == NULL) return -1;
+
+  pRecv->bufferSize = 5000000;
+  pRecv->buffer = malloc(pRecv->bufferSize);
+  if (pRecv->buffer == NULL) {
+    free(pRecv);
+    return -1;
+  }
+
+  pRecv->offset = pRecv->buffer;
+  pRecv->forwards = 0;
+
+  pNode->pRecv = pRecv;
+
+  return 0;
+}
+
+static int syncRestoreDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  nodeSStatus = TAOS_SYNC_STATUS_FILE;
+  uint64_t fversion = 0;
+
+  sDebug("%s, start to restore file", pPeer->id);
+  int code = syncRestoreFile(pPeer, &fversion);
+  if (code < 0) {
+    sError("%s, failed to restore file", pPeer->id);
+    return -1;
+  }
+
+  // if code > 0, data file is changed, notify app, and pass the version 
+  if (code > 0 && pNode->notifyFileSynced) {
+    if ( (*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0 ) {
+      sError("%s, app not in ready state", pPeer->id);
+      return -1;
+    }
+  }
+
+  nodeVersion = fversion;
+
+  sDebug("%s, start to restore wal", pPeer->id);
+  if (syncRestoreWal(pPeer) < 0) {
+    sError("%s, failed to restore wal", pPeer->id);
+    return -1;
+  }
+
+  nodeSStatus = TAOS_SYNC_STATUS_CACHE;
+  sDebug("%s, start to insert buffered points", pPeer->id);
+  if (syncProcessBufferedFwd(pPeer) < 0) {
+    sError("%s, failed to insert buffered points", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRestoreData(void *param)
+{
+  SSyncPeer  *pPeer = (SSyncPeer *)param;
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  taosBlockSIGPIPE();
+  __sync_fetch_and_add(&tsSyncNum, 1);
+
+  (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING);
+
+  if (syncOpenRecvBuffer(pNode) < 0) {
+    sError("%s, failed to allocate recv buffer", pPeer->id);
+  } else { 
+    if ( syncRestoreDataStepByStep(pPeer) == 0) {
+      sInfo("%s, it is synced successfully", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_SLAVE;
+      syncBroadcastStatus(pNode);
+    } else {
+      sError("%s, failed to restore data, restart connection", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+
+  nodeSStatus = TAOS_SYNC_STATUS_INIT;
+  tclose(pPeer->syncFd)
+  syncCloseRecvBuffer(pNode);
+  __sync_fetch_and_sub(&tsSyncNum, 1);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}
+
diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c
new file mode 100644
index 0000000000..8aa7d101e7
--- /dev/null
+++ b/src/sync/src/syncRetrieve.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) 
+{
+  sDebug("%s, start to monitor:%s", pPeer->id, name);
+
+  if (pPeer->notifyFd <=0) {
+    pPeer->watchNum = 0;
+    pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+    if (pPeer->notifyFd < 0) {
+      sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+      return -1;
+    }
+
+    if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+    if (pPeer->watchFd == NULL) {
+      sError("%s, failed to allocate watchFd", pPeer->id);
+      return -1;
+    }
+
+    memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  }
+
+  int *wd = pPeer->watchFd + pPeer->watchNum;
+
+  if (*wd >= 0) {
+    if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
+      sError("%s, failed to remove wd:%d(%s)", pPeer->id, *wd, strerror(errno));
+      return -1;
+    }
+  }
+
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY);
+  if (*wd == -1) {
+    sError("%s, failed to add %s(%s)", pPeer->id, name, strerror(errno));
+    return -1;
+  }
+
+  pPeer->watchNum++;
+  pPeer->watchNum = (pPeer->watchNum +1) % tsMaxWatchFiles;
+
+  return 0;
+}
+
+static int syncAreFilesModified(SSyncPeer *pPeer) 
+{
+  if (pPeer->notifyFd <=0) return 0;
+
+  char buf[2048]; 
+  int len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  int code = 0; 
+  if (len >0) { 
+    sDebug("%s, processed file is changed", pPeer->id);    
+    pPeer->fileChanged = 1;
+    code = 1;
+  }
+
+  return code;  
+}
+
+static int syncRetrieveFile(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFileInfo   fileInfo;
+  SFileAck    fileAck;
+  int         code = -1;
+  char        name[TSDB_FILENAME_LEN * 2] = {0};
+
+  memset(&fileInfo, 0, sizeof(fileInfo));
+  memset(&fileAck, 0, sizeof(fileAck));
+
+  while (1) {
+    // retrieve file info
+    fileInfo.name[0] = 0;
+    fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion);   
+    //fileInfo.size = htonl(size);
+
+    // send the file info
+    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
+    if (ret < 0 ) break;
+
+    // if no file anymore, break
+    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { 
+      sDebug("%s, no more files to sync", pPeer->id);    
+      code = 0; break; 
+    }
+
+    // wait for the ack from peer
+    ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+
+    // set the peer sync version
+    pPeer->sversion = fileInfo.fversion;
+
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+    
+    // add the file into watch list
+    if ( syncAddIntoWatchList(pPeer, name) <0) break;
+
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      fileInfo.index++; 
+      sDebug("%s, %s is the same", pPeer->id, fileInfo.name);    
+      continue; 
+    }
+
+    // send the file to peer
+    int sfd = open(name, O_RDONLY);
+    if ( sfd < 0 ) break;
+
+    ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); 
+    close(sfd); 
+    if (ret <0) break;
+
+    sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size);    
+    fileInfo.index++; 
+
+    // check if processed files are modified 
+    if (syncAreFilesModified(pPeer) != 0) break;
+  }
+
+  if (code < 0) {
+    sError("%s, failed to retrieve file(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+/* if only a partial record is read out, set the IN_MODIFY flag in event,
+   so upper layer will reload the file to get a complete record */
+static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) 
+{ 
+  int ret;
+
+  ret = read(sfd, pHead, sizeof(SWalHead));
+  if (ret < 0) return -1;
+  if (ret == 0) return 0;
+
+  if (ret != sizeof(SWalHead)) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  ret = read(sfd, pHead->cont, pHead->len);
+  if (ret <0) return -1;
+
+  if (ret != pHead->len) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  return sizeof(SWalHead) + pHead->len;
+}    
+
+static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) 
+{ 
+  pPeer->watchNum = 0;
+  tclose(pPeer->notifyFd);
+  pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+  if (pPeer->notifyFd < 0) {
+    sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+  if (pPeer->watchFd == NULL) {
+    sError("%s, failed to allocate watchFd", pPeer->id);
+    return -1;
+  }
+
+  memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  int *wd = pPeer->watchFd;
+ 
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
+  if (*wd == -1) {
+    sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  return 0; 
+}
+
+static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) 
+{
+  char       buf[2048]; 
+  int  len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  if (len == 0) return 0;
+
+  struct inotify_event *event;
+  for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
+    event = (struct inotify_event *) ptr;
+    if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
+    if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
+  }
+
+  if (pEvent != 0)
+    sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
+
+  return 0;
+}
+
+static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) 
+{
+  SWalHead  *pHead = (SWalHead *) malloc(640000);
+  int        code = -1;
+  int32_t    bytes = 0;
+  int        sfd;
+
+  sfd = open(name, O_RDONLY);
+  if (sfd < 0) return -1;
+  lseek(sfd, offset, SEEK_SET);
+  sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
+
+  while (1) {
+    int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); 
+    if (wsize <0) break;
+    if (wsize == 0) { code = 0; break; }
+
+    sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+    if ( ret != wsize ) break;
+    pPeer->sversion = pHead->version;
+
+    bytes += wsize;
+ 
+    if (pHead->version >= fversion && fversion > 0) {
+      code = 0; 
+      bytes = 0; 
+      break;
+    }
+  }
+
+  free(pHead);
+  tclose(sfd); 
+
+  if (code == 0) return bytes;
+  return -1;
+}
+
+static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         code = -1;
+  char        fname[TSDB_FILENAME_LEN * 2];  // full path to wal file
+
+  if (syncAreFilesModified(pPeer) != 0) return -1;
+
+  while (1) {
+    int32_t  once = 0; // last WAL has once ever been processed 
+    int64_t  offset = 0;
+    uint64_t fversion = 0;
+    uint32_t event = 0;
+
+    // get full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+    sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
+
+    // monitor last wal
+    if (syncMonitorLastWal(pPeer, fname) <0) break;
+
+    while (1) {
+      int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
+      if (bytes < 0) break;
+
+      // check file changes
+      if (syncCheckLastWalChanges(pPeer, &event) <0) break;
+
+      // if file is not updated or updated once, set the fversion and sstatus
+      if (((event & IN_MODIFY) == 0) || once) {
+        if (fversion == 0) {
+          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
+          fversion = nodeVersion;    // must read data to fversion
+        }
+      }
+
+      // if all data up to fversion is read out, it is over
+      if (pPeer->sversion >= fversion && fversion > 0) {
+        code = 0; 
+        sDebug("%s, data up to fversion:%ld has been read out, bytes:%d", pPeer->id, fversion, bytes);
+        break;
+      }  
+
+      // if all data are read out, and no update
+      if ((bytes == 0) && ((event & IN_MODIFY) == 0)) {
+        // wal file is closed, break
+        if (event & IN_CLOSE_WRITE) { 
+          code = 0; 
+          sDebug("%s, current wal is closed", pPeer->id);
+          break;
+        }
+ 
+        // wal not closed, it means some data not flushed to disk, wait for a while
+        usleep(10000);
+      }
+
+      // if bytes>0, file is updated, or fversion is not reached but file still open, read again 
+      once = 1;
+      offset += bytes; 
+      sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes);
+      event = event & (~IN_MODIFY); // clear IN_MODIFY flag 
+    }
+
+    if (code < 0) break;
+    if (pPeer->sversion >= fversion && fversion > 0) break;  
+
+    index++;  wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);
+    if ( code < 0) break;  
+    if ( wname[0] == 0 ) {code = 0; break;}
+
+    // current last wal is closed, there is a new one 
+    sDebug("%s, last wal is closed, try new one", pPeer->id);
+  }
+
+  tclose(pPeer->notifyFd);
+
+  return code;
+}
+
+static int syncRetrieveWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  char        fname[TSDB_FILENAME_LEN * 3];
+  char        wname[TSDB_FILENAME_LEN * 2];
+  int32_t     size;
+  struct stat fstat;
+  int         code = -1;
+  uint32_t    index = 0;
+
+  while (1) {
+    // retrieve wal info
+    wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);   
+    if (code < 0) break;  // error
+    if (wname[0] == 0) {  // no wal file
+      sDebug("%s, no wal file", pPeer->id);
+      break;
+    }    
+      
+    if (code == 0) {   // last wal 
+      code = syncProcessLastWal(pPeer, wname, index);
+      break;
+    }
+
+    // get the full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+
+    // send wal file, 
+    // inotify is not required, old wal file won't be modified, even remove is ok
+    if ( stat(fname, &fstat) < 0 ) break;
+    size = fstat.st_size;
+
+    sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);    
+    int sfd = open(fname, O_RDONLY);
+    if (sfd < 0) break;
+
+    code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); 
+    close(sfd); 
+    if (code <0) break;
+
+    index++; 
+
+    if (syncAreFilesModified(pPeer) != 0) break; 
+  }
+
+  if (code == 0) {
+    sDebug("%s, wal retrieve is finished", pPeer->id);    
+    pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+    SWalHead walHead;
+    memset(&walHead, 0, sizeof(walHead));
+    code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
+  } else {
+    sError("%s, failed to send wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRetrieveDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+
+  if (write(pPeer->syncFd, (char *) &firstPkt, sizeof(firstPkt)) < 0) {
+    sError("%s, failed to send syncCmd", pPeer->id);
+    return -1;
+  }
+
+  pPeer->sversion = 0;  
+  pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
+  sDebug("%s, start to retrieve file", pPeer->id);
+  if (syncRetrieveFile(pPeer) < 0) {
+    sError("%s, failed to retrieve file", pPeer->id);
+    return -1;
+  }
+
+  // if no files are synced, there must be wal to sync, sversion must be larger than one
+  if (pPeer->sversion == 0) 
+    pPeer->sversion = 1;
+
+  sDebug("%s, start to retrieve wal", pPeer->id);
+  if (syncRetrieveWal(pPeer) < 0) {
+    sError("%s, failed to retrieve wal", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRetrieveData(void *param)
+{
+  SSyncPeer  *pPeer = (SSyncPeer *)param;
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  taosBlockSIGPIPE();
+
+  pPeer->fileChanged = 0;
+  pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (pPeer->syncFd < 0) {
+    sError("%s, failed to open socket to sync", pPeer->id);
+  } else {
+    sInfo("%s, sync tcp is setup", pPeer->id);
+  
+    if (syncRetrieveDataStepByStep(pPeer) == 0) {
+      sDebug("%s, sync retrieve process is successful", pPeer->id);
+    } else {
+      sError("%s, failed to retrieve data, restart connection", pPeer->id);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  if (pPeer->fileChanged) {
+    // if file is changed 3 times continuously, start flow control
+    pPeer->numOfRetrieves++;
+    if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl)  
+      (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2));
+  } else {
+    pPeer->numOfRetrieves = 0;
+    if (pNode->notifyFlowCtrl)
+      (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
+  }
+
+  pPeer->fileChanged = 0;
+  tclose(pPeer->notifyFd);
+  tclose(pPeer->syncFd);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}
diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c
new file mode 100644
index 0000000000..fa94caeed7
--- /dev/null
+++ b/src/sync/src/taosTcpPool.c
@@ -0,0 +1,325 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tsocket.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+
+typedef struct SThreadObj {
+  pthread_t thread;
+  bool      stop;
+  int       pollFd;
+  int       numOfFds;
+  struct SPoolObj *pPool;
+} SThreadObj;
+
+typedef struct SPoolObj {
+  SPoolInfo    info;
+  SThreadObj **pThread;
+  pthread_t    thread;
+  int          nextId;
+  int          acceptFd;  // FD for accept new connection
+} SPoolObj;
+
+typedef struct {
+  SThreadObj *pThread;
+  void       *ahandle;
+  int         fd;
+  int         closedByApp; 
+} SConnObj;
+
+static void *taosAcceptPeerTcpConnection(void *argv);
+static void *taosProcessTcpData(void *param);
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool);
+static void taosStopPoolThread(SThreadObj* pThread);
+
+void *taosOpenTcpThreadPool(SPoolInfo *pInfo)
+{
+  pthread_attr_t thattr;
+
+  SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
+  if (pPool == NULL) {
+    uError("TCP server, no enough memory");
+    return NULL;
+  }
+
+  pPool->info = *pInfo;
+  
+  pPool->pThread = (SThreadObj **) calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
+  if (pPool->pThread == NULL) {
+    uError("TCP server, no enough memory");
+    free(pPool);
+    return NULL;
+  }
+
+  pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
+  if (pPool->acceptFd < 0) {
+    free(pPool->pThread); free(pPool);
+    uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
+    return NULL;
+  }
+
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&(pPool->thread), &thattr, (void *) taosAcceptPeerTcpConnection, pPool) != 0) {
+    uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
+    close(pPool->acceptFd);
+    free(pPool->pThread); free(pPool);
+    return NULL;
+  }
+
+  pthread_attr_destroy(&thattr);
+
+  uDebug("%p TCP pool is created", pPool);
+  return pPool;
+}
+
+void taosCloseTcpThreadPool(void *param)
+{
+  SPoolObj    *pPool = (SPoolObj *)param;
+  SThreadObj  *pThread;
+
+  shutdown(pPool->acceptFd, SHUT_RD);  
+  pthread_join(pPool->thread, NULL);
+
+  for (int i = 0; i < pPool->info.numOfThreads; ++i) {
+    pThread = pPool->pThread[i];
+    if (pThread) taosStopPoolThread(pThread); 
+  }
+
+  tfree(pPool->pThread);
+  free(pPool);
+  uDebug("%p TCP pool is closed", pPool);
+}
+
+void *taosAllocateTcpConn(void *param, void *pPeer, int connFd)
+{
+  struct epoll_event event;
+  SPoolObj *pPool = (SPoolObj *)param;
+
+  SConnObj *pConn = (SConnObj *) calloc(sizeof(SConnObj), 1);
+  if (pConn == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    return NULL;
+  }
+
+  SThreadObj *pThread = taosGetTcpThread(pPool);
+  if (pThread == NULL) {
+    free(pConn);
+    return NULL;
+  }
+
+  pConn->fd = connFd;
+  pConn->pThread = pThread;
+  pConn->ahandle = pPeer;
+  pConn->closedByApp = 0;
+
+  event.events = EPOLLIN | EPOLLRDHUP;
+  event.data.ptr = pConn;
+
+  if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
+    uError("failed to add fd:%d(%s)", connFd, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    free(pConn);
+    pConn = NULL;
+  } else {
+    pThread->numOfFds++;
+    uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
+  }
+
+  return pConn;
+}
+
+void taosFreeTcpConn(void *param)
+{
+  SConnObj *pConn = (SConnObj *)param;
+  SThreadObj   *pThread = pConn->pThread;
+
+  uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
+  pConn->closedByApp = 1;
+  shutdown(pConn->fd, SHUT_WR);
+}
+
+static void taosProcessBrokenLink(SConnObj *pConn) {
+  SThreadObj *pThread = pConn->pThread;
+  SPoolObj   *pPool = pThread->pPool;
+  SPoolInfo  *pInfo = &pPool->info;
+  
+  if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
+  (*pInfo->processBrokenLink)(pConn->ahandle);
+
+  pThread->numOfFds--;
+  epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
+  uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
+  tclose(pConn->fd);
+  free(pConn);
+}
+
+#define maxEvents 10
+
+static void *taosProcessTcpData(void *param) {
+  SThreadObj     *pThread = (SThreadObj *) param;
+  SPoolObj       *pPool = pThread->pPool;
+  SPoolInfo      *pInfo = &pPool->info;
+  SConnObj       *pConn = NULL;
+  struct epoll_event events[maxEvents];
+
+  void *buffer = malloc(pInfo->bufferSize);
+  taosBlockSIGPIPE();
+
+  while (1) {
+    if (pThread->stop) break; 
+    int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, -1);
+    if (pThread->stop) {
+      uDebug("%p TCP epoll thread is exiting...", pThread);
+      break;
+    }
+
+    if (fdNum < 0) { 
+      uError("epoll_wait failed (%s)", strerror(errno));
+      continue;
+    }
+
+    for (int i = 0; i < fdNum; ++i) {
+      pConn = events[i].data.ptr;
+      assert(pConn);
+
+      if (events[i].events & EPOLLERR) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLRDHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (pConn->closedByApp == 0) {
+        if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) {
+          taosFreeTcpConn(pConn);
+          continue;
+        }
+      } 
+    }
+  }
+
+  close(pThread->pollFd);
+  free(pThread);
+  free(buffer);
+  uDebug("%p TCP epoll thread exits", pThread);
+  return NULL;  
+}
+
+static void *taosAcceptPeerTcpConnection(void *argv) {
+  SPoolObj   *pPool = (SPoolObj *)argv;
+  SPoolInfo  *pInfo = &pPool->info;
+
+  taosBlockSIGPIPE();
+
+  while (1) {
+    struct sockaddr_in clientAddr;
+    socklen_t addrlen = sizeof(clientAddr);
+    int connFd = accept(pPool->acceptFd, (struct sockaddr *) &clientAddr, &addrlen);
+    if (connFd < 0) {
+      if (errno == EINVAL) {
+        uDebug("%p TCP server accept is exiting...", pPool);
+        break;
+      } else {
+        uError("TCP accept failure, reason:%s", strerror(errno));
+        continue;
+      }
+    }
+
+    //uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); 
+    taosKeepTcpAlive(connFd);
+    (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
+  }
+
+  tclose(pPool->acceptFd);
+  return NULL;
+}
+
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
+  SThreadObj *pThread = pPool->pThread[pPool->nextId];
+
+  if (pThread) return pThread;
+
+  pThread = (SThreadObj *) calloc(1, sizeof(SThreadObj));
+  if (pThread == NULL) return NULL;
+
+  pThread->pPool = pPool;
+  pThread->pollFd = epoll_create(10);  // size does not matter
+  if (pThread->pollFd < 0) {
+    free(pThread);
+    return NULL;
+  }
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  int ret = pthread_create(&(pThread->thread), &thattr, (void *) taosProcessTcpData, pThread);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    close(pThread->pollFd);
+    free(pThread);
+    return NULL;
+  }
+
+  uDebug("%p TCP epoll thread is created", pThread);
+  pPool->pThread[pPool->nextId] = pThread;
+  pPool->nextId++;
+  pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
+
+  return pThread;
+}
+
+static void taosStopPoolThread(SThreadObj* pThread) {
+  pThread->stop = true;
+  
+  if (pThread->thread == pthread_self()) {
+    pthread_detach(pthread_self());
+    return;
+  }
+
+  // save thread ID into a local variable, since pThread is freed when the thread exits 
+  pthread_t thread = pThread->thread;
+
+  // signal the thread to stop, try graceful method first,
+  // and use pthread_cancel when failed
+  struct epoll_event event = { .events = EPOLLIN };
+  eventfd_t fd = eventfd(1, 0);
+  if (fd == -1) {
+    // failed to create eventfd, call pthread_cancel instead, which may result in data corruption
+    uError("failed to create eventfd(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
+    uError("failed to call epoll_ctl(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  }
+
+  pthread_join(thread, NULL);
+  tclose(fd);
+}
+
diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c
new file mode 100644
index 0000000000..c308c2a454
--- /dev/null
+++ b/src/sync/src/tarbitrator.c
@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
+static void arbProcessBrokenLink(void *param);
+static int  arbProcessPeerMsg(void *param, void *buffer);
+static sem_t    tsArbSem;
+static ttpool_h tsArbTcpPool;
+
+typedef struct {
+  char  id[TSDB_EP_LEN+24];
+  int   nodeFd;
+  void *pConn;
+} SNodeConn;
+
+int main(int argc, char *argv[]) {
+  char     arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      tsServerPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      debugFlag = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-g")==0 && i < argc-1) {
+      if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue; 
+      tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath));
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", tsServerPort);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", debugFlag);
+      printf("  [-g logFilePath]: log file pathe, default:%s\n", arbLogPath);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+  
+ if (sem_init(&tsArbSem, 0, 0) != 0) {
+    printf("failed to create exit semphore\n");
+    exit(EXIT_FAILURE);
+  }
+
+  /* Set termination handler. */
+  struct sigaction act = {{0}};
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = arbSignalHandler;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  tsAsyncLog = 0;
+  strcat(arbLogPath, "/arbitrator.log");
+  taosInitLog(arbLogPath, 1000000, 10);
+
+  taosGetFqdn(tsNodeFqdn);
+  tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
+
+  SPoolInfo info;
+  info.numOfThreads = 1;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = arbProcessBrokenLink;
+  info.processIncomingMsg = arbProcessPeerMsg;
+  info.processIncomingConn = arbProcessIncommingConnection;
+  tsArbTcpPool = taosOpenTcpThreadPool(&info);
+  
+  if (tsArbTcpPool == NULL) {
+   sDebug("failed to open TCP thread pool, exit...");
+   return -1;
+  }
+
+  sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsServerPort);
+
+  for (int res = sem_wait(&tsArbSem); res != 0; res = sem_wait(&tsArbSem)) {
+    if (res != EINTR) break;
+  }
+
+  taosCloseTcpThreadPool(tsArbTcpPool);
+  sInfo("TAOS arbitrator is shut down\n");
+  closelog();
+
+  return 0;
+}
+
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp)
+{
+  char  ipstr[24];
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SNodeConn *pNode = (SNodeConn *) calloc(sizeof(SNodeConn), 1);
+  if (pNode == NULL) {
+    sError("failed to allocate memory(%s)", strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); 
+  if (firstPkt.syncHead.vgId) {  
+    sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
+    tfree(pNode);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  sDebug("%s, arbitrator request is accepted", pNode->id);
+  pNode->nodeFd = connFd;
+  pNode->pConn = taosAllocateTcpConn(tsArbTcpPool, pNode, connFd);
+
+  return;
+}
+
+static void arbProcessBrokenLink(void *param) {
+  SNodeConn *pNode = param;
+
+  sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
+  tfree(pNode);
+}
+
+static int arbProcessPeerMsg(void *param, void *buffer)
+{
+  SNodeConn  *pNode = param;
+  SSyncHead   head;
+  int         bytes = 0;
+  char       *cont = (char *)buffer;
+
+  int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
+  if (hlen != sizeof(head)) {
+    sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
+    return -1;
+  }
+
+  bytes = taosReadMsg(pNode->nodeFd, cont, head.len);
+  if (bytes != head.len) {
+    sDebug("%s, failed to read, bytes:%d len:%d", pNode->id, bytes, head.len);
+    return -1;
+  }
+
+  sDebug("%s, msg is received, len:%d", pNode->id, head.len);
+  return 0;
+}
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) {
+
+  struct sigaction act = {{0}};
+  act.sa_handler = SIG_IGN;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
+
+  // inform main thread to exit
+  sem_post(&tsArbSem);
+}
+
diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt
new file mode 100644
index 0000000000..89d739080f
--- /dev/null
+++ b/src/sync/test/CMakeLists.txt
@@ -0,0 +1,20 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(../inc)
+
+  LIST(APPEND CLIENT_SRC ./syncClient.c)
+  ADD_EXECUTABLE(syncClient ${CLIENT_SRC})
+  TARGET_LINK_LIBRARIES(syncClient sync trpc common)
+
+  LIST(APPEND SERVER_SRC ./syncServer.c)
+  ADD_EXECUTABLE(syncServer ${SERVER_SRC})
+  TARGET_LINK_LIBRARIES(syncServer sync trpc common)
+ENDIF ()
+
+
diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c
new file mode 100644
index 0000000000..cd873b758b
--- /dev/null
+++ b/src/sync/test/syncClient.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "trpc.h"
+#include "taoserror.h"
+
+typedef struct {
+  int       index;
+  SRpcEpSet epSet;
+  int       num;
+  int       numOfReqs;
+  int       msgSize;
+  sem_t     rspSem; 
+  sem_t    *pOverSem; 
+  pthread_t thread;
+  void     *pRpc;
+} SInfo;
+
+void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  SInfo *pInfo = (SInfo *)pMsg->ahandle;
+  uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
+
+  if (pEpSet) pInfo->epSet = *pEpSet;
+  rpcFreeCont(pMsg->pCont);
+
+  sem_post(&pInfo->rspSem); 
+}
+
+int tcount = 0;
+
+void *sendRequest(void *param) {
+  SInfo  *pInfo = (SInfo *)param;
+  SRpcMsg rpcMsg = {0}; 
+  
+  uDebug("thread:%d, start to send request", pInfo->index);
+
+  while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
+    pInfo->num++;
+    rpcMsg.pCont = rpcMallocCont(pInfo->msgSize);
+    rpcMsg.contLen = pInfo->msgSize;
+    rpcMsg.ahandle = pInfo;
+    rpcMsg.msgType = 1;
+    uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
+    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+    if ( pInfo->num % 20000 == 0 ) 
+      uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
+    sem_wait(&pInfo->rspSem);
+  }
+
+  uDebug("thread:%d, it is over", pInfo->index);
+  tcount++;
+
+  return NULL;
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit  rpcInit;
+  SRpcEpSet epSet;
+  char      secret[TSDB_KEY_LEN] = "mypassword";
+  int      msgSize = 128;
+  int      numOfReqs = 0;
+  int      appThreads = 1;
+  char     serverIp[40] = "127.0.0.1";
+  struct   timeval systemTime;
+  int64_t  startTime, endTime;
+  pthread_attr_t thattr;
+
+  // server info
+  epSet.numOfEps = 1;
+  epSet.inUse = 0;
+  epSet.port[0] = 7000;
+  epSet.port[1] = 7000;
+  strcpy(epSet.fqdn[0], serverIp);
+  strcpy(epSet.fqdn[1], "192.168.0.1");
+
+  // client info
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 0;
+  rpcInit.label        = "APP";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processResponse;
+  rpcInit.sessions     = 100;
+  rpcInit.idleTime     = tsShellActivityTimer*1000;
+  rpcInit.user         = "michael";
+  rpcInit.secret       = secret;
+  rpcInit.ckey         = "key";
+  rpcInit.spi          = 1;
+  rpcInit.connType     = TAOS_CONN_CLIENT;
+
+  for (int i=1; i<argc; ++i) { 
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      epSet.port[0] = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
+      strcpy(epSet.fqdn[0], argv[++i]); 
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-n")==0 && i < argc-1) {
+      numOfReqs = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-a")==0 && i < argc-1) {
+      appThreads = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-u")==0 && i < argc-1) {
+      rpcInit.user = argv[++i];
+    } else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
+      rpcInit.secret = argv[++i];
+    } else if (strcmp(argv[i], "-spi")==0 && i < argc-1) {
+      rpcInit.spi = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-i ip]: first server IP address, default is:%s\n", serverIp);
+      printf("  [-p port]: server port number, default is:%d\n", epSet.port[0]);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-a threads]: number of app threads, default is:%d\n", appThreads);
+      printf("  [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-u user]: user name for the connection, default is:%s\n", rpcInit.user);
+      printf("  [-k secret]: password for the connection, default is:%s\n", rpcInit.secret);
+      printf("  [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+
+  taosInitLog("client.log", 100000, 10);
+
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to initialize RPC");
+    return -1;
+  }
+
+  uInfo("client is initialized");
+
+  gettimeofday(&systemTime, NULL);
+  startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;
+
+  SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads);
+ 
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+  for (int i=0; i<appThreads; ++i) {
+    pInfo->index = i;
+    pInfo->epSet = epSet;
+    pInfo->numOfReqs = numOfReqs;
+    pInfo->msgSize = msgSize;
+    sem_init(&pInfo->rspSem, 0, 0);
+    pInfo->pRpc = pRpc;
+    pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo);
+    pInfo++;
+  }
+
+  do {
+    usleep(1);
+  } while ( tcount < appThreads);
+
+  gettimeofday(&systemTime, NULL);
+  endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;  
+  float usedTime = (endTime - startTime)/1000.0;  // mseconds
+
+  uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
+  uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
+
+  taosCloseLog();
+
+  return 0;
+}
+
+
diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c
new file mode 100644
index 0000000000..44019ad96e
--- /dev/null
+++ b/src/sync/test/syncServer.c
@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include <stdint.h>
+#include "os.h"
+#include "tulog.h"
+#include "tglobal.h"
+#include "tsocket.h"
+#include "trpc.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+
+int msgSize = 128;
+int commit = 0;
+int dataFd = -1;
+void *qhandle = NULL;
+int walNum = 0;
+uint64_t  tversion = 0;
+void *syncHandle;
+int   role;
+int  nodeId;
+char path[256];
+int  numOfWrites ;
+SSyncInfo syncInfo;
+SSyncCfg *pCfg;
+
+int writeIntoWal(SWalHead *pHead)
+{ 
+  if (dataFd < 0) {
+    char  walName[280];
+    snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum);
+    remove(walName);
+    dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);  
+    if (dataFd < 0) { 
+      uInfo("failed to open wal file:%s(%s)", walName, strerror(errno));
+      return -1;
+    } else {
+      walNum++;
+      uInfo("file:%s is opened to write, walNum:%d", walName, walNum);
+    }
+  }
+
+  if (write(dataFd, pHead, sizeof(SWalHead) + pHead->len) < 0) {
+    uError("ver:%" PRIu64 ", failed to write wal file(%s)", pHead->version, strerror(errno));
+  } else {
+    uDebug("ver:%" PRIu64 ", written to wal", pHead->version);
+  }
+
+  numOfWrites++;
+  if (numOfWrites >= 10000) {
+    uInfo("%d request have been written into disk", numOfWrites);
+    close(dataFd);
+    dataFd = -1;
+    numOfWrites = 0;
+  }
+ 
+  return 0;
+}
+
+void confirmForward(void *ahandle, void *mhandle, int32_t code)
+{
+  SRpcMsg  *pMsg = (SRpcMsg *)mhandle;
+  SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+
+  uDebug("ver:%" PRIu64 ", confirm is received", pHead->version);
+
+  rpcFreeCont(pMsg->pCont);
+
+  SRpcMsg    rpcMsg;
+  rpcMsg.pCont = rpcMallocCont(msgSize);
+  rpcMsg.contLen = msgSize;
+  rpcMsg.handle = pMsg->handle;
+  rpcMsg.code = code;
+  rpcSendResponse(&rpcMsg);
+
+  taosFreeQitem(mhandle); 
+}
+
+int processRpcMsg(void *item) {
+  SRpcMsg   *pMsg = (SRpcMsg *)item;
+  SWalHead  *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+  int        code = -1;
+
+  if (role != TAOS_SYNC_ROLE_MASTER) {
+    uError("not master, write failed, role:%s", syncRole[role]);
+  } else {
+
+    pHead->version = ++tversion;
+    pHead->msgType = pMsg->msgType;
+    pHead->len = pMsg->contLen;
+
+    uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version);
+    writeIntoWal(pHead); 
+    syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC);
+
+    code = 0;
+  }
+
+  if (pCfg->quorum <= 1) { 
+    taosFreeQitem(item); 
+    rpcFreeCont(pMsg->pCont);
+
+    SRpcMsg    rpcMsg;
+    rpcMsg.pCont = rpcMallocCont(msgSize);
+    rpcMsg.contLen = msgSize;
+    rpcMsg.handle = pMsg->handle;
+    rpcMsg.code = code;
+    rpcSendResponse(&rpcMsg);
+  }
+
+  return code;
+}
+
+int processFwdMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", forward is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  }
+
+  uDebug("ver:%" PRIu64 ", forward from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  if (pCfg->quorum > 1) syncConfirmForward(syncHandle, pHead->version, 0);
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+int processWalMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", wal is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  };
+
+  uDebug("ver:%" PRIu64 ", wal from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+void *processWriteQueue(void *param) {
+  int        type;
+  void      *item;
+
+  while (1) {
+    int ret = taosReadQitem(qhandle, &type, &item);
+    if (ret <= 0) {
+      usleep(1000);
+      continue;
+    }     
+
+    if (type == TAOS_QTYPE_RPC) {
+      processRpcMsg(item);
+    } else if (type == TAOS_QTYPE_WAL) {
+      processWalMsg(item);
+    } else if (type == TAOS_QTYPE_FWD) {
+      processFwdMsg(item);
+    } 
+
+  }
+
+  return NULL;
+}
+
+int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
+  // app shall retrieve the auth info based on meterID from DB or a data file
+  // demo code here only for simple demo
+  int ret = 0;
+
+  if (strcmp(meterId, "michael") == 0) {
+    *spi = 1;
+    *encrypt = 0;
+    strcpy(secret, "mypassword");
+    strcpy(ckey, "key");
+  } else if (strcmp(meterId, "jeff") == 0) {
+    *spi = 0;
+    *encrypt = 0;
+  } else {
+    ret = -1;  // user not there
+  }
+
+  return ret;
+}
+
+void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  
+  SRpcMsg *pTemp;
+
+  pTemp = taosAllocateQitem(sizeof(SRpcMsg));
+  memcpy(pTemp, pMsg, sizeof(SRpcMsg));
+  
+  uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen);
+  taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); 
+}
+
+uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) 
+{
+  uint32_t     magic;
+  struct stat  fstat;
+  char         aname[280];
+
+  if (*index == 2) {
+    uInfo("wait for a while .....");
+    sleep(3);
+  }
+
+  if (name[0] == 0) {
+    // find the file 
+    snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index);
+    sprintf(name, "data/data.%d", *index); 
+  } else {
+    snprintf(aname, sizeof(aname), "%s/%s", path, name);
+  }
+
+  uInfo("get file info:%s", aname);
+  if ( stat(aname, &fstat) < 0 ) return 0; 
+
+  *size = fstat.st_size;
+  magic = fstat.st_size;
+
+  return magic;
+}
+
+int  getWalInfo(void *ahandle, char *name, uint32_t *index) {
+
+  struct stat  fstat;
+  char         aname[280];
+
+  name[0] = 0;
+  if (*index + 1> walNum) return 0;
+
+  snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index);
+  sprintf(name, "wal/wal.%d", *index); 
+  uInfo("get wal info:%s", aname);
+
+  if ( stat(aname, &fstat) < 0 ) return -1;
+
+  if (*index >= walNum-1) return 0;  // no more
+
+  return 1;
+
+}
+
+int writeToCache(void *ahandle, void *data, int type) {
+  SWalHead *pHead = data;
+
+  uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type);
+
+  int   msgSize = pHead->len + sizeof(SWalHead);
+  void *pMsg = taosAllocateQitem(msgSize);
+  memcpy(pMsg, pHead, msgSize);
+  taosWriteQitem(qhandle, type, pMsg); 
+
+  return 0;
+}
+
+void confirmFwd(void *ahandle, int64_t version) {
+
+  return;
+}
+
+void notifyRole(void *ahandle, int8_t r) {
+  role = r;
+  printf("current role:%s\n", syncRole[role]);
+}
+
+
+void initSync() {
+
+  pCfg->replica = 1;
+  pCfg->quorum = 1;
+  syncInfo.vgId = 1;
+  syncInfo.ahandle = &syncInfo;
+  syncInfo.getFileInfo = getFileInfo;
+  syncInfo.getWalInfo = getWalInfo;
+  syncInfo.writeToCache = writeToCache;
+  syncInfo.confirmForward = confirmForward;
+  syncInfo.notifyRole = notifyRole;
+
+  pCfg->nodeInfo[0].nodeId = 1;
+  pCfg->nodeInfo[0].nodePort = 7010;
+  taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  pCfg->nodeInfo[1].nodeId = 2;
+  pCfg->nodeInfo[1].nodePort = 7110;
+  taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
+
+  pCfg->nodeInfo[2].nodeId = 3;
+  pCfg->nodeInfo[2].nodePort = 7210;
+  taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+
+  pCfg->nodeInfo[3].nodeId = 4;
+  pCfg->nodeInfo[3].nodePort = 7310;
+  taosGetFqdn(pCfg->nodeInfo[3].nodeFqdn);
+
+  pCfg->nodeInfo[4].nodeId = 5;
+  pCfg->nodeInfo[4].nodePort = 7410;
+  taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn);
+}
+
+void doSync()
+{
+  for (int i=0; i<5; ++i) {
+    if (tsSyncPort == pCfg->nodeInfo[i].nodePort) 
+      nodeId = pCfg->nodeInfo[i].nodeId;
+  }
+
+  snprintf(path, sizeof(path), "/root/test/d%d", nodeId);
+  strcpy(syncInfo.path, path);
+
+  if ( syncHandle == NULL) {
+      syncHandle = syncStart(&syncInfo);
+  } else {
+      if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL;
+  }
+
+  uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort);
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit rpcInit;
+  char     dataName[20] = "server.data";
+  pCfg =  &syncInfo.syncCfg;
+
+  initSync();
+
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 7000;
+  rpcInit.label        = "SER";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processRequestMsg;
+  rpcInit.sessions     = 1000;
+  rpcInit.idleTime     = tsShellActivityTimer*1500; 
+  rpcInit.afp          = retrieveAuthInfo;
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      rpcInit.localPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-w")==0 && i < argc-1) {
+      commit = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
+      syncInfo.version = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
+      pCfg->replica = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q")==0 && i < argc-1) {
+      pCfg->quorum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", rpcInit.localPort);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit);
+      printf("  [-v version]: initial node version, default is:%ld\n", syncInfo.version);
+      printf("  [-r replica]: replicacation number, default is:%d\n", pCfg->replica);
+      printf("  [-q quorum]: quorum, default is:%d\n", pCfg->quorum);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+ 
+  uDebugFlag = rpcDebugFlag;
+  dDebugFlag = rpcDebugFlag; 
+  //tmrDebugFlag = rpcDebugFlag; 
+  tsAsyncLog = 0;
+  taosInitLog("server.log", 1000000, 10);
+
+  rpcInit.connType = TAOS_CONN_SERVER;
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to start RPC server");
+    return -1;
+  }
+
+  tsSyncPort = rpcInit.localPort + 10;
+  qhandle = taosOpenQueue();
+
+  doSync();
+
+  pthread_attr_t thattr;
+  pthread_t      thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&thread, &thattr, processWriteQueue, NULL) != 0) {
+    uError("failed to create thread, reason:%s", strerror(errno));
+    return -1;
+  }
+
+  printf("server is running, localPort:%d\n", rpcInit.localPort);
+  SNodesRole nroles;
+
+  while (1) {
+    char c = getchar();
+
+    switch(c) {
+      case '1':
+        pCfg->replica = 1; doSync();
+        break;        
+      case '2':
+        pCfg->replica = 2; doSync();
+        break;
+      case '3':
+        pCfg->replica = 3; doSync();
+        break;
+      case '4':
+        pCfg->replica = 4; doSync();
+        break;
+      case '5':
+        pCfg->replica = 5; doSync();
+        break;
+      case 's':
+        syncGetNodesRole(syncHandle, &nroles);
+        for (int i=0; i<pCfg->replica; ++i) 
+          printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]);
+        break;
+      default:
+        break;
+    }
+
+    if (c=='q') break;
+
+  }
+
+  syncStop(syncHandle);
+
+  if (dataFd >= 0) {
+    close(dataFd);
+    remove(dataName);
+  }
+
+  return 0;
+}
+
+
diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt
index b2154969d6..450afe337b 100644
--- a/src/tsdb/CMakeLists.txt
+++ b/src/tsdb/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   # INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/common/inc)
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index f698878698..6e7b39830e 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -12,23 +12,14 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <libgen.h>
+#define _DEFAULT_SOURCE
+#include "os.h"
 
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tsdbMain.h"
 #include "tutil.h"
 #include "ttime.h"
-#include "tfile.h"
 
 #ifdef TSDB_IDX
 const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"};
@@ -317,7 +308,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
-  if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -383,7 +374,7 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
   pFile->fd = -1;
   if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
 
-  if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index 89a19c15af..3a453249a9 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -545,7 +545,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
 
-  if (twrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -586,7 +586,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
     goto _err;
   }
 
-  if (tread(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index 09bbbd8f4d..ddb935b7e5 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -69,8 +69,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
 
   if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) {
     if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) {
-      tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
-                TABLE_TID(pTable), TABLE_UID(pTable));
+      tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]),
+                TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid]));
       return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
     } else {
       tsdbError("vgId:%d table %s at tid %d uid %" PRIu64
@@ -1295,4 +1295,4 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
   tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables);
 
   return 0;
-}
\ No newline at end of file
+}
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
index 263e1ea3d5..9456996071 100644
--- a/src/tsdb/src/tsdbRWHelper.c
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tscompression.h"
 #include "tsdbMain.h"
-#include "tfile.h"
 
 #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
 #define TSDB_KEY_COL_OFFSET 0
@@ -379,7 +379,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
         return -1;
       }
 
-      if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
+      if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
         tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
                   helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -424,7 +424,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     pIdx->tid = pHelper->tableInfo.tid;
     ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-    if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+    if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
       tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                 pFile->fname, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -481,7 +481,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
 
   pFile->info.offset = offset;
 
-  if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
+  if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -514,7 +514,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
         return -1;
       }
 
-      if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
+      if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
                   pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -585,7 +585,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) {
       }
 
       pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len);
-      if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+      if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                   helperHeadF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -626,7 +626,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
     return -1;
   }
 
-  if (tread(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
+  if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
     tsdbError("vgId:%d failed to read %zu bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -841,7 +841,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
                                        sizeof(TSCKSUM));
 
   // Write the whole block to file
-  if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+  if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1222,7 +1222,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl
     return -1;
   }
 
-  if (tread(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
+  if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1260,13 +1260,21 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock,
     SCompCol *pCompCol = NULL;
 
     while (true) {
-      ASSERT(dcol < pDataCols->numOfCols);
+      if (dcol >= pDataCols->numOfCols) {
+        pDataCol = NULL;
+        break;
+      }
       pDataCol = &pDataCols->cols[dcol];
-      ASSERT(pDataCol->colId <= colId);
-      if (pDataCol->colId == colId) break;
-      dcol++;
+      if (pDataCol->colId > colId) {
+        pDataCol = NULL;
+        break;
+      } else {
+        dcol++;
+        if (pDataCol->colId == colId) break;
+      }
     }
 
+    if (pDataCol == NULL) continue;
     ASSERT(pDataCol->colId == colId);
 
     if (colId == 0) {  // load the key row
@@ -1276,15 +1284,24 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock,
       compCol.offset = TSDB_KEY_COL_OFFSET;
       pCompCol = &compCol;
     } else {  // load non-key rows
-      while (ccol < pCompBlock->numOfCols) {
-        pCompCol = &pHelper->pCompData->cols[ccol];
-        if (pCompCol->colId >= colId) break;
-        ccol++;
+      while (true) {
+        if (ccol >= pCompBlock->numOfCols) {
+          pCompCol = NULL;
+          break;
+        }
+
+        pCompCol = &(pHelper->pCompData->cols[ccol]);
+        if (pCompCol->colId > colId) {
+          pCompCol = NULL;
+          break;
+        } else {
+          ccol++;
+          if (pCompCol->colId == colId) break;
+        }
       }
 
-      if (ccol >= pCompBlock->numOfCols || pCompCol->colId > colId) {
+      if (pCompCol == NULL) {
         dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
-        dcol++;
         continue;
       }
 
@@ -1292,8 +1309,6 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock,
     }
 
     if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err;
-    dcol++;
-    if (colId != 0) ccol++;
   }
 
   return 0;
@@ -1322,7 +1337,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-  if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+  if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index c0f6c82b99..0e3a657fde 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -30,11 +30,6 @@
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)
 #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
 
-enum {
-  QUERY_RANGE_LESS_EQUAL = 0,
-  QUERY_RANGE_GREATER_EQUAL = 1,
-};
-
 enum {
   TSDB_QUERY_TYPE_ALL      = 1,
   TSDB_QUERY_TYPE_LAST     = 2,
@@ -210,7 +205,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   if (pQueryHandle->pColumns == NULL) {
     goto out_of_memory;
   }
-  
+
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {{0}, 0};
   
@@ -222,29 +217,29 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
     taosArrayPush(pQueryHandle->pColumns, &colInfo);
     pQueryHandle->statis[i].colId = colInfo.info.colId;
   }
-  
+
   pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
   if (pQueryHandle->pTableCheckInfo == NULL) {
     goto out_of_memory;
   }
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL);
-  
+
   for (int32_t i = 0; i < sizeOfGroup; ++i) {
     SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
-    
+
     size_t gsize = taosArrayGetSize(group);
     assert(gsize > 0);
-    
+
     for (int32_t j = 0; j < gsize; ++j) {
       STable* pTable = (STable*) taosArrayGetP(group, j);
-      
+
       STableCheckInfo info = {
           .lastKey = pQueryHandle->window.skey,
           .tableId = pTable->tableId,
           .pTableObj = pTable,
       };
-      
+
       assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
       info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
 
@@ -280,17 +275,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
 
 SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   assert(pHandle != NULL);
-  
+
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle;
-  
+
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   SArray* res = taosArrayInit(size, POINTER_BYTES);
-  
+
   for(int32_t i = 0; i < size; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     taosArrayPush(res, &pCheckInfo->pTableObj);
   }
-  
+
   return res;
 }
 
@@ -306,11 +301,11 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond
 static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) {
   STable* pTable = pCheckInfo->pTableObj;
   assert(pTable != NULL);
-  
+
   if (pCheckInfo->initBuf) {
     return true;
   }
-  
+
   pCheckInfo->initBuf = true;
   int32_t order = pHandle->order;
 
@@ -318,7 +313,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pHandle->mem == NULL && pHandle->imem == NULL) {
     return false;
   }
-  
+
   assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
 
   // TODO: add uid check
@@ -338,17 +333,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) {
     return false;
   }
-  
+
   bool memEmpty  = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
   bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
   if (memEmpty && imemEmpty) { // buffer is empty
     return false;
   }
-  
+
   if (!memEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -357,11 +352,11 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   if (!imemEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -370,7 +365,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   return true;
 }
 
@@ -473,7 +468,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
   pHandle->cur.fid = -1;
-  
+
   STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
 
   STable* pTable = pCheckInfo->pTableObj;
@@ -491,17 +486,17 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   pCheckInfo->lastKey = dataRowKey(row);  // first timestamp in buffer
   tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle,
       pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo);
-  
+
   // all data in mem are checked already.
   if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) ||
       (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_TRAVERSE(pHandle->order))) {
     return false;
   }
-  
+
   int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
   STimeWindow* win = &pHandle->cur.win;
   pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle);
-  
+
   // update the last key value
   pCheckInfo->lastKey = win->ekey + step;
   pHandle->cur.lastKey = win->ekey + step;
@@ -510,7 +505,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   if (!ASCENDING_TRAVERSE(pHandle->order)) {
     SWAP(win->skey, win->ekey, TSKEY);
   }
-  
+
   return true;
 }
 
@@ -519,31 +514,31 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   if (key == TSKEY_INITIAL_VAL) {
     return INT32_MIN;
   }
-  
+
   int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
     fid = INT32_MIN;
   }
-  
+
   if (fid > 0L && fid > INT32_MAX) {
     fid = INT32_MAX;
   }
-  
+
   return fid;
 }
 
 static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
   int32_t firstSlot = 0;
   int32_t lastSlot = numOfBlocks - 1;
-  
+
   int32_t midSlot = firstSlot;
-  
+
   while (1) {
     numOfBlocks = lastSlot - firstSlot + 1;
     midSlot = (firstSlot + (numOfBlocks >> 1));
-    
+
     if (numOfBlocks == 1) break;
-    
+
     if (skey > pBlock[midSlot].keyLast) {
       if (numOfBlocks == 2) break;
       if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break;
@@ -555,7 +550,7 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK
       break;  // got the slot
     }
   }
-  
+
   return midSlot;
 }
 
@@ -644,7 +639,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
                     .uid = (_checkInfo)->tableId.uid})
 
 
-static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
+static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
   STsdbRepo *pRepo = pQueryHandle->pTsdb;
   bool       blockLoaded = false;
   int64_t    st = taosGetTimestampUs();
@@ -678,8 +673,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 
   int64_t elapsedTime = (taosGetTimestampUs() - st);
   pQueryHandle->cost.blockLoadTime += elapsedTime;
-  tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, elapsedTime);
 
+  tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p",
+      pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo);
   return blockLoaded;
 }
 
@@ -692,18 +688,17 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
 
   TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
-  
+
   if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
       (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-    
+
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
         (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
 
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
 
-      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step,
-                                        pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
+      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
       pQueryHandle->realNumOfRows = cur->rows;
 
       // update the last key value
@@ -711,13 +706,13 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
       if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
         SWAP(cur->win.skey, cur->win.ekey, TSKEY);
       }
-      
+
       cur->mixBlock = true;
       cur->blockCompleted = false;
       return;
     }
-  
-    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
+
+    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot);
     doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
   } else {
     /*
@@ -742,39 +737,41 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-    // query ended in current block
+    // query ended in/started from current block
     if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
 
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
-      
+
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
         cur->pos =
             binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = 0;
       }
-      
+
+      assert(pCheckInfo->lastKey <= pBlock->keyLast);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {  // the whole block is loaded in to buffer
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
     }
   } else {  //desc order, query ended in current block
     if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
-  
+
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
         cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
-      
+
+      assert(pCheckInfo->lastKey >= pBlock->keyFirst);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
@@ -790,7 +787,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
   TSKEY* keyList;
 
   assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
-  
+
   if (num <= 0) return -1;
 
   keyList = (TSKEY*)pValue;
@@ -849,13 +846,19 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
   char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
-  
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
-  
+
   int32_t num = end - start + 1;
+  assert(num >= 0);
+
+  if (num == 0) {
+    return numOfRows;
+  }
+
   int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-  
+
   //data in buffer has greater timestamp, copy data in file block
   int32_t i = 0, j = 0;
   while(i < requiredNumOfCols && j < pCols->numOfCols) {
@@ -928,7 +931,7 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
 
     i++;
   }
-  
+
   pQueryHandle->cur.win.ekey = tsArray[end];
   pQueryHandle->cur.lastKey = tsArray[end] + step;
 
@@ -995,24 +998,99 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
   }
 }
 
+static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) {
+  if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    return;
+  }
+
+  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+  if (numOfRows < pQueryHandle->outputCapacity) {
+    int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
+    for(int32_t i = 0; i < numOfCols; ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+      memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+    }
+  }
+}
+
+static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos,
+    int32_t numOfExisted, int32_t *start, int32_t *end) {
+  *start = -1;
+
+  if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    int32_t remain = endPos - startPos + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1;
+    } else {
+      *end = endPos;
+    }
+
+    *start = startPos;
+  } else {
+    int32_t remain = (startPos - endPos) + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted);
+    } else {
+      *end = endPos;
+    }
+
+    *start = *end;
+    *end = startPos;
+  }
+}
+
+static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  pCheckInfo->lastKey = cur->lastKey;
+  pQueryHandle->realNumOfRows = numOfRows;
+  cur->rows = numOfRows;
+  cur->pos = endPos;
+}
+
+static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  if (cur->rows > 0) {
+    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey);
+    } else {
+      assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey);
+    }
+
+    SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0);
+    assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]);
+  } else {
+    cur->win = pQueryHandle->window;
+
+    int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
+    cur->lastKey = pQueryHandle->window.ekey + step;
+  }
+}
+
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
 static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
-  
+
   initTableMemIterator(pQueryHandle, pCheckInfo);
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
+      cur->pos >= 0 && cur->pos < pBlock->numOfRows);
+
+  TSKEY* tsArray = pCols->cols[0].pData;
 
   // for search the endPos, so the order needs to reverse
   int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
 
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
-  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+  int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
 
   STable* pTable = pCheckInfo->pTableObj;
-
   int32_t endPos = cur->pos;
+
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
     endPos = blockInfo.rows - 1;
     cur->mixBlock = (cur->pos != 0);
@@ -1024,48 +1102,36 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
     cur->mixBlock = true;
   }
-  
+
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
-  int32_t pos = cur->pos;
-  
-  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == 0);
-  TSKEY* tsArray = pCols->cols[0].pData;
-  
   int32_t numOfRows = 0;
-  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
+  int32_t pos = cur->pos;
+  cur->win = TSWINDOW_INITIALIZER;
 
   // no data in buffer, load data from file directly
   if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
     int32_t start = cur->pos;
     int32_t end = endPos;
-    if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
-      end = cur->pos;
-      start = endPos;
-    }
-    
-    cur->win.skey = tsArray[start];
-    cur->win.ekey = tsArray[end];
-    
-    // todo opt in case of no data in buffer
-    numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-    
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
 
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
+    if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      SWAP(start, end, int32_t);
     }
-  
+
+    numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
+
+    // the time window should always be right order: skey <= ekey
+    cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]};
+    cur->lastKey = tsArray[endPos];
     pos += (end - start + 1) * step;
-    cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-        ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
-    
-    pCheckInfo->lastKey = cur->lastKey;
-    pQueryHandle->realNumOfRows = numOfRows;
-    cur->rows = numOfRows;
+
+    cur->blockCompleted =
+        (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+         ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+    moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+    updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+    doCheckGeneratedBlockRange(pQueryHandle);
     return;
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
@@ -1111,30 +1177,18 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
           moveToNextRowInMem(pCheckInfo);
         }
-        
-        int32_t start = -1;
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = end - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
 
-          start = pos;
-        } else {
-          int32_t remain = (pos - end) + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
+        int32_t qstart = 0, qend = 0;
+        getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend);
 
-          start = end;
-          end = pos;
-        }
+        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend);
+        pos += (qend - qstart + 1) * step;
 
-        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-        pos += (end - start + 1) * step;
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart];
+        cur->lastKey  = cur->win.ekey + step;
       }
     } while (numOfRows < pQueryHandle->outputCapacity);
-    
+
     if (numOfRows < pQueryHandle->outputCapacity) {
       /**
        * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
@@ -1148,54 +1202,29 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
           cur->win.skey = tsArray[pos];
         }
 
-        int32_t start = -1;
-        int32_t end = -1;
-
-        // all remain data are qualified, but check the remain capacity in the first place.
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = endPos - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
-
-          start = pos;
-          end = endPos;
-        } else {
-          int32_t remain = pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
-
-          start = endPos;
-          end = pos;
-        }
+        int32_t start = -1, end = -1;
+        getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end);
 
         numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
+
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start];
+        cur->lastKey  = cur->win.ekey + step;
       }
     }
   }
-  
-  cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-      ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+  cur->blockCompleted =
+      (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+       ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
 
   if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
     SWAP(cur->win.skey, cur->win.ekey, TSKEY);
-  
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
-    }
   }
-  
-  pCheckInfo->lastKey = cur->lastKey;
-  pQueryHandle->realNumOfRows = numOfRows;
-  cur->rows = numOfRows;
-  cur->pos = pos;
+
+  moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+  updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+  doCheckGeneratedBlockRange(pQueryHandle);
 
   tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey,
       cur->win.ekey, cur->rows, pQueryHandle->qinfo);
@@ -1332,16 +1361,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     cleanBlockOrderSupporter(&sup, 0);
     return TSDB_CODE_TDB_OUT_OF_MEMORY;
   }
-  
+
   int32_t cnt = 0;
   int32_t numOfQualTables = 0;
-  
+
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j);
     if (pTableCheck->numOfBlocks <= 0) {
       continue;
     }
-    
+
     SCompBlock* pBlock = pTableCheck->pCompInfo->blocks;
     sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks;
 
@@ -1437,37 +1466,39 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
-    (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
-      tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo)
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
+                pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
       pQueryHandle->pFileGroup = NULL;
+      assert(pQueryHandle->numOfBlocks == 0);
       break;
     }
 
     if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
-    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks,
-           numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
-    
+
+    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
+              pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
+
     assert(numOfBlocks >= 0);
     if (numOfBlocks == 0) {
       continue;
     }
-    
+
     // todo return error code to query engine
-    if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) {
+    if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
+
     assert(numOfBlocks >= pQueryHandle->numOfBlocks);
     if (pQueryHandle->numOfBlocks > 0) {
       break;
     }
   }
-  
+
   // no data in file anymore
-  if (pQueryHandle->numOfBlocks <= 0) {
+  if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) {
     if (code == TSDB_CODE_SUCCESS) {
       assert(pQueryHandle->pFileGroup == NULL);
     }
@@ -1476,10 +1507,11 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
     *exists = false;
     return code;
   }
-  
+
+  assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0);
   cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1;
   cur->fid = pQueryHandle->pFileGroup->fileId;
-  
+
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
   *exists = loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo);
 
@@ -1495,7 +1527,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     pQueryHandle->locateStart = true;
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
     int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
-    
+
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
 
@@ -1504,7 +1536,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     // check if current file block is all consumed
     STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
     STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
-    
+
     // current block is done, try next
     if (!cur->mixBlock || cur->blockCompleted) {
       if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
@@ -1515,10 +1547,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
         // next block of the same file
         int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
         cur->slot += step;
-        
+
         cur->mixBlock = false;
         cur->blockCompleted = false;
-        
+
         STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
         *exists = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo);
 
@@ -1540,10 +1572,10 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     if (hasMoreDataInCache(pQueryHandle)) {
       return true;
     }
-    
+
     pQueryHandle->activeIndex += 1;
   }
-  
+
   return false;
 }
 
@@ -1561,14 +1593,14 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
     pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
     pQueryHandle->order = TSDB_ORDER_DESC;
-    
+
     if (!tsdbNextDataBlock(pHandle)) {
       return false;
     }
-    
+
     /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
     /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
-  
+
     if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
       // data already retrieve, discard other data rows and return
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
@@ -1576,7 +1608,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
       }
-  
+
       pQueryHandle->cur.win  = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
       pQueryHandle->window   = pQueryHandle->cur.win;
       pQueryHandle->cur.rows = 1;
@@ -1593,7 +1625,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pSecQueryHandle->checkFiles  = true;
       pSecQueryHandle->activeIndex = 0;
       pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock;
-  
+
       if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) {
         free(pSecQueryHandle);
         return false;
@@ -1603,24 +1635,24 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       // allocate buffer in order to load data blocks from file
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
-  
+
       pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
       pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
 
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData colInfo = {{0}, 0};
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
-        
+
         colInfo.info = pCol->info;
         colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes);
         taosArrayPush(pSecQueryHandle->pColumns, &colInfo);
       }
-  
+
       size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
       pSecQueryHandle->pTableCheckInfo = taosArrayInit(si, sizeof(STableCheckInfo));
       STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
       assert(pMeta != NULL);
-  
+
       for (int32_t j = 0; j < si; ++j) {
         STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
         STableCheckInfo info = {
@@ -1628,10 +1660,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
             .tableId = pCheckInfo->tableId,
             .pTableObj = pCheckInfo->pTableObj,
         };
-        
+
         taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info);
       }
-  
+
       tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo);
       tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo);
       pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn);
@@ -1641,17 +1673,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
       tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
-  
+
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
-  
+
         SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
         assert(pCol->info.colId == pCol1->info.colId);
-        
+
         memcpy(pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
       }
-  
+
       SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
 
       // it is ascending order
@@ -1675,7 +1707,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
     pQueryHandle->checkFiles = false;
     return true;
   }
-  
+
   if (pQueryHandle->checkFiles) {
     bool exists = true;
     int32_t code = getDataBlocksInFiles(pQueryHandle, &exists);
@@ -1688,11 +1720,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pQueryHandle->cost.checkForNextTime += elapsedTime;
       return exists;
     }
-  
+
     pQueryHandle->activeIndex = 0;
     pQueryHandle->checkFiles  = false;
   }
-  
+
   // TODO: opt by consider the scan order
   bool ret = doHasDataInBuffer(pQueryHandle);
   terrno = TSDB_CODE_SUCCESS;
@@ -1705,15 +1737,15 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
   assert(!ASCENDING_TRAVERSE(pQueryHandle->order));
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
-  
+
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   TSKEY key = TSKEY_INITIAL_VAL;
   int32_t index = -1;
-  
+
   for(int32_t i = 0; i < numOfTables; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     if (pCheckInfo->pTableObj->lastKey > key) {
@@ -1721,36 +1753,36 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
       index = i;
     }
   }
-  
+
   if (index == -1) {
     // todo add failure test cases
     return;
   }
-  
+
   // erase all other elements in array list
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   for (int32_t i = 0; i < size; ++i) {
     if (i == index) {
       continue;
     }
-    
+
     STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     tSkipListDestroyIter(pTableCheckInfo->iter);
-    
+
     if (pTableCheckInfo->pDataCols != NULL) {
       tfree(pTableCheckInfo->pDataCols->buf);
     }
-    
+
     tfree(pTableCheckInfo->pDataCols);
     tfree(pTableCheckInfo->pCompInfo);
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = key;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {key, key};
 }
@@ -1759,13 +1791,13 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
   // filter the queried time stamp in the first place
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
   pQueryHandle->order = TSDB_ORDER_DESC;
-  
+
   assert(pQueryHandle->window.skey == pQueryHandle->window.ekey);
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   int32_t i = 0;
   while(i < numOfTables) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
@@ -1773,21 +1805,21 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
         pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) {
       break;
     }
-    
+
     i++;
   }
-  
+
   // there are no data in all the tables
   if (i == numOfTables) {
     return;
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = pQueryHandle->window.skey;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL};
 }
@@ -1811,7 +1843,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
       tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
           pQueryHandle->window.ekey);
-      
+
       break;
     }
 
@@ -1826,24 +1858,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
       moveToNextRowInMem(pCheckInfo);
       break;
     }
-    
+
   } while(moveToNextRowInMem(pCheckInfo));
 
   assert(numOfRows <= maxRowsToRead);
-  
+
   // if the buffer is not full in case of descending order query, move the data in the front of the buffer
   if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) {
     int32_t emptySize = maxRowsToRead - numOfRows;
-    
+
     for(int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
       memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
     }
   }
-  
+
   int64_t elapsedTime = taosGetTimestampUs() - st;
-  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle,
-            elapsedTime, numOfRows, numOfCols);
+  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle,
+            elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo);
 
   return numOfRows;
 }
@@ -1852,7 +1884,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
   SQueryFilePos* cur = &pHandle->cur;
   STable* pTable = NULL;
-  
+
   // there are data in file
   if (pHandle->cur.fid >= 0) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
@@ -1874,13 +1906,13 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
  */
 int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
-  
+
   SQueryFilePos* c = &pHandle->cur;
   if (c->mixBlock) {
     *pBlockStatis = NULL;
     return TSDB_CODE_SUCCESS;
   }
-  
+
   STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[c->slot];
   assert((c->slot >= 0 && c->slot < pHandle->numOfBlocks) || ((c->slot == pHandle->numOfBlocks) && (c->slot == 0)));
 
@@ -1900,7 +1932,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   for(int32_t i = 0; i < numOfCols; ++i) {
     pHandle->statis[i].colId = colIds[i];
   }
-  
+
   tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols);
 
   // always load the first primary timestamp column data
@@ -1949,31 +1981,31 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
     } else {
       SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
       assert(pHandle->realNumOfRows <= binfo.rows);
-  
+
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
-      
+
       if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
         SCompBlock* pBlock = pBlockInfo->compBlock;
-        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
-  
+        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot);
+
         // todo refactor
         int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-  
+
         // if the buffer is not full in case of descending order query, move the data in the front of the buffer
         if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
           int32_t emptySize = pHandle->outputCapacity - numOfRows;
           int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns);
-    
+
           for(int32_t i = 0; i < reqNumOfCols; ++i) {
             SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
             memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
           }
         }
-        
+
         return pHandle->pColumns;
       }
     }
@@ -1984,11 +2016,11 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
   SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
-    
+
     STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
     taosArrayPush(list, pTable);
   }
-  
+
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
@@ -1998,12 +2030,12 @@ static void destroyHelper(void* param) {
     return;
   }
 
-  
+
   tQueryInfo* pInfo = (tQueryInfo*)param;
   if (pInfo->optr != TSDB_RELATION_IN) {
     tfree(pInfo->q);
   }
-  
+
 //  tVariantDestroy(&(pInfo->q));
   free(param);
 }
@@ -2015,7 +2047,7 @@ void filterPrepare(void* expr, void* param) {
   }
 
   pExpr->_node.info = calloc(1, sizeof(tQueryInfo));
-  
+
   STSchema*   pTSSchema = (STSchema*) param;
   tQueryInfo* pInfo = pExpr->_node.info;
   tVariant*   pCond = pExpr->_node.pRight->pVal;
@@ -2025,7 +2057,7 @@ void filterPrepare(void* expr, void* param) {
   pInfo->optr     = pExpr->_node.optr;
   pInfo->compare  = getComparFunc(pSchema->type, pInfo->optr);
   pInfo->param    = pTSSchema;
-  
+
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
   } else {
@@ -2038,25 +2070,24 @@ typedef struct STableGroupSupporter {
   int32_t    numOfCols;
   SColIndex* pCols;
   STSchema*  pTagSchema;
-//  void*      tsdbMeta;
 } STableGroupSupporter;
 
 int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
   STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
   STable* pTable1 = *(STable**) p1;
   STable* pTable2 = *(STable**) p2;
-  
+
   for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
     SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
     int32_t colIndex = pColIndex->colIndex;
-    
+
     assert(colIndex >= TSDB_TBNAME_COLUMN_INDEX);
-    
+
     char *  f1 = NULL;
     char *  f2 = NULL;
     int32_t type = 0;
     int32_t bytes = 0;
-    
+
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       f1 = (char*) TABLE_NAME(pTable1);
       f2 = (char*) TABLE_NAME(pTable2);
@@ -2090,14 +2121,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
       return ret;
     }
   }
-  
+
   return 0;
 }
 
 void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp,
     __ext_compar_fn_t compareFn) {
   STable* pTable = taosArrayGetP(pTableList, 0);
-  
+
   SArray* g = taosArrayInit(16, POINTER_BYTES);
   taosArrayPush(g, &pTable);
   tsdbRefTable(pTable);
@@ -2105,10 +2136,10 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
   for (int32_t i = 1; i < numOfTables; ++i) {
     STable** prev = taosArrayGet(pTableList, i - 1);
     STable** p = taosArrayGet(pTableList, i);
-    
+
     int32_t ret = compareFn(prev, p, pSupp);
     assert(ret == 0 || ret == -1);
-    
+
     tsdbRefTable(*p);
     assert((*p)->type == TSDB_CHILD_TABLE);
 
@@ -2120,20 +2151,20 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
       taosArrayPush(g, p);
     }
   }
-  
+
   taosArrayPush(pGroups, &g);
 }
 
 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) {
   assert(pTableList != NULL);
   SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
-  
+
   size_t size = taosArrayGetSize(pTableList);
   if (size == 0) {
     tsdbDebug("no qualified tables");
     return pTableGroup;
   }
-  
+
   if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table
     SArray* sa = taosArrayInit(size, POINTER_BYTES);
     for(int32_t i = 0; i < size; ++i) {
@@ -2143,7 +2174,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
       tsdbRefTable(*pTable);
       taosArrayPush(sa, pTable);
     }
-    
+
     taosArrayPush(pTableGroup, &sa);
     tsdbDebug("all %zu tables belong to one group", size);
   } else {
@@ -2151,18 +2182,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
     pSupp->numOfCols = numOfOrderCols;
     pSupp->pTagSchema = pTagSchema;
     pSupp->pCols = pCols;
-    
+
     taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn);
     createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn);
     tfree(pSupp);
   }
-  
+
   return pTableGroup;
 }
 
 bool indexedNodeFilterFp(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
-  
+
   STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
@@ -2172,7 +2203,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
   } else {
     val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
   }
-  
+
   int32_t ret = 0;
   if (val == NULL) { //the val is possible to be null, so check it out carefully
     ret = -1; // val is missing in table tags value pairs
@@ -2209,7 +2240,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
     default:
       assert(false);
   }
-  
+
   return true;
 }
 
@@ -2239,7 +2270,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     goto _error;
   }
-  
+
   if (pTable->type != TSDB_SUPER_TABLE) {
     tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
         pTable->name->data);
@@ -2252,7 +2283,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   //NOTE: not add ref count for super table
   SArray* res = taosArrayInit(8, POINTER_BYTES);
   STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
-  
+
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
     int32_t ret = getAllTableList(pTable, res);
@@ -2263,7 +2294,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     pGroupInfo->numOfTables = taosArrayGetSize(res);
     pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
-      
+
     tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
     taosArrayDestroy(res);
 
@@ -2299,7 +2330,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   } CATCH( code ) {
     CLEANUP_EXECUTE();
     terrno = code;
-    goto _error; 
+    goto _error;
     // TODO: more error handling
   } END_TRY
 
@@ -2335,12 +2366,12 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p
 
   pGroupInfo->numOfTables = 1;
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
-  
+
   SArray* group = taosArrayInit(1, POINTER_BYTES);
-  
+
   taosArrayPush(group, &pTable);
   taosArrayPush(pGroupInfo->pGroupList, &group);
-  
+
   return TSDB_CODE_SUCCESS;
 
   _error:
@@ -2428,8 +2459,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   tsdbDestroyHelper(&pQueryHandle->rhelper);
 
   SIOCostSummary* pCost = &pQueryHandle->cost;
-  tsdbDebug(":io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
-      pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
+  tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
+      pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
 
   tfree(pQueryHandle);
 }
diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt
index d4350fc8b2..2521c582d1 100644
--- a/src/util/CMakeLists.txt
+++ b/src/util/CMakeLists.txt
@@ -1,8 +1,8 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 
@@ -11,10 +11,11 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
+  TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4)
+  ADD_SUBDIRECTORY(tests)
+
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
     FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64)
     FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64)
     IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST)
@@ -28,9 +29,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     MESSAGE(STATUS "Failed to find iconv, use default encoding method")
   ENDIF ()
 
-  ADD_SUBDIRECTORY(tests)
 ELSEIF (TD_WINDOWS_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
@@ -68,44 +67,11 @@ ELSEIF (TD_WINDOWS_64)
   LIST(APPEND SRC ./src/tutil.c)
   LIST(APPEND SRC ./src/version.c)
   ADD_LIBRARY(tutil ${SRC}) 
-  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
-  LIST(APPEND SRC ./src/hash.c)
-  LIST(APPEND SRC ./src/ihash.c)
-  LIST(APPEND SRC ./src/lz4.c)
-  LIST(APPEND SRC ./src/shash.c)
-  LIST(APPEND SRC ./src/tbase64.c)
-  LIST(APPEND SRC ./src/tcache.c)
-  LIST(APPEND SRC ./src/tcompression.c)
-  LIST(APPEND SRC ./src/textbuffer.c)
-  LIST(APPEND SRC ./src/tglobalcfg.c)
-  LIST(APPEND SRC ./src/thash.c)
-  LIST(APPEND SRC ./src/thashutil.c)
-  LIST(APPEND SRC ./src/thistogram.c)
-  LIST(APPEND SRC ./src/tidpool.c)
-  LIST(APPEND SRC ./src/tinterpolation.c)
-  LIST(APPEND SRC ./src/tlog.c)
-  LIST(APPEND SRC ./src/tlosertree.c)
-  LIST(APPEND SRC ./src/tmd5.c)
-  LIST(APPEND SRC ./src/tmem.c)
-  LIST(APPEND SRC ./src/tmempool.c)
-  LIST(APPEND SRC ./src/tmodule.c)
-  LIST(APPEND SRC ./src/tnote.c)
-  LIST(APPEND SRC ./src/tpercentile.c)
-  LIST(APPEND SRC ./src/tsched.c)
-  LIST(APPEND SRC ./src/tskiplist.c)
-  LIST(APPEND SRC ./src/tsocket.c)
-  LIST(APPEND SRC ./src/tstrbuild.c)
-  LIST(APPEND SRC ./src/ttime.c)
-  LIST(APPEND SRC ./src/ttimer.c)
-  LIST(APPEND SRC ./src/ttokenizer.c)
-  LIST(APPEND SRC ./src/ttypes.c)
-  LIST(APPEND SRC ./src/tutil.c)
-  LIST(APPEND SRC ./src/version.c)
-  LIST(APPEND SRC ./src/hash.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4)
 ENDIF()
 
 #  TARGET_LINK_LIBRARIES(tutil mstorage)
diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h
index 5a3545fd8f..1e2aeae394 100644
--- a/src/util/inc/tcache.h
+++ b/src/util/inc/tcache.h
@@ -68,8 +68,6 @@ typedef struct {
   int64_t         refreshTime;
   STrashElem *    pTrash;
   char*           name;
-//  void *          tmrCtrl;
-//  void *          pTimer;
   SCacheStatis    statistics;
   SHashObj *      pHashTable;
   __cache_free_fn_t freeFp;
diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h
deleted file mode 100644
index de52a40967..0000000000
--- a/src/util/inc/tfile.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TFILE_H
-#define TDENGINE_TFILE_H
-
-#ifdef TAOS_RANDOM_FILE_FAIL
-
-void taosSetRandomFileFailFactor(int factor);
-void taosSetRandomFileFailOutput(const char *path);
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line);
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line);
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line);
-
-#define tread(fd, buf, count)  taos_tread(fd, buf, count, __FILE__, __LINE__)
-#define twrite(fd, buf, count)  taos_twrite(fd, buf, count, __FILE__, __LINE__)
-#define lseek(fd, offset, whence)  taos_lseek(fd, offset, whence, __FILE__, __LINE__)
-
-#endif  // TAOS_RANDOM_FILE_FAIL
-
-#endif  // TDENGINE_TFILE_H
diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h
index a4ed9311e2..e8380294da 100644
--- a/src/util/inc/tlist.h
+++ b/src/util/inc/tlist.h
@@ -55,6 +55,8 @@ int        tdListPrepend(SList *list, void *data);
 int        tdListAppend(SList *list, void *data);
 SListNode *tdListPopHead(SList *list);
 SListNode *tdListPopTail(SList *list);
+SListNode *tdListGetHead(SList *list);
+SListNode *tsListGetTail(SList *list);
 SListNode *tdListPopNode(SList *list, SListNode *node);
 void       tdListMove(SList *src, SList *dst);
 void       tdListDiscard(SList *list);
diff --git a/src/util/inc/ttime.h b/src/util/inc/ttime.h
index 576c9a51f6..25d1bdb23e 100644
--- a/src/util/inc/ttime.h
+++ b/src/util/inc/ttime.h
@@ -20,9 +20,8 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
-#include <time.h>
-#include "tutil.h"
+#include "os.h"
+#include "taosdef.h"
 
 //@return timestamp in second
 int32_t taosGetTimestampSec();
diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h
index 34f35c3807..e312513ff3 100644
--- a/src/util/inc/tutil.h
+++ b/src/util/inc/tutil.h
@@ -25,150 +25,24 @@ extern "C" {
 #include "tcrc32c.h"
 #include "taosdef.h"
 
-#ifndef STDERR_FILENO
-#define STDERR_FILENO (2)
-#endif
-
-#define FD_VALID(x) ((x) > STDERR_FILENO)
-#define FD_INITIALIZER  ((int32_t)-1)
-
-#define WCHAR wchar_t
-
-#define tfree(x)         \
-  do {                   \
-    if (x) {             \
-      free((void *)(x)); \
-      x = 0;             \
-    }                    \
-  } while(0);
-
-#define tstrncpy(dst, src, size)   \
-  do {                             \
-    strncpy((dst), (src), (size)); \
-    (dst)[(size)-1] = 0;           \
-  } while (0);
-
-#define tclose(x) taosCloseSocket(x)
-
-// Pointer p drift right by b bytes
-#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
-#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
-
-#ifndef NDEBUG
-#define ASSERT(x) assert(x)
-#else
-#define ASSERT(x)
-#endif
-
-#ifdef UNUSED
-#undefine UNUSED
-#endif
-#define UNUSED(x) ((void)(x))
-
-#ifdef UNUSED_FUNC
-#undefine UNUSED_FUNC
-#endif
-
-#ifdef UNUSED_PARAM
-#undef UNUSED_PARAM
-#endif
-
-#if defined(__GNUC__)
-#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
-#define UNUSED_FUNC __attribute__((unused))
-#else
-#define UNUSED_PARAM(x) x
-#define UNUSED_FUNC
-#endif
-
-#ifdef tListLen
-#undefine tListLen
-#endif
-#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
-
-#if defined(__GNUC__)
-#define FORCE_INLINE inline __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
-#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
-  
-#define DEFAULT_COMP(x, y)       \
-  do {                           \
-    if ((x) == (y)) {            \
-      return 0;                  \
-    } else {                     \
-      return (x) < (y) ? -1 : 1; \
-    }                            \
-  } while (0)
-
-#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
-
-// align to 8bytes
-#define ALIGN8(n) ALIGN_NUM(n, 8)
-
-#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
-#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
-#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
-#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
-#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
-#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
-
-#define POW2(x) ((x) * (x))
-
-uint32_t taosRand(void);
-
-uint32_t trand(void);
-
-size_t twcslen(const wchar_t *wcs);
-
 int32_t strdequote(char *src);
-
-size_t strtrim(char *src);
-
-char *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
-
-char **strsplit(char *src, const char *delim, int32_t *num);
-
-char* strtolower(char *dst, const char *src);
-
-char* strntolower(char *dst, const char *src, int32_t n);
-
+size_t  strtrim(char *src);
+char *  strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char ** strsplit(char *src, const char *delim, int32_t *num);
+char *  strtolower(char *dst, const char *src);
+char *  strntolower(char *dst, const char *src, int32_t n);
 int64_t strnatoi(char *num, int32_t len);
-
-//char* strreplace(const char* str, const char* pattern, const char* rep);
-
-char *strbetween(char *string, char *begin, char *end);
-
-char *paGetToken(char *src, char **token, int32_t *tokenLen);
-
-void taosMsleep(int32_t mseconds);
+char *  strbetween(char *string, char *begin, char *end);
+char *  paGetToken(char *src, char **token, int32_t *tokenLen);
 
 int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
-
 int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
 
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
-/**
- *
- * @param fileNamePattern
- * @param dstPath
- */
-void getTmpfilePath(const char *fileNamePattern, char *dstPath);
-
-bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t* len);
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes);
-
-void taosRandStr(char* str, int32_t size);
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-
-bool taosValidateEncodec(const char *encodec);
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer);
+int  taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
+
+char *   taosIpStr(uint32_t ipInt);
+uint32_t ip2uint(const char *const ip_addr);
 
 static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, char *target) {
   MD5_CTX context;
@@ -178,53 +52,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha
   memcpy(target, context.digest, TSDB_KEY_LEN);
 }
 
-int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
-
-char *taosIpStr(uint32_t ipInt);
-
-uint32_t ip2uint(const char *const ip_addr);
-
-void taosRemoveDir(char *rootDir);
-int  tmkdir(const char *pathname, mode_t mode); 
-void taosMvDir(char* destDir, char *srcDir);
-
-#define TAOS_ALLOC_MODE_DEFAULT 0
-#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
-#define TAOS_ALLOC_MODE_DETECT_LEAK 2
-void taosSetAllocMode(int mode, const char* path, bool autoDump);
-void taosDumpMemoryLeak();
-
-void * tmalloc(size_t size);
-void * tcalloc(size_t nmemb, size_t size);
-size_t tsizeof(void *ptr);
-void   tmemset(void *ptr, int c);
-void * trealloc(void *ptr, size_t size);
-void   tzfree(void *ptr);
-
-#ifdef TAOS_MEM_CHECK
-
-void *  taos_malloc(size_t size, const char *file, uint32_t line);
-void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
-void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
-void    taos_free(void *ptr, const char *file, uint32_t line);
-char *  taos_strdup(const char *str, const char *file, uint32_t line);
-char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
-ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
-
-#ifndef TAOS_MEM_CHECK_IMPL
-
-#define malloc(size) taos_malloc(size, __FILE__, __LINE__)
-#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
-#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
-#define free(ptr) taos_free(ptr, __FILE__, __LINE__)
-#define strdup(str) taos_strdup(str, __FILE__, __LINE__)
-#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
-#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
-
-#endif  // TAOS_MEM_CHECK_IMPL
-
-#endif // TAOS_MEM_CHECK
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 5dff6286f0..695529d02f 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -14,10 +14,9 @@
  */
 
 #include "os.h"
-
 #include "hash.h"
 #include "tulog.h"
-#include "tutil.h"
+#include "taosdef.h"
 
 static FORCE_INLINE void __wr_lock(void *lock) {
   if (lock == NULL) {
diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c
index 4b37cbd2d5..4af435798c 100644
--- a/src/util/src/tbuffer.c
+++ b/src/util/src/tbuffer.c
@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "os.h"
 #include <stdlib.h>
 #include <memory.h>
 #include <assert.h>
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 92d4b2caac..dc9128d4a9 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -259,7 +259,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
     return NULL;
   }
 
-  pthread_attr_t thattr = {{0}};
+  pthread_attr_t thattr;
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
@@ -288,7 +288,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
       uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64
              "bytes size:%" PRId64 "bytes",
              pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime,
-             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, dataSize);
+             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
     } else {
       uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
     }
@@ -343,7 +343,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke
   SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
   if (ptNode != NULL) {
      T_REF_INC(*ptNode);
-    (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan;
+    (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan;
   }
 
   __cache_unlock(pCacheObj);
@@ -381,7 +381,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
 }
 
 void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
-  if (pCacheObj == NULL || data == NULL) return NULL;
+  if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
   
   size_t          offset = offsetof(SCacheDataNode, data);
   SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset);
@@ -419,7 +419,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   // note: extend lifespan before dec ref count
   bool inTrashCan = pNode->inTrashCan;
 
-  if (pCacheObj->extendLifespan && (!inTrashCan)) {
+  if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) {
     atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs());
     uDebug("cache:%s data:%p extend life time to %"PRId64 "  before release", pCacheObj->name, pNode->data, pNode->expireTime);
   }
@@ -457,8 +457,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   } else {
     // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
     int32_t ref = T_REF_DEC(pNode);
-    uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, ref,
-        inTrashCan);
+
+    uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data,
+           ref, inTrashCan);
   }
 }
 
@@ -572,6 +573,7 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
   free(pElem);
 }
 
+// TODO add another lock when scanning trashcan
 void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
   __cache_wr_lock(pCacheObj);
 
@@ -643,6 +645,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t
   __cache_wr_lock(pCacheObj);
   while (taosHashIterNext(pIter)) {
     SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
+
     if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) {
       taosCacheReleaseNode(pCacheObj, pNode);
       continue;
@@ -674,6 +677,7 @@ void* taosCacheTimedRefresh(void *handle) {
 
     // check if current cache object will be deleted every 500ms.
     if (pCacheObj->deleting) {
+      uDebug("%s refresh threads quit", pCacheObj->name);
       break;
     }
 
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index 889d38ff20..9564588254 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -1,6 +1,6 @@
 #include "taosdef.h"
 #include "tcompare.h"
-#include <tarray.h>
+#include "tarray.h"
 #include "tutil.h"
 
 int32_t compareInt32Val(const void *pLeft, const void *pRight) {
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index d561e8ba5f..59bc09e877 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -118,7 +118,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
       
       wordfree(&full_path);
 
-      int code = tmkdir(option, 0755);
+      int code = taosMkDir(option, 0755);
       if (code != 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         uError("config option:%s, input value:%s, directory not exist, create fail:%s",
diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c
index 0704285da5..d7bf9d7857 100644
--- a/src/util/src/tkvstore.c
+++ b/src/util/src/tkvstore.c
@@ -12,22 +12,15 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <errno.h>
-#include <fcntl.h>
-#include <libgen.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 
-#include "hash.h"
+#define _DEFAULT_SOURCE
 #include "os.h"
+#include "hash.h"
 #include "taoserror.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tkvstore.h"
 #include "tulog.h"
-#include "tfile.h"
 
 #define TD_KVSTORE_HEADER_SIZE 512
 #define TD_KVSTORE_MAJOR_VERSION 1
@@ -182,7 +175,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
     goto _err;
   }
 
-  if (tsendfile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -242,13 +235,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
   ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
   ASSERT(tlen == sizeof(SKVRecord));
 
-  if (twrite(pStore->fd, buf, tlen) < tlen) {
+  if (taosTWrite(pStore->fd, buf, tlen) < tlen) {
     uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  if (twrite(pStore->fd, cont, contLen) < contLen) {
+  if (taosTWrite(pStore->fd, cont, contLen) < contLen) {
     uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
     return -1;
   }
@@ -286,8 +279,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
   void *pBuf = buf;
   tdEncodeKVRecord(&pBuf, &rInfo);
 
-  if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
-    uError("failed to write %" PRId64 " bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
+  if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
+    uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -335,7 +328,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
     return -1;
   }
 
-  if (tread(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -366,7 +359,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
   ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
-  if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -499,7 +492,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
   ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
 
   while (true) {
-    ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord));
+    ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord));
     if (tsize == 0) break;
     if (tsize < sizeof(SKVRecord)) {
       uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
@@ -562,7 +555,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
       goto _err;
     }
 
-    if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) {
+    if (taosTRead(pStore->fd, buf, pRecord->size) < pRecord->size) {
       uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
              strerror(errno), pRecord->offset);
       terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c
index f402c2307e..8c2ad83de1 100644
--- a/src/util/src/tlist.c
+++ b/src/util/src/tlist.c
@@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
   SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
   if (node == NULL) return -1;
 
+  node->next = node->prev = NULL;
   memcpy((void *)(node->data), data, list->eleSize);
   tdListPrependNode(list, node);
 
@@ -121,6 +122,22 @@ SListNode *tdListPopTail(SList *list) {
   return node;
 }
 
+SListNode *tdListGetHead(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->head;
+}
+
+SListNode *tsListGetTail(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->tail;
+}
+
 SListNode *tdListPopNode(SList *list, SListNode *node) {
   if (list->head == node) {
     list->head = node->next;
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index c2c56f29db..2489ae56be 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -301,11 +301,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   lseek(tsLogObj.logHandle->fd, 0, SEEK_END);
 
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "                new log file                      \n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
 
   return 0;
 }
@@ -355,7 +355,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
 
     if (tsLogObj.maxLines > 0) {
@@ -365,7 +365,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 void taosDumpData(unsigned char *msg, int32_t len) {
@@ -384,7 +384,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
     pos += 3;
     if (c >= 16) {
       temp[pos++] = '\n';
-      twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+      taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
       c = 0;
       pos = 0;
     }
@@ -392,7 +392,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
 
   temp[pos++] = '\n';
 
-  twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+  taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
 
   return;
 }
@@ -432,7 +432,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
     
     if (tsLogObj.maxLines > 0) {
@@ -442,7 +442,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 #if 0
@@ -571,7 +571,7 @@ static void *taosAsyncOutputLog(void *param) {
     while (1) {
       log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT);
       if (log_size) {
-        twrite(tLogBuff->fd, tempBuffer, log_size);
+        taosTWrite(tLogBuff->fd, tempBuffer, log_size);
         LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff);
       } else {
         break;
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 12a7fc2b9b..fdf11eb5c4 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -264,7 +264,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
     buffer[len] = 0;
 
     if (pNote->taosNoteFd >= 0)  {
-        twrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+        taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
 
         if (pNote->taosNoteMaxLines > 0) {
             pNote->taosNoteLines++;
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 5de61a3d57..6f4d2b8afd 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -164,7 +164,7 @@ int taosNonblockwrite(int fd, char *ptr, int nbytes) {
       return -1;
     }
 
-    nwritten = (int)send(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+    nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
     if (nwritten <= 0) {
       if (errno == EAGAIN || errno == EINTR) continue;
 
@@ -304,10 +304,10 @@ int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientI
     //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
     close(sockFd);
     sockFd = -1;
+  } else {
+    taosKeepTcpAlive(sockFd);
   }
 
-  // taosKeepTcpAlive(sockFd);
-
   return sockFd;
 }
 
@@ -349,7 +349,7 @@ int taosKeepTcpAlive(int sockFd) {
 
   struct linger linger = {0};
   linger.l_onoff = 1;
-  //linger.l_linger = 0;
+  linger.l_linger = 3;
   if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
     uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
     close(sockFd);
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 7b73b1e17c..eaa60d739b 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -14,11 +14,6 @@
  */
 
 #include "os.h"
-
-#ifdef USE_LIBICONV
-#include "iconv.h"
-#endif
-
 #include "tcrc32c.h"
 #include "tglobal.h"
 #include "ttime.h"
@@ -27,76 +22,6 @@
 #include "tulog.h"
 #include "taoserror.h"
 
-
-#ifdef WINDOWS
-uint32_t taosRand(void)
-{
-  return rand();
-}
-#else
-uint32_t taosRand(void)
-{
-/*
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-*/
-  return rand();
-}
-
-uint32_t trand(void)
-{
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-}
-#endif
-
-size_t twcslen(const wchar_t *wcs) {
-#ifdef WINDOWS
-  int *wstr = (int *)wcs;
-  if (NULL == wstr) {
-    return 0;
-  }
-
-  size_t n = 0;
-  while (1) {
-    if (0 == *wstr++) {
-      break;
-    }
-    n++;
-  }
-
-  return n;
-#else
-  return wcslen(wcs);
-#endif
-}
-
 int32_t strdequote(char *z) {
   if (z == NULL) {
     return 0;
@@ -234,6 +159,7 @@ char* strtolower(char *dst, const char *src) {
   *p = 0;
   return dst;
 }
+
 char* strntolower(char *dst, const char *src, int32_t n) {
   int esc = 0;
   char quote = 0, *p = dst, c;
@@ -343,96 +269,6 @@ int64_t strnatoi(char *num, int32_t len) {
   return ret;
 }
 
-#if 0
-FORCE_INLINE size_t getLen(size_t old, size_t size) {
-  if (old == 1) {
-    old = 2;
-  }
-
-  while (old < size) {
-    old = (old * 1.5);
-  }
-
-  return old;
-}
-
-static char *ensureSpace(char *dest, size_t *curSize, size_t size) {
-  if (*curSize < size) {
-    *curSize = getLen(*curSize, size);
-
-    char *tmp = realloc(dest, *curSize);
-    if (tmp == NULL) {
-      free(dest);
-      return NULL;
-    }
-
-    return tmp;
-  }
-
-  return dest;
-}
-
-char *strreplace(const char *str, const char *pattern, const char *rep) {
-  if (str == NULL || pattern == NULL || rep == NULL) {
-    return NULL;
-  }
-
-  const char *s = str;
-
-  size_t oldLen = strlen(str);
-  size_t newLen = oldLen;
-
-  size_t repLen = strlen(rep);
-  size_t patternLen = strlen(pattern);
-
-  char *dest = calloc(1, oldLen + 1);
-  if (dest == NULL) {
-    return NULL;
-  }
-
-  if (patternLen == 0) {
-    return strcpy(dest, str);
-  }
-
-  int32_t start = 0;
-
-  while (1) {
-    char *p = strstr(str, pattern);
-    if (p == NULL) {  // remain does not contain pattern
-      size_t remain = (oldLen - (str - s));
-      size_t size = remain + start + 1;
-
-      dest = ensureSpace(dest, &newLen, size);
-      if (dest == NULL) {
-        return NULL;
-      }
-
-      strcpy(dest + start, str);
-      dest[start + remain] = 0;
-      break;
-    }
-
-    size_t len = p - str;
-    size_t size = start + len + repLen + 1;
-
-    dest = ensureSpace(dest, &newLen, size);
-    if (dest == NULL) {
-      return NULL;
-    }
-
-    memcpy(dest + start, str, len);
-
-    str += (len + patternLen);
-    start += len;
-
-    memcpy(dest + start, rep, repLen);
-    start += repLen;
-  }
-
-  return dest;
-}
-#endif
-
 char *strbetween(char *string, char *begin, char *end) {
   char *result = NULL;
   char *_begin = strstr(string, begin);
@@ -491,170 +327,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
   return 0;
 }
 
-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
-  int32_t ts = taosGetTimestampSec();
-
-  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
-
-  char *delimiterPos = strrchr(fullPath, delimiter);
-  if (delimiterPos == NULL) return -1;
-
-  int32_t fileNameLen = 0;
-  if (suffix)
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
-  else
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
-
-  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
-  if (*dstPath == NULL) {
-    *dstPath = calloc(1, len + 1);
-    if (*dstPath == NULL) return -1;
-  }
-
-  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
-  strncat(*dstPath, fname, (size_t)fileNameLen);
-  (*dstPath)[len] = 0;
-
-  return rename(fullPath, *dstPath);
-}
-
-void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
-  const char* tdengineTmpFileNamePrefix = "tdengine-";
-  
-  char tmpPath[PATH_MAX];
-
-#ifdef WINDOWS
-  char *tmpDir = getenv("tmp");
-  if (tmpDir == NULL) {
-    tmpDir = "";
-  }
-#else
-  char *tmpDir = "/tmp/";
-#endif
-  
-  strcpy(tmpPath, tmpDir);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
-  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
-    strcat(tmpPath, fileNamePrefix);
-    strcat(tmpPath, "-%d-%s");
-  }
-  
-  char rand[8] = {0};
-  taosRandStr(rand, tListLen(rand) - 1);
-  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
-}
-
-void taosRandStr(char* str, int32_t size) {
-  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
-  int32_t len = 39;
-
-  for (int32_t i = 0; i < size; ++i) {
-    str[i] = set[taosRand() % len];
-  }
-}
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
-#if defined WINDOWS
-  for (int i = 0; i < bytes; ++i) {
-    int32_t f1 = *(int32_t*)((char*)f1_ucs4 + i * 4);
-    int32_t f2 = *(int32_t*)((char*)f2_ucs4 + i * 4);
-
-    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
-      return f1 - f2;
-    }
-    else if (f1 == 0 && f2 == 0) {
-      return 0;
-    }
-
-    if (f1 != f2) {
-      return f1 - f2;
-    }
-  }
-  return 0;
-
-#if 0
-  int32_t ucs4_max_len = bytes + 4;
-  char *f1_mbs = calloc(bytes, 1);
-  char *f2_mbs = calloc(bytes, 1);
-  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
-    return -1;
-  }
-  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
-    return -1;
-  }
-  int32_t ret = strcmp(f1_mbs, f2_mbs);
-  free(f1_mbs);
-  free(f2_mbs);
-  return ret;
-#endif
-
-#else
-  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
-#endif
-}
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
-  size_t ucs4_input_len = ucs4_max_len;
-  size_t outLen = ucs4_max_len;
-  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
-    iconv_close(cd);
-    return -1;
-  }
-  iconv_close(cd);
-  return (int32_t)(ucs4_max_len - outLen);
-#else
-  mbstate_t state = {0};
-  int32_t len = (int32_t) wcsnrtombs(NULL, (const wchar_t **) &ucs4, ucs4_max_len / 4, 0, &state);
-  if (len < 0) {
-    return -1;
-  }
-  memset(&state, 0, sizeof(state));
-  len = wcsnrtombs(mbs, (const wchar_t **) &ucs4, ucs4_max_len / 4, (size_t) len, &state);
-  if (len < 0) {
-    return -1;
-  }
-  return len;
-#endif
-}
-
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t* len) {
-  memset(ucs4, 0, ucs4_max_len);
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
-  size_t ucs4_input_len = mbsLength;
-  size_t outLeft = ucs4_max_len;
-  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
-    iconv_close(cd);
-    return false;
-  }
-  
-  iconv_close(cd);
-  if (len != NULL) {
-    *len = ucs4_max_len - outLeft;
-  }
-  
-  return true;
-#else
-  mbstate_t state = {0};
-  int32_t len = mbsnrtowcs((wchar_t *) ucs4, (const char **) &mbs, mbsLength, ucs4_max_len / 4, &state);
-  return len >= 0;
-#endif
-}
-
-bool taosValidateEncodec(const char *encodec) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
-  if (cd == (iconv_t)(-1)) {
-    return false;
-  }
-  iconv_close(cd);
-#endif
-  return true;
-}
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
   if (versionStr == NULL || versionNubmer == NULL) {
     return false;
@@ -734,112 +406,3 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) {
   *(int64_t*)(&dv) = *(int64_t*)pBuf;
   return dv; 
 }
-
-typedef struct CharsetPair {
-  char *oldCharset;
-  char *newCharset;
-} CharsetPair;
-
-char *taosCharsetReplace(char *charsetstr) {
-  CharsetPair charsetRep[] = {
-      { "utf8", "UTF-8" }, { "936", "CP936" },
-  };
-
-  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
-    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
-      return strdup(charsetRep[i].newCharset);
-    }
-  }
-
-  return strdup(charsetstr);
-}
-
-void *tmalloc(size_t size) {
-  if (size <= 0) return NULL;
-
-  void *ret = malloc(size + sizeof(size_t));
-  if (ret == NULL) return NULL;
-
-  *(size_t *)ret = size;
-
-  return (void *)((char *)ret + sizeof(size_t));
-}
-
-void *tcalloc(size_t nmemb, size_t size) {
-  size_t tsize = nmemb * size;
-  void * ret = tmalloc(tsize);
-  if (ret == NULL) return NULL;
-
-  tmemset(ret, 0);
-  return ret;
-}
-
-size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
-
-void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
-
-void * trealloc(void *ptr, size_t size) {
-  if (ptr == NULL) return tmalloc(size);
-
-  if (size <= tsizeof(ptr)) return ptr;
-
-  void * tptr = (void *)((char *)ptr - sizeof(size_t));
-  size_t tsize = size + sizeof(size_t);
-  tptr = realloc(tptr, tsize);
-  if (tptr == NULL) return NULL;
-
-  *(size_t *)tptr = size;
-
-  return (void *)((char *)tptr + sizeof(size_t));
-}
-
-void tzfree(void *ptr) {
-  if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
-  }
-}
-
-void taosRemoveDir(char *rootDir) {
-  DIR *dir = opendir(rootDir);
-  if (dir == NULL) return;
-
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-     
-    char filename[1024];
-    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
-    if (de->d_type & DT_DIR) {
-      taosRemoveDir(filename);
-    } else {
-      (void)remove(filename);
-      uInfo("file:%s is removed", filename);
-    }
-  }
-
-  closedir(dir);
-  rmdir(rootDir);
-
-  uInfo("dir:%s is removed", rootDir);
-}
-
-int tmkdir(const char *path, mode_t mode) {
-  int code = mkdir(path, 0755);
-  if (code < 0 && errno == EEXIST) code = 0;
-  return code;
-}
-
-void taosMvDir(char* destDir, char *srcDir) {
-  if (0 == tsEnableVnodeBak) {
-    uInfo("vnode backup not enabled");
-    return;
-  }
-
-  char shellCmd[1024+1] = {0}; 
-  
-  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
-  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
-  tSystem(shellCmd);
-  uInfo("shell cmd:%s is executed", shellCmd);
-}
-
diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt
index a1c56b32b5..9b6b03209e 100644
--- a/src/vnode/CMakeLists.txt
+++ b/src/vnode/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index 74cfbf1e73..38f7c8e605 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -37,6 +37,7 @@ extern int32_t vDebugFlag;
 typedef struct {
   int32_t      vgId;      // global vnode group ID
   int32_t      refCount;  // reference count
+  int32_t      delay;
   int8_t       status; 
   int8_t       role;   
   int8_t       accessState;
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index dd5ec18439..8a56a2c500 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -44,17 +44,9 @@ static int      vnodeProcessTsdbStatus(void *arg, int status);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion);
 static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
+static void     vnodeCtrlFlow(void *handle, int32_t mseconds); 
 static int      vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
 
-#ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void    syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
-#endif
-
 int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
@@ -286,17 +278,14 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.writeToCache = vnodeWriteToQueue;
   syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp; 
   syncInfo.notifyRole = vnodeNotifyRole;
+  syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
   pVnode->sync = syncStart(&syncInfo);
 
-#ifndef _SYNC
-  pVnode->role = TAOS_SYNC_ROLE_MASTER;
-#else
   if (pVnode->sync == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
   }
-#endif
 
   pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId);
   pVnode->events = NULL;
@@ -430,10 +419,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
   if (pVnode->status != TAOS_VN_STATUS_READY) return;
   if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
 
-  // still need report status when unsynced
-  if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) {
-  } else if (pVnode->tsdb == NULL) {
-  } else {
+  if (pVnode->tsdb) {
     tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
   }
 
@@ -562,6 +548,13 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
     cqStop(pVnode->cq);
 }
 
+static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) {
+  SVnodeObj *pVnode = ahandle;
+  if (pVnode->delay != mseconds) 
+    vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds);
+  pVnode->delay = mseconds;
+}
+
 static int vnodeResetTsdb(SVnodeObj *pVnode)
 {
   char rootDir[128] = "\0";
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 973df7c5a1..066770e1bb 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -71,11 +71,45 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) {
   pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
   pRead->pCont = qhandle;
   pRead->contLen = 0;
+  pRead->rpcMsg.handle = NULL;
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
   taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }
 
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) {
+  bool continueExec = false;
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) {
+    if (continueExec) {
+      vDebug("QInfo:%p add to query task queue for exec", handle);
+      vnodePutItemIntoReadQueue(pVnode, handle);
+      pRet->qhandle = handle;
+      *freeHandle = false;
+    } else {
+      vDebug("QInfo:%p exec completed", handle);
+      *freeHandle = true;
+    }
+  } else {
+    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+    *freeHandle = true;
+  }
+
+  return code;
+}
+
+static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+  pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+  pRet->len = sizeof(SRetrieveTableRsp);
+
+  memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+  SRetrieveTableRsp* pRsp = pRet->rsp;
+
+  pRsp->completed = true;
+}
+
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   void    *pCont = pReadMsg->pCont;
   int32_t  contLen = pReadMsg->contLen;
@@ -98,6 +132,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
       vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     } else {
       assert(*qhandle == (void*) killQueryMsg->qhandle);
+
       qKillQuery(*qhandle);
       qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
     }
@@ -110,7 +145,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
-    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo);
+    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
 
     SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code    = code;
@@ -133,7 +168,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
         pRsp->qhandle = htobe64((uint64_t) pQInfo);
       }
 
-      pQInfo = NULL;
       if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
         vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
@@ -153,16 +187,34 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   } else {
     assert(pCont != NULL);
+
     handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont);
     if (handle == NULL) {
       vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle);
       code = TSDB_CODE_QRY_INVALID_QHANDLE;
     } else {
       vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont);
-      code = TSDB_CODE_VND_ACTION_IN_PROGRESS;
-      qTableQuery(*handle); // do execute query
+
+      bool freehandle = false;
+      bool buildRes = qTableQuery(*handle); // do execute query
+
+      // build query rsp
+      if (buildRes) {
+        // update the connection info according to the retrieve connection
+        pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle);
+        assert(pReadMsg->rpcMsg.handle != NULL);
+
+        vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+        code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle);
+
+        // todo test the error code case
+        if (code == TSDB_CODE_SUCCESS) {
+          code = TSDB_CODE_QRY_HAS_RSP;
+        }
+      }
+
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle);
     }
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
   }
 
   return code;
@@ -176,7 +228,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
   pRetrieve->free = htons(pRetrieve->free);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
@@ -185,16 +237,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
     vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
-
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->useconds  = 0;
-    pRsp->completed = true;
-
+    
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
@@ -203,35 +247,25 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
 
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->completed = true;
-    pRsp->useconds  = 0;
-
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
   bool freeHandle = true;
-  code = qRetrieveQueryResultInfo(*handle);
+  bool buildRes   = false;
+
+  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
   if (code != TSDB_CODE_SUCCESS) {
     //TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  } else { // if failed to dump result, free qhandle immediately
-    if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) {
-      if (qHasMoreResultsToRetrieve(*handle)) {
-        vnodePutItemIntoReadQueue(pVnode, *handle);
-        pRet->qhandle = *handle;
-        freeHandle = false;
-      } else {
-        qKillQuery(*handle);
-        freeHandle = true;
-      }
+  } else { // result is not ready, return immediately
+    if (!buildRes) {
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      return TSDB_CODE_QRY_NOT_READY;
     }
+
+    code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle);
   }
 
   qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle);
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 6b9b8ca4fd..900ff1fbba 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -78,6 +78,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     // assign version
     pVnode->version++;
     pHead->version = pVnode->version;
+    if (pVnode->delay) usleep(pVnode->delay*1000);
+
   } else { // from wal or forward 
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt
index 684616ba0e..a9c20f643a 100644
--- a/src/wal/CMakeLists.txt
+++ b/src/wal/CMakeLists.txt
@@ -1,9 +1,9 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(inc)
 
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC)
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
index a0e2ccb51d..bf3b0f63ea 100644
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@@ -13,14 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h> 
-
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tlog.h"
 #include "tchecksum.h"
@@ -29,7 +22,6 @@
 #include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
-#include "tfile.h"
 
 #define walPrefix "wal"
 
@@ -109,7 +101,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     }
   }
 
-  if (tmkdir(path, 0755) != 0) {
+  if (taosMkDir(path, 0755) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
     walRelease(pWal);
@@ -214,7 +206,7 @@ int walWrite(void *handle, SWalHead *pHead) {
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(twrite(pWal->fd, pHead, contLen) != contLen) {
+  if(taosTWrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -373,7 +365,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   wDebug("wal:%s, start to restore", name);
 
   while (1) {
-    int ret = tread(fd, pHead, sizeof(SWalHead));
+    int ret = taosTRead(fd, pHead, sizeof(SWalHead));
     if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
@@ -388,7 +380,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
       break;
     } 
 
-    ret = tread(fd, pHead->cont, pHead->len);
+    ret = taosTRead(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -428,7 +420,7 @@ int walHandleExistingFiles(const char *path) {
       if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
         snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
         snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
-        if (tmkdir(opath, 0755) != 0) {
+        if (taosMkDir(opath, 0755) != 0) {
           wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
           terrno = TAOS_SYSTEM_ERROR(errno);
           break; 
diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt
index f007bc3882..be0b8a948d 100644
--- a/src/wal/test/CMakeLists.txt
+++ b/src/wal/test/CMakeLists.txt
@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(../inc)
diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
index 4e307db079..52067343ef 100644
--- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
+++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
@@ -94,7 +94,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>24.1.1</version>
+      <version>29.0-jre</version>
     </dependency>
 
     <dependency>
diff --git a/tests/examples/go/src/taosapp/taosapp.go b/tests/examples/go/src/taosapp/taosapp.go
index 5c7b92f6fb..30126ea571 100644
--- a/tests/examples/go/src/taosapp/taosapp.go
+++ b/tests/examples/go/src/taosapp/taosapp.go
@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 package main
- 
+
 import (
 	"database/sql"
     "time"
     "log"
     "fmt"
-	_ "taosSql"
+	_ "github.com/taosdata/driver-go/taosSql"
 )
 
 func main() {
@@ -29,7 +29,7 @@ func main() {
 
     fmt.Printf("\n======== start demo test ========\n")
     // open connect to taos server
-    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/demodb")
+    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/")
     if err != nil {
         log.Fatalf("Open database error: %s\n", err)
     }
@@ -58,11 +58,11 @@ func main() {
 
 func drop_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
-    res, err := db.Exec("drop database " + demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database if exists " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -73,10 +73,10 @@ func create_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // create database
     res, err := db.Exec("create database " + demodb)
-    checkErr(err)
+    checkErr(err, "create db, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -89,10 +89,10 @@ func use_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // use database
     res, err := db.Exec("use " +  demodb)  // notes: must no quote to db name
-    checkErr(err)
+    checkErr(err, "use db db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -103,10 +103,10 @@ func create_table(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
     // create table
     res, err := db.Exec("create table " + demot + " (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)")
-    checkErr(err)
+    checkErr(err, "create table db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -120,10 +120,10 @@ func insert_data(db *sql.DB, demot string) {
                                       " (now+1s, 101, 'shanghai', 11, true, 'two', 789.123, 789.123)" +
                                       " (now+2s, 102, 'shenzhen', 12,  false, 'three', 456.789, 456.789)")
 
-    checkErr(err)
+    checkErr(err, "insert data, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "insert data res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -133,7 +133,7 @@ func select_data(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     rows, err := db.Query("select * from ? " , demot)  // go text mode
-    checkErr(err)
+    checkErr(err, "select db.Query")
 
     fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %4s%s %5s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
@@ -148,7 +148,7 @@ func select_data(db *sql.DB, demot string) {
         var dv float64
 
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
-        checkErr(err)
+        checkErr(err, "select rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -170,15 +170,11 @@ func select_data(db *sql.DB, demot string) {
 func drop_database_stmt(db *sql.DB,demodb string) {
     st := time.Now().Nanosecond()
     // drop test db
-    stmt, err := db.Prepare("drop database ?")
-    checkErr(err)
-    defer stmt.Close()
-
-    res, err := stmt.Exec(demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -189,15 +185,15 @@ func create_database_stmt(db *sql.DB,demodb string)  {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("create database ?")
-    checkErr(err)
+    checkErr(err, "create db, db.Prepare")
 
     //var res driver.Result
     res, err := stmt.Exec(demodb)
-    checkErr(err)
+    checkErr(err, "create db, stmt.Exec")
 
     //fmt.Printf("Query OK, %d row(s) affected()", res.RowsAffected())
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -208,13 +204,13 @@ func use_database_stmt (db *sql.DB,demodb string) {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("use " + demodb)
-    checkErr(err)
+    checkErr(err, "use db, db.Prepare")
 
     res, err := stmt.Exec()
-    checkErr(err)
+    checkErr(err, "use db, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -224,14 +220,14 @@ func create_table_stmt (db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // create table
     // (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)
-    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(8), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
-    checkErr(err)
+    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(10), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
+    checkErr(err, "create table db.Prepare")
 
     res, err := stmt.Exec(demot, "ts", "id", "name", "len", "flag", "notes", "fv", "dv")
-    checkErr(err)
+    checkErr(err, "create table stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -241,15 +237,15 @@ func insert_data_stmt(db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // insert data into table
     stmt, err := db.Prepare("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?)")
-    checkErr(err)
+    checkErr(err, "insert db.Prepare")
 
     res, err := stmt.Exec(demot, "now" ,    1000, "'haidian'" ,    6, true, "'AI world'",  6987.654, 321.987,
                                  "now+1s", 1001, "'changyang'" ,  7, false,  "'DeepMode'", 12356.456, 128634.456,
                                  "now+2s", 1002, "'chuangping'" , 8, true, "'database'",  3879.456, 65433478.456,)
-    checkErr(err)
+    checkErr(err, "insert data, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -259,12 +255,12 @@ func select_data_stmt(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     stmt, err := db.Prepare("select ?, ?, ?, ?, ?, ?, ?, ? from ?" )  // go binary mode
-    checkErr(err)
+    checkErr(err, "db.Prepare")
 
     rows, err := stmt.Query("ts", "id","name","len", "flag","notes", "fv", "dv", demot)
-    checkErr(err)
+    checkErr(err, "stmt.Query")
 
-    fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
+    fmt.Printf("%10s%s%8s %5s %8s%s %s %10s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
     for rows.Next() {
         var ts string
@@ -279,7 +275,7 @@ func select_data_stmt(db *sql.DB, demot string) {
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
         //fmt.Println("start scan fields from row.rs, &fv:", &fv)
         //err = rows.Scan(&fv)
-        checkErr(err)
+        checkErr(err, "rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -298,8 +294,9 @@ func select_data_stmt(db *sql.DB, demot string) {
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
 }
 
-func checkErr(err error) {
+func checkErr(err error, prompt string) {
     if err != nil {
+	fmt.Printf("%s\n", prompt)
         panic(err)
     }
 }
diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py
index 73c4b95214..dd1475ec14 100644
--- a/tests/examples/python/read_example.py
+++ b/tests/examples/python/read_example.py
@@ -26,7 +26,6 @@ if __name__ == '__main__':
 
     # Generate a cursor object to run SQL commands
     c1 = conn.cursor()
-
     # Create a database named db
     try:
         c1.execute('create database if not exists db ')
@@ -50,9 +49,11 @@ if __name__ == '__main__':
         raise(err)
 
     # insert data 
-    for i in range(10000):
+    for i in range(10):
         try:
-           c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello'))
+           value = c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello'))
+           #if insert, value is the affected rows
+           print(value)
         except Exception as err:
             conn.close()
             raise(err)
@@ -70,6 +71,11 @@ if __name__ == '__main__':
     # Use fetchall to fetch data in a list
     data = c1.fetchall()
 
+    for col in data:
+        print(col)
+
+    print('Another query method ')
+
     try:
         c1.execute('select * from db.t')
     except Exception as err:
diff --git a/tests/perftest-scripts/cassandraTestQ1Loop.sh b/tests/perftest-scripts/cassandraTestQ1Loop.sh
new file mode 100755
index 0000000000..e65ec2575c
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ1Loop.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ1OUT=cassandraTestQ1.out
+
+function runTest {
+  totalG0=0
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q1.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q1.txt \
+      > $CASTESTQ1OUT
+    G0=`grep "devgroup=0" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG0=`echo "scale=4; $totalG0 + $G0" | bc`
+    G10=`grep "devgroup=10" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup=20" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup=30" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup=40" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup=50" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup=60" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup=70" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup=80" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup=90" $CASTESTQ1OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+  done
+  avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90"
+  echo "Cassandra, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90"
+}
+
+################ Main ################
+
+master=false
+develop=true
+verbose=false
+
+clients=1
+
+while : ; do
+  case $1 in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      break ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ2Loop.sh b/tests/perftest-scripts/cassandraTestQ2Loop.sh
new file mode 100755
index 0000000000..76e42cf89d
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ2Loop.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ2OUT=cassandraTestQ2.out
+
+function runTest {
+  totalCount10=0
+  totalCount20=0
+  totalCount30=0
+  totalCount40=0
+  totalCount50=0
+  totalCount60=0
+  totalCount70=0
+  totalCount80=0
+  totalCount90=0
+  totalCount100=0
+
+  totalAvg10=0
+  totalAvg20=0
+  totalAvg30=0
+  totalAvg40=0
+  totalAvg50=0
+  totalAvg60=0
+  totalAvg70=0
+  totalAvg80=0
+  totalAvg90=0
+  totalAvg100=0
+
+  totalSum10=0
+  totalSum20=0
+  totalSum30=0
+  totalSum40=0
+  totalSum50=0
+  totalSum60=0
+  totalSum70=0
+  totalSum80=0
+  totalSum90=0
+  totalSum100=0
+
+  totalMax10=0
+  totalMax20=0
+  totalMax30=0
+  totalMax40=0
+  totalMax50=0
+  totalMax60=0
+  totalMax70=0
+  totalMax80=0
+  totalMax90=0
+  totalMax100=0
+
+  totalMin10=0
+  totalMin20=0
+  totalMin30=0
+  totalMin40=0
+  totalMin50=0
+  totalMin60=0
+  totalMin70=0
+  totalMin80=0
+  totalMin90=0
+  totalMin100=0
+
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q2.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q2.txt \
+      > $CASTESTQ2OUT
+
+    Count10=`cat $CASTESTQ2OUT | grep count | grep "devgroup<10" | awk '{print $2}'`
+    totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc`
+    Count20=`cat $CASTESTQ2OUT | grep count | grep "devgroup<20" | awk '{print $2}'`
+    totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc`
+    Count30=`cat $CASTESTQ2OUT | grep count | grep "devgroup<30" | awk '{print $2}'`
+    totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc`
+    Count40=`cat $CASTESTQ2OUT | grep count | grep "devgroup<40" | awk '{print $2}'`
+    totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc`
+    Count50=`cat $CASTESTQ2OUT | grep count | grep "devgroup<50" | awk '{print $2}'`
+    totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc`
+    Count60=`cat $CASTESTQ2OUT | grep count | grep "devgroup<60" | awk '{print $2}'`
+    totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc`
+    Count70=`cat $CASTESTQ2OUT | grep count | grep "devgroup<70" | awk '{print $2}'`
+    totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc`
+    Count80=`cat $CASTESTQ2OUT | grep count | grep "devgroup<80" | awk '{print $2}'`
+    totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc`
+    Count90=`cat $CASTESTQ2OUT | grep count | grep "devgroup<90" | awk '{print $2}'`
+    totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc`
+    Count100=`cat $CASTESTQ2OUT | grep count | grep "test allow filtering;" | awk '{print $2}'`
+    totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc`
+
+    Avg10=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<10" | awk '{print $2}'`
+    totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc`
+    Avg20=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<20" | awk '{print $2}'`
+    totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc`
+    Avg30=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<30" | awk '{print $2}'`
+    totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc`
+    Avg40=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<40" | awk '{print $2}'`
+    totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc`
+    Avg50=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<50" | awk '{print $2}'`
+    totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc`
+    Avg60=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<60" | awk '{print $2}'`
+    totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc`
+    Avg70=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<70" | awk '{print $2}'`
+    totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc`
+    Avg80=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<80" | awk '{print $2}'`
+    totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc`
+    Avg90=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<90" | awk '{print $2}'`
+    totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc`
+    Avg100=`cat $CASTESTQ2OUT | grep avg | grep "test allow filtering;" | awk '{print $2}'`
+    totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc`
+
+    Sum10=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<10" | awk '{print $2}'`
+    totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc`
+    Sum20=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<20" | awk '{print $2}'`
+    totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc`
+    Sum30=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<30" | awk '{print $2}'`
+    totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc`
+    Sum40=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<40" | awk '{print $2}'`
+    totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc`
+    Sum50=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<50" | awk '{print $2}'`
+    totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc`
+    Sum60=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<60" | awk '{print $2}'`
+    totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc`
+    Sum70=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<70" | awk '{print $2}'`
+    totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc`
+    Sum80=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<80" | awk '{print $2}'`
+    totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc`
+    Sum90=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<90" | awk '{print $2}'`
+    totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc`
+    Sum100=`cat $CASTESTQ2OUT | grep sum | grep "test allow filtering;" | awk '{print $2}'`
+    totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc`
+
+    Max10=`cat $CASTESTQ2OUT | grep max | grep "devgroup<10" | awk '{print $2}'`
+    totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc`
+    Max20=`cat $CASTESTQ2OUT | grep max | grep "devgroup<20" | awk '{print $2}'`
+    totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc`
+    Max30=`cat $CASTESTQ2OUT | grep max | grep "devgroup<30" | awk '{print $2}'`
+    totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc`
+    Max40=`cat $CASTESTQ2OUT | grep max | grep "devgroup<40" | awk '{print $2}'`
+    totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc`
+    Max50=`cat $CASTESTQ2OUT | grep max | grep "devgroup<50" | awk '{print $2}'`
+    totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc`
+    Max60=`cat $CASTESTQ2OUT | grep max | grep "devgroup<60" | awk '{print $2}'`
+    totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc`
+    Max70=`cat $CASTESTQ2OUT | grep max | grep "devgroup<70" | awk '{print $2}'`
+    totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc`
+    Max80=`cat $CASTESTQ2OUT | grep max | grep "devgroup<80" | awk '{print $2}'`
+    totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc`
+    Max90=`cat $CASTESTQ2OUT | grep max | grep "devgroup<90" | awk '{print $2}'`
+    totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc`
+    Max100=`cat $CASTESTQ2OUT | grep max | grep "test allow filtering;" | awk '{print $2}'`
+    totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc`
+
+    Min10=`cat $CASTESTQ2OUT | grep min | grep "devgroup<10" | awk '{print $2}'`
+    totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc`
+    Min20=`cat $CASTESTQ2OUT | grep min | grep "devgroup<20" | awk '{print $2}'`
+    totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc`
+    Min30=`cat $CASTESTQ2OUT | grep min | grep "devgroup<30" | awk '{print $2}'`
+    totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc`
+    Min40=`cat $CASTESTQ2OUT | grep min | grep "devgroup<40" | awk '{print $2}'`
+    totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc`
+    Min50=`cat $CASTESTQ2OUT | grep min | grep "devgroup<50" | awk '{print $2}'`
+    totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc`
+    Min60=`cat $CASTESTQ2OUT | grep min | grep "devgroup<60" | awk '{print $2}'`
+    totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc`
+    Min70=`cat $CASTESTQ2OUT | grep min | grep "devgroup<70" | awk '{print $2}'`
+    totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc`
+    Min80=`cat $CASTESTQ2OUT | grep min | grep "devgroup<80" | awk '{print $2}'`
+    totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc`
+    Min90=`cat $CASTESTQ2OUT | grep min | grep "devgroup<90" | awk '{print $2}'`
+    totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc`
+    Min100=`cat $CASTESTQ2OUT | grep min | grep "test allow filtering;" | awk '{print $2}'`
+    totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc`
+
+  done
+  avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100"
+  echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100"
+  echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100"
+  echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100"
+  echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100"
+}
+
+################ Main ################
+
+verbose=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ3Loop.sh b/tests/perftest-scripts/cassandraTestQ3Loop.sh
new file mode 100755
index 0000000000..ce81fcb818
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ3Loop.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ3OUT=cassandraTestQ3.out
+
+function runTest {
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  totalG100=0
+  for i in `seq 1 $NUM_LOOP`; do
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q3.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q3.txt \
+      > $CASTESTQ3OUT
+    G10=`grep "devgroup<10" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup<20" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup<30" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup<40" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup<50" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup<60" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup<70" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup<80" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup<90" $CASTESTQ3OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+    G100=`grep "test group by " $CASTESTQ3OUT| awk '{print $2}'`
+    totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
+  done
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
+}
+
+################ Main ################
+
+verbose=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestQ4Loop.sh b/tests/perftest-scripts/cassandraTestQ4Loop.sh
new file mode 100755
index 0000000000..85e45787b8
--- /dev/null
+++ b/tests/perftest-scripts/cassandraTestQ4Loop.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+DATA_DIR=/mnt/root/testdata
+NUM_LOOP=5
+
+function printTo {
+  if $verbose ; then
+    echo $1
+  fi
+}
+
+CASTESTQ4OUT=cassandraTestQ4.out
+
+function runTest {
+  totalG10=0
+  totalG20=0
+  totalG30=0
+  totalG40=0
+  totalG50=0
+  totalG60=0
+  totalG70=0
+  totalG80=0
+  totalG90=0
+  totalG100=0
+  for i in `seq 1 $NUM_LOOP`; do
+    if $regeneratedata ; then
+      printTo "java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+        -datadir $DATA_DIR \
+        -numofFiles 100 \
+        -rowsperrequest 100 \
+        -writeclients 4 \
+        -conf $CASTEST_DIR/application.conf \
+        -timetest"
+      java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+        -datadir $DATA_DIR \
+        -numofFiles 100 \
+        -rowsperrequest 100 \
+        -writeclients 4 \
+        -conf $CASTEST_DIR/application.conf \
+        -timetest
+    fi 
+
+    printTo "loop i:$i, java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q4.txt"
+    java -jar \
+      $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+      -conf $CASTEST_DIR/application.conf \
+      -sql $CASTEST_DIR/q4.txt \
+      > $CASTESTQ4OUT
+    G10=`grep "devgroup<10" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG10=`echo "scale=4; $totalG10 + $G10" | bc`
+    G20=`grep "devgroup<20" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG20=`echo "scale=4; $totalG20 + $G20" | bc`
+    G30=`grep "devgroup<30" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG30=`echo "scale=4; $totalG30 + $G30" | bc`
+    G40=`grep "devgroup<40" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG40=`echo "scale=4; $totalG40 + $G40" | bc`
+    G50=`grep "devgroup<50" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG50=`echo "scale=4; $totalG50 + $G50" | bc`
+    G60=`grep "devgroup<60" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG60=`echo "scale=4; $totalG60 + $G60" | bc`
+    G70=`grep "devgroup<70" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG70=`echo "scale=4; $totalG70 + $G70" | bc`
+    G80=`grep "devgroup<80" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG80=`echo "scale=4; $totalG80 + $G80" | bc`
+    G90=`grep "devgroup<90" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG90=`echo "scale=4; $totalG90 + $G90" | bc`
+    G100=`grep "test group by minute;" $CASTESTQ4OUT| awk '{print $2}'`
+    totalG100=`echo "scale=4; $totalG100 + $G100" | bc`
+  done
+  avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc`
+  echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%"
+  echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100"
+}
+
+################ Main ################
+
+master=false
+develop=true
+verbose=false
+regeneratedata=false
+
+for arg in "$@"
+do
+  case $arg in
+    -v)
+      verbose=true
+      shift ;;
+
+    -c)
+      clients=$2
+      shift 2;;
+
+    -r)
+      regeneratedata=true
+      ;;
+
+    -n)
+      NUM_LOOP=$2
+      shift 2;;
+
+    *)
+      ;;
+  esac
+done
+
+WORK_DIR=/mnt/root/TDengine
+CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra
+
+runTest
+
+printTo "Test done!"
diff --git a/tests/perftest-scripts/cassandraTestWriteLoop.sh b/tests/perftest-scripts/cassandraTestWriteLoop.sh
index 4e42c4b3df..a218f0d0a0 100755
--- a/tests/perftest-scripts/cassandraTestWriteLoop.sh
+++ b/tests/perftest-scripts/cassandraTestWriteLoop.sh
@@ -3,7 +3,6 @@
 DATA_DIR=/mnt/root/testdata
 NUM_LOOP=1
 NUM_OF_FILES=100
-OUT_FILE=cassandraWrite.out
 
 rowsPerRequest=(1 10 50 100 500 1000 2000)
 
@@ -14,8 +13,40 @@ function printTo {
 }
 
 function runTest {
-  for c in `seq 1 $clients`; do
-    avgRPR[$c]=0
+  declare -A avgRPR
+
+  for r in ${!rowsPerRequest[@]}; do
+    for c in `seq 1 $clients`; do
+      avgRPR[$r, $c]=0
+    done
+  done
+
+  for r in ${!rowsPerRequest[@]}; do
+    for c in `seq 1 $clients`; do
+      totalRPR=0
+      OUT_FILE=cassandraWrite-rows${rowsPerRequest[$r]}-clients$c.out
+      for i in `seq 1 $NUM_LOOP`; do
+        printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+            -datadir $DATA_DIR \
+            -numofFiles $NUM_OF_FILES \
+            -rowsperrequest ${rowsPerRequest[$r]} \
+            -writeclients $c \
+            -conf $CAS_TEST_DIR/application.conf"
+        java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
+            -datadir $DATA_DIR \
+            -numofFiles $NUM_OF_FILES \
+            -rowsperrequest ${rowsPerRequest[$r]} \
+            -writeclients $c \
+            -conf $CAS_TEST_DIR/application.conf \
+            2>&1 | tee $OUT_FILE
+        RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'`
+        totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
+        printTo "r:$r rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR"
+      done
+      avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+      printTo "r:$r c:$c avgRPR:${avgRPR[$r,$c]}"
+    done
+
   done
 
   printf "R/R, "
@@ -28,32 +59,10 @@ function runTest {
   done
   printf "\n"
 
-  for r in ${rowsPerRequest[@]}; do
-    printf "$r, "
+  for r in ${!rowsPerRequest[@]}; do
+    printf "${rowsPerRequest[$r]}, "
     for c in `seq 1 $clients`; do
-      totalRPR=0
-      for i in `seq 1 $NUM_LOOP`; do
-        printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
-            -datadir $DATA_DIR \
-            -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
-            -writeclients $c \
-            -conf $CAS_TEST_DIR/application.conf"
-        java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
-            -datadir $DATA_DIR \
-            -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
-            -writeclients $c \
-            -conf $CAS_TEST_DIR/application.conf \
-            2>&1 > $OUT_FILE
-        RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'`
-        totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
-        printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
-      done
-      avgRPR[$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
-    done
-    for c in `seq 1 $clients`; do
-      printf "${avgRPR[$c]}, "
+      printf "${avgRPR[$r,$c]}, "
     done
     printf "\n"
   done
diff --git a/tests/perftest-scripts/coverage_test.sh b/tests/perftest-scripts/coverage_test.sh
new file mode 100755
index 0000000000..152e20756b
--- /dev/null
+++ b/tests/perftest-scripts/coverage_test.sh
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.cover
+TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+	echo "check if TDengine need build"
+	cd $TDENGINE_DIR
+  git remote prune origin > /dev/null
+	git remote update > /dev/null
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+
+	# reset counter
+	lcov -d . --zerocounters
+
+	cd $TDENGINE_DIR/debug
+
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+    git reset --hard
+		git pull
+
+		LOCAL_COMMIT=`git rev-parse --short @`
+
+		rm -rf *
+		cmake -DCOVER=true -DRANDOM_FILE_FAIL=true .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runGeneralCaseOneByOne {
+	while read -r line; do
+		if [[ $line =~ ^./test.sh* ]]; then
+			general_case=`echo $line | grep -w general`
+
+			if [ -n "$general_case" ]; then
+				case=`echo $line |grep general| awk '{print $NF}'`
+				./test.sh -f $case > /dev/null 2>&1 && \
+				echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \
+				echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+			fi
+		fi
+	done < $1
+}
+
+function runTest {
+	echo "run Test"
+
+	cd $TDENGINE_DIR/tests/script
+
+	[ -d ../../sim ] && rm -rf ../../sim
+	[ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
+
+	runGeneralCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess coverage test case(s) succeed! ### ${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed coverage test case(s) failed! ### ${NC}\n" | tee -a $TDENGINE_COVERAGE_REPORT
+#  exit $totalPyFailed
+	fi
+
+	cd $TDENGINE_DIR/tests
+	rm -rf ../sim
+	./test-all.sh full python | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test Connector
+	stopTaosd
+	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+
+	cd $TDENGINE_DIR/src/connector/jdbc
+	mvn clean package
+	mvn test | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test C Demo
+	stopTaosd
+	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+	yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test waltest
+	dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+	walDir=`find $dataDir -name "wal"|head -n1`
+	echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	if [ -n "$walDir" ]; then
+		yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDENGINE_COVERAGE_REPORT
+	fi
+
+  # run Unit Test
+  echo "Run Unit Test: utilTest, queryTest and cliTest"
+  $TDENGINE_DIR/debug/build/bin/utilTest > /dev/null && echo "utilTest pass!" || echo "utilTest failed!"
+  $TDENGINE_DIR/debug/build/bin/queryTest > /dev/null && echo "queryTest pass!" || echo "queryTest failed!"
+  $TDENGINE_DIR/debug/build/bin/cliTest > /dev/null && echo "cliTest pass!" || echo "cliTest failed!"
+
+	stopTaosd
+}
+
+function lcovFunc {
+	echo "collect data by lcov"
+	cd $TDENGINE_DIR
+
+	# collect data
+	lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+	# remove exclude paths
+	lcov --remove coverage.info \
+    '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
+    --rc lcov_branch_coverage=1 -o coverage.info
+
+	# generate result
+	lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# push result to coveralls.io
+	coveralls-lcov coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+}
+
+function sendReport {
+	echo "send report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDENGINE_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
+	BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
+	echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+    pkill -TERM -x taosd
+    sleep 1
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+function runTestRandomFail {
+  exec_random_fail_sh=$1
+  default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
+	[ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
+
+  dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
+  default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
+	[ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
+
+  runTest NoRandomFail
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
+
+rm /tmp/core-*
+
+stopTaosd
+buildTDengine
+
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh 
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh 
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh 
+
+lcovFunc
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/perftest-scripts/full_test.sh b/tests/perftest-scripts/full_test.sh
new file mode 100755
index 0000000000..1738f69f35
--- /dev/null
+++ b/tests/perftest-scripts/full_test.sh
@@ -0,0 +1,191 @@
+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.orig
+TDENGINE_FULLTEST_REPORT=$TDENGINE_DIR/tests/full-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+  echo "check if TDengine need build"
+
+  need_rebuild=false
+
+  if [ ! -d $TDENGINE_DIR ]; then
+    echo "No TDengine source code found!"
+    git clone https://github.com/taosdata/TDengine $TDENGINE_DIR
+    need_rebuild=true
+  fi
+
+  cd $TDENGINE_DIR
+  git remote prune origin > /dev/null
+  git remote update > /dev/null
+  REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+  LOCAL_COMMIT=`git rev-parse --short @`
+  echo " LOCAL: $LOCAL_COMMIT"
+  echo "REMOTE: $REMOTE_COMMIT"
+
+  if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+    echo "repo up-to-date"
+  else
+    echo "repo need to pull"
+    git pull
+    need_rebuild=true
+  fi
+
+  [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+  cd $TDENGINE_DIR/debug
+  [ -f $TDENGINE_DIR/debug/build/bin/taosd ] || need_rebuild=true
+
+  if $need_rebuild ; then
+    echo "rebuild.."
+
+    LOCAL_COMMIT=`git rev-parse --short @`
+    rm -rf *
+    cmake .. > /dev/null
+    make > /dev/null
+  fi
+
+  make install > /dev/null
+}
+
+function runGeneralCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      general_case=`echo $line | grep -w general`
+
+      if [ -n "$general_case" ]; then
+        case=`echo $line | awk '{print $NF}'`
+
+        start_time=`date +%s`
+        ./test.sh -f $case > /dev/null 2>&1 && ret=0 || ret = 1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_FULLTEST_REPORT
+      fi
+    fi
+  done < $1
+}
+
+function runPyCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^python.* ]]; then
+      if [[ $line != *sleep* ]]; then
+        case=`echo $line|awk '{print $NF}'`
+        start_time=`date +%s`
+        $line > /dev/null 2>&1 && ret=0 || ret=1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+      else
+        $line > /dev/null 2>&1
+      fi
+    fi
+  done < $1
+}
+
+function runTest {
+  echo "Run Test"
+  cd $TDENGINE_DIR/tests/script
+
+  [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
+  [ -f $TDENGINE_FULLTEST_REPORT ] && rm $TDENGINE_FULLTEST_REPORT
+
+  runGeneralCaseOneByOne jenkins/basic.txt
+
+  totalSuccess=`grep 'success' $TDENGINE_FULLTEST_REPORT | wc -l`
+
+  if [ "$totalSuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalSuccess SIM case(s) succeed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  totalFailed=`grep 'failed\|fault' $TDENGINE_FULLTEST_REPORT | wc -l`
+  if [ "$totalFailed" -ne "0" ]; then
+    echo -e "${RED} ### Total $totalFailed SIM case(s) failed! ### ${NC}\n" \
+    | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  cd $TDENGINE_DIR/tests/pytest
+  [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
+  [ -f pytest-out.log ] && rm -f pytest-out.log
+  runPyCaseOneByOne fulltest.sh
+
+  totalPySuccess=`grep 'success' pytest-out.log | wc -l`
+  totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
+
+  cat pytest-out.log >> $TDENGINE_FULLTEST_REPORT
+  if [ "$totalPySuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  if [ "$totalPyFailed" -ne "0" ]; then
+    echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+}
+
+function sendReport {
+  echo "Send Report"
+  receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+  mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+  cd $TDENGINE_DIR/tests
+
+  sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_FULLTEST_REPORT
+  BODY_CONTENT=`cat $TDENGINE_FULLTEST_REPORT`
+
+  cd $TDENGINE_DIR
+  tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
+
+  echo -e "to: ${receiver}\nsubject: Full test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+  (cat - && uuencode $TDENGINE_FULLTEST_REPORT fulltest-report-$today.log) | \
+  (cat - && uuencode $TDENGINE_DIR/fulltest-$today.tar.gz fulltest-$today.tar.gz) | \
+  ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+  echo "Stop taosd"
+  systemctl stop taosd
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+  while [ -n "$PID" ]
+  do
+    pkill -TERM -x taosd
+    sleep 1
+    PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+  done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Full Test" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDengine
+runTest
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Full Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/perftest-scripts/tdinternal_coverage_test.sh b/tests/perftest-scripts/tdinternal_coverage_test.sh
new file mode 100755
index 0000000000..91d204baf9
--- /dev/null
+++ b/tests/perftest-scripts/tdinternal_coverage_test.sh
@@ -0,0 +1,186 @@
+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal.cover
+TDINTERNAL_COVERAGE_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDinternal {
+	echo "check if TDinternal need build"
+	cd $TDINTERNAL_DIR
+	NEED_COMPILE=0
+#	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "TDinternal repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git pull
+
+#		NEED_COMPILE=1
+	fi
+
+  lcov -d . --zerocounters
+#	git submodule update --init --recursive
+	cd $TDINTERNAL_DIR/community
+	TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
+	if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
+		echo "community repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git checkout develop
+#		git pull
+#		NEED_COMPILE=1
+	fi
+
+	cd $TDINTERNAL_DIR/debug
+
+	if [[ $NEED_COMPILE -eq 1 ]]; then
+		LOCAL_COMMIT=`git rev-parse --short @`
+		rm -rf *
+		cmake .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runUniqueCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      case=`echo $line | awk '{print $NF}'`
+      start_time=`date +%s`
+      ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT || \
+        echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
+      end_time=`date +%s`
+      echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_COVERAGE_REPORT
+    fi
+  done < $1
+}
+
+function runTest {
+	echo "Run Test"
+	cd $TDINTERNAL_DIR/community/tests/script
+	[ -d ../../sim ] && rm -rf ../../sim
+
+	[ -f $TDINTERNAL_COVERAGE_REPORT ] && rm $TDINTERNAL_COVERAGE_REPORT
+
+	runUniqueCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDINTERNAL_COVERAGE_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDINTERNAL_COVERAGE_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_COVERAGE_REPORT
+#  exit $totalPyFailed
+	fi
+
+	# Test Python test case
+	cd $TDINTERNAL_DIR/community/tests
+	/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test Connector
+	stopTaosd
+	$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+
+	cd $TDINTERNAL_DIR/community/src/connector/jdbc
+	mvn clean package
+	mvn test | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test C Demo
+	stopTaosd
+	$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+	yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test waltest
+	dataDir=`grep dataDir $TDINTERNAL_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+	walDir=`find $dataDir -name "wal"|head -n1`
+	echo "dataDir: $dataDir\nwalDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
+	if [ -n "$walDir" ]; then
+		yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDINTERNAL_COVERAGE_REPORT
+	fi
+
+	stopTaosd
+}
+
+function sendReport {
+	echo "Send Report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
+
+	BODY_CONTENT=`cat $TDINTERNAL_COVERAGE_REPORT`
+	echo -e "to: ${receiver}\nsubject: TDinternal coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode tdinternal-coverage-report-$today.tar.gz tdinternal-coverage-report-$today.tar.gz) | \
+	(cat - && uuencode $TDINTERNAL_COVERAGE_REPORT tdinternal-coverage-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function lcovFunc {
+	echo "collect data by lcov"
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
+	# collect data
+	lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtmml_branch_coverage=1 --no-external -b $TDINTERNAL_DIR -o coverage.info
+
+	# remove exclude paths
+	lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
+    --rc lcov_branch_coverage=1 -o coverage.info
+
+	# generate result
+	lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	genhtml -o html coverage.info
+
+	tar czf tdinternal-coverage-report-$today.tar.gz html coverage.info $TDINTERNAL_COVERAGE_REPORT
+	# push result to coveralls.io
+#	coveralls-lcov coverage.info | tee -a tdinternal-coverage-report-$today.log
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+        	pkill -TERM -x taosd
+        	sleep 1
+  		PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test for TDinternal" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDinternal
+runTest
+lcovFunc
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of TDinternal Coverage Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/perftest-scripts/tdinternal_test.sh b/tests/perftest-scripts/tdinternal_test.sh
new file mode 100755
index 0000000000..4de0549be0
--- /dev/null
+++ b/tests/perftest-scripts/tdinternal_test.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal
+TDINTERNAL_TEST_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDinternal {
+	echo "check if TDinternal need build"
+	cd $TDINTERNAL_DIR
+	NEED_COMPILE=0
+#	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "TDinternal repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git pull
+
+#		NEED_COMPILE=1
+	fi
+
+#	git submodule update --init --recursive
+	cd $TDINTERNAL_DIR/community
+	TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
+	if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
+		echo "community repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git checkout develop
+#		git pull
+#		NEED_COMPILE=1
+	fi
+
+	cd $TDINTERNAL_DIR/debug
+
+	if [[ $NEED_COMPILE -eq 1 ]]; then
+		LOCAL_COMMIT=`git rev-parse --short @`
+		rm -rf *
+		cmake .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runUniqueCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      case=`echo $line | awk '{print $NF}'`
+      start_time=`date +%s`
+      ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_TEST_REPORT || \
+        echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_TEST_REPORT
+      end_time=`date +%s`
+      echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_TEST_REPORT
+    fi
+  done < $1
+}
+
+function runPyCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^python.* ]]; then
+      if [[ $line != *sleep* ]]; then
+        case=`echo $line|awk '{print $NF}'`
+        start_time=`date +%s`
+        $line > /dev/null 2>&1 && ret=0 || ret=1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDINTERNAL_DIR/community/sim -name "*log" -exec tar czf $TDINTERNAL_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+      else
+        $line > /dev/null 2>&1
+      fi
+    fi
+  done < $1
+}
+
+function runTest {
+	echo "Run Test"
+	cd $TDINTERNAL_DIR/community/tests/script
+	[ -d $TDINTERNAL_DIR/sim ] && rm -rf $TDINTERNAL_DIR/sim
+
+	[ -f $TDINTERNAL_TEST_REPORT ] && rm $TDINTERNAL_TEST_REPORT
+
+	runUniqueCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDINTERNAL_TEST_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_TEST_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDINTERNAL_TEST_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_TEST_REPORT
+#  exit $totalPyFailed
+	fi
+
+	cd $TDINTERNAL_DIR/community/tests/pytest
+  [ -d $TDINTERNAL_DIR/community/sim ] && rm -rf $TDINTERNAL_DIR/community/sim
+  [ -f pytest-out.log ] && rm -f pytest-out.log
+
+	/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_TEST_REPORT
+  runPyCaseOneByOne fulltest.sh
+
+  totalPySuccess=`grep 'success' pytest-out.log | wc -l`
+  totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
+
+  cat pytest-out.log >> $TDINTERNAL_TEST_REPORT
+  if [ "$totalPySuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
+      | tee -a $TDINTERNAL_TEST_REPORT
+  fi
+
+  if [ "$totalPyFailed" -ne "0" ]; then
+    echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
+      | tee -a $TDINTERNAL_TEST_REPORT
+  fi
+}
+
+function sendReport {
+	echo "Send Report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_TEST_REPORT
+	BODY_CONTENT=`cat $TDINTERNAL_TEST_REPORT`
+
+  cd $TDINTERNAL_DIR
+  tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
+
+	echo -e "to: ${receiver}\nsubject: TDinternal test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $TDINTERNAL_TEST_REPORT tdinternal-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+        	pkill -KILL -x taosd
+        	sleep 1
+  		PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Test for TDinternal" | tee -a $WORK_DIR/cron.log
+
+buildTDinternal
+runTest
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of TDinternal Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py
index 9e38e04b63..98181180e2 100755
--- a/tests/pytest/crash_gen.py
+++ b/tests/pytest/crash_gen.py
@@ -42,6 +42,13 @@ import os
 import io
 import signal
 import traceback
+
+try:
+    import psutil
+except:
+    print("Psutil module needed, please install: sudo pip3 install psutil")
+    sys.exit(-1)
+
 # Require Python 3
 if sys.version_info[0] < 3:
     raise Exception("Must be using Python 3")
@@ -52,13 +59,12 @@ if sys.version_info[0] < 3:
 # Command-line/Environment Configurations, will set a bit later
 # ConfigNameSpace = argparse.Namespace
 gConfig = argparse.Namespace()  # Dummy value, will be replaced later
+gSvcMgr = None # TODO: refactor this hack, use dep injection
 logger = None
 
-
 def runThread(wt: WorkerThread):
     wt.run()
 
-
 class CrashGenError(Exception):
     def __init__(self, msg=None, errno=None):
         self.msg = msg
@@ -69,8 +75,7 @@ class CrashGenError(Exception):
 
 
 class WorkerThread:
-    def __init__(self, pool: ThreadPool, tid,
-                 tc: ThreadCoordinator,
+    def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator,
                  # te: TaskExecutor,
                  ):  # note: main thread context!
         # self._curStep = -1
@@ -131,18 +136,28 @@ class WorkerThread:
 
         # clean up
         if (gConfig.per_thread_db_connection):  # type: ignore
-            self._dbConn.close()
+            if self._dbConn.isOpen: #sometimes it is not open
+                self._dbConn.close()
+            else:
+                logger.warning("Cleaning up worker thread, dbConn already closed")
 
     def _doTaskLoop(self):
         # while self._curStep < self._pool.maxSteps:
         # tc = ThreadCoordinator(None)
         while True:
             tc = self._tc  # Thread Coordinator, the overall master
-            tc.crossStepBarrier()  # shared barrier first, INCLUDING the last one
+            try:
+                tc.crossStepBarrier()  # shared barrier first, INCLUDING the last one
+            except threading.BrokenBarrierError as err: # main thread timed out
+                print("_bto", end="")
+                logger.debug("[TRD] Worker thread exiting due to main thread barrier time-out")
+                break
+
             logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid))
             self.crossStepGate()   # then per-thread gate, after being tapped
             logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid))
             if not self._tc.isRunning():
+                print("_wts", end="")
                 logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
                 break
 
@@ -159,6 +174,7 @@ class WorkerThread:
             logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid))
 
             self._dbInUse = False  # there may be changes between steps
+        # print("_wtd", end=None) # worker thread died
 
     def verifyThreadSelf(self):  # ensure we are called by this own thread
         if (threading.get_ident() != self._thread.ident):
@@ -187,30 +203,24 @@ class WorkerThread:
         # self._curStep += 1  # off to a new step...
 
     def tapStepGate(self):  # give it a tap, release the thread waiting there
-        self.verifyThreadAlive()
+        # self.verifyThreadAlive()
         self.verifyThreadMain()  # only allowed for main thread
 
-        logger.debug("[TRD] Tapping worker thread {}".format(self._tid))
-        self._stepGate.set()  # wake up!
-        time.sleep(0)  # let the released thread run a bit
+        if self._thread.is_alive():
+            logger.debug("[TRD] Tapping worker thread {}".format(self._tid))
+            self._stepGate.set()  # wake up!
+            time.sleep(0)  # let the released thread run a bit
+        else:
+            print("_tad", end="") # Thread already dead
 
     def execSql(self, sql):  # TODO: expose DbConn directly
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.execute(sql)
-        else:
-            return self._tc.getDbManager().getDbConn().execute(sql)
+        return self.getDbConn().execute(sql)
 
     def querySql(self, sql):  # TODO: expose DbConn directly
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.query(sql)
-        else:
-            return self._tc.getDbManager().getDbConn().query(sql)
+        return self.getDbConn().query(sql)
 
     def getQueryResult(self):
-        if (gConfig.per_thread_db_connection):
-            return self._dbConn.getQueryResult()
-        else:
-            return self._tc.getDbManager().getDbConn().getQueryResult()
+        return self.getDbConn().getQueryResult()
 
     def getDbConn(self):
         if (gConfig.per_thread_db_connection):
@@ -228,6 +238,8 @@ class WorkerThread:
 
 
 class ThreadCoordinator:
+    WORKER_THREAD_TIMEOUT = 30
+
     def __init__(self, pool: ThreadPool, dbManager):
         self._curStep = -1  # first step is 0
         self._pool = pool
@@ -248,14 +260,14 @@ class ThreadCoordinator:
     def getDbManager(self) -> DbManager:
         return self._dbManager
 
-    def crossStepBarrier(self):
-        self._stepBarrier.wait()
+    def crossStepBarrier(self, timeout=None):
+        self._stepBarrier.wait(timeout) 
 
     def requestToStop(self):
         self._runStatus = MainExec.STATUS_STOPPING
         self._execStats.registerFailure("User Interruption")
 
-    def _runShouldEnd(self, transitionFailed, hasAbortedTask):
+    def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout):
         maxSteps = gConfig.max_steps  # type: ignore
         if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9
             return True
@@ -265,6 +277,8 @@ class ThreadCoordinator:
             return True
         if hasAbortedTask:
             return True
+        if workerTimeout:
+            return True
         return False
 
     def _hasAbortedTask(self): # from execution of previous step
@@ -296,7 +310,7 @@ class ThreadCoordinator:
         # let other threads go past the pool barrier, but wait at the
         # thread gate
         logger.debug("[TRD] Main thread about to cross the barrier")
-        self.crossStepBarrier()
+        self.crossStepBarrier(timeout=self.WORKER_THREAD_TIMEOUT)
         self._stepBarrier.reset()  # Other worker threads should now be at the "gate"
         logger.debug("[TRD] Main thread finished crossing the barrier")
 
@@ -327,6 +341,7 @@ class ThreadCoordinator:
                 # end, and maybe signal them to stop
             else:
                 raise
+        return transitionFailed
 
         self.resetExecutedTasks()  # clear the tasks after we are done
         # Get ready for next step
@@ -342,11 +357,21 @@ class ThreadCoordinator:
         self._execStats.startExec()  # start the stop watch
         transitionFailed = False
         hasAbortedTask = False
-        while not self._runShouldEnd(transitionFailed, hasAbortedTask):
+        workerTimeout = False
+        while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout):
             if not gConfig.debug: # print this only if we are not in debug mode                
                 print(".", end="", flush=True)
                         
-            self._syncAtBarrier() # For now just cross the barrier
+            try:
+                self._syncAtBarrier() # For now just cross the barrier
+            except threading.BrokenBarrierError as err:
+                logger.info("Main loop aborted, caused by worker thread time-out")
+                self._execStats.registerFailure("Aborted due to worker thread timeout")
+                print("\n\nWorker Thread time-out detected, important thread info:")
+                ts = ThreadStacks()
+                ts.print(filterInternal=True)
+                workerTimeout = True
+                break
 
             # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate"
             # We use this period to do house keeping work, when all worker
@@ -358,12 +383,20 @@ class ThreadCoordinator:
                 break # do transition only if tasks are error free
 
             # Ending previous step
-            transitionFailed = self._doTransition() # To start, we end step -1 first
+            try:
+                transitionFailed = self._doTransition() # To start, we end step -1 first
+            except taos.error.ProgrammingError as err:
+                transitionFailed = True
+                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+                logger.info("Transition failed: errno=0x{:X}, msg: {}".format(errno2, err))
+
             # Then we move on to the next step
             self._releaseAllWorkerThreads(transitionFailed)                    
 
         if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate"
             logger.debug("Abnormal ending of main thraed")
+        elif workerTimeout:
+            logger.debug("Abnormal ending of main thread, due to worker timeout")
         else: # regular ending, workers waiting at "barrier"
             logger.debug("Regular ending, main thread waiting for all worker threads to stop...")
             self._syncAtBarrier()
@@ -561,6 +594,10 @@ class DbConn:
     def __init__(self):
         self.isOpen = False
         self._type = self.TYPE_INVALID
+        self._lastSql = None
+
+    def getLastSql(self):
+        return self._lastSql
 
     def open(self):
         if (self.isOpen):
@@ -569,9 +606,7 @@ class DbConn:
         # below implemented by child classes
         self.openByType()
 
-        logger.debug(
-            "[DB] data connection opened, type = {}".format(
-                self._type))
+        logger.debug("[DB] data connection opened, type = {}".format(self._type))
         self.isOpen = True
 
     def resetDb(self):  # reset the whole database, etc.
@@ -594,21 +629,29 @@ class DbConn:
 
     def _queryAny(self, sql):  # actual query result as an int
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot query database until connection is open")
+            raise RuntimeError("Cannot query database until connection is open")
         nRows = self.query(sql)
         if nRows != 1:
-            raise RuntimeError(
-                "Unexpected result for query: {}, rows = {}".format(
-                    sql, nRows))
+            raise RuntimeError("Unexpected result for query: {}, rows = {}".format(sql, nRows))
         if self.getResultRows() != 1 or self.getResultCols() != 1:
-            raise RuntimeError(
-                "Unexpected result set for query: {}".format(sql))
+            raise RuntimeError("Unexpected result set for query: {}".format(sql))
         return self.getQueryResult()[0][0]
 
+    def use(self, dbName):
+        self.execute("use {}".format(dbName))
+
+    def hasDatabases(self):
+        return self.query("show databases") > 0
+
+    def hasTables(self):
+        return self.query("show tables") > 0
+
     def execute(self, sql):
         raise RuntimeError("Unexpected execution, should be overriden")
 
+    def query(self, sql) -> int: # return num rows returned
+        raise RuntimeError("Unexpected execution, should be overriden")
+
     def openByType(self):
         raise RuntimeError("Unexpected execution, should be overriden")
 
@@ -643,10 +686,11 @@ class DbConnRest(DbConn):
         self.isOpen = False
 
     def _doSql(self, sql):
+        self._lastSql = sql # remember this, last SQL attempted
         try:
             r = requests.post(self._url, 
                 data = sql,
-                auth = HTTPBasicAuth('root', 'taosdata'))            
+                auth = HTTPBasicAuth('root', 'taosdata'))         
         except:
             print("REST API Failure (TODO: more info here)")
             raise
@@ -742,11 +786,16 @@ class MyTDSql:
 
 
 class DbConnNative(DbConn):
+    # Class variables
+    _lock = threading.Lock()
+    _connInfoDisplayed = False
+
     def __init__(self):
         super().__init__()
         self._type = self.TYPE_NATIVE
         self._conn = None
         self._cursor = None
+        
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -755,31 +804,32 @@ class DbConnNative(DbConn):
         else:
             projPath = selfPath[:selfPath.find("tests")]
 
+        buildPath = None
         for root, dirs, files in os.walk(projPath):
             if ("taosd" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
+        if buildPath == None:
+            raise RuntimeError("Failed to determine buildPath, selfPath={}".format(self_path))
         return buildPath
 
-    connInfoDisplayed = False
+    
     def openByType(self):  # Open connection
         cfgPath = self.getBuildPath() + "/test/cfg"
         hostAddr = "127.0.0.1"
-        if not self.connInfoDisplayed:
-            logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
-            self.connInfoDisplayed = True
 
-        self._conn = taos.connect(
-            host=hostAddr,
-            config=cfgPath)  # TODO: make configurable
-        self._cursor = self._conn.cursor()
-
-        # Get the connection/cursor ready
+        with self._lock: # force single threading for opening DB connections
+            if not self._connInfoDisplayed:
+                self.__class__._connInfoDisplayed = True # updating CLASS variable
+                logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
+            
+            self._conn = taos.connect(host=hostAddr, config=cfgPath)  # TODO: make configurable
+            self._cursor = self._conn.cursor()
+        
         self._cursor.execute('reset query cache')
         # self._cursor.execute('use db') # do this at the beginning of every
-        # step
 
         # Open connection
         self._tdSql = MyTDSql()
@@ -984,29 +1034,11 @@ class StateDbOnly(AnyState):
         if (not self.hasTask(tasks, TaskCreateDb)):
             # only if we don't create any more
             self.assertAtMostOneSuccess(tasks, TaskDropDb)
-        self.assertIfExistThenSuccess(tasks, TaskDropDb)
-        # self.assertAtMostOneSuccess(tasks, CreateFixedTableTask) # not true in massively parrallel cases
-        # Nothing to be said about adding data task
-        # if ( self.hasSuccess(tasks, DropDbTask) ): # dropped the DB
-        # self.assertHasTask(tasks, DropDbTask) # implied by hasSuccess
-        # self.assertAtMostOneSuccess(tasks, DropDbTask)
-        # self._state = self.STATE_EMPTY
-        # if ( self.hasSuccess(tasks, TaskCreateSuperTable) ): # did not drop db, create table success
-        #     # self.assertHasTask(tasks, CreateFixedTableTask) # tried to create table
-        #     if ( not self.hasTask(tasks, TaskDropSuperTable) ):
-        #         self.assertAtMostOneSuccess(tasks, TaskCreateSuperTable) # at most 1 attempt is successful, if we don't drop anything
-        # self.assertNoTask(tasks, DropDbTask) # should have have tried
-        # if ( not self.hasSuccess(tasks, AddFixedDataTask) ): # just created table, no data yet
-        #     # can't say there's add-data attempts, since they may all fail
-        #     self._state = self.STATE_TABLE_ONLY
-        # else:
-        #     self._state = self.STATE_HAS_DATA
-        # What about AddFixedData?
-        # elif ( self.hasSuccess(tasks, AddFixedDataTask) ):
-        #     self._state = self.STATE_HAS_DATA
-        # else: # no success in dropping db tasks, no success in create fixed table? read data should also fail
-        #     # raise RuntimeError("Unexpected no-success scenario")   # We might just landed all failure tasks,
-        #     self._state = self.STATE_DB_ONLY  # no change
+
+        # TODO: restore the below, the problem exists, although unlikely in real-world
+        # if (gSvcMgr!=None) and gSvcMgr.isRestarting():     
+        # if (gSvcMgr == None) or (not gSvcMgr.isRestarting()) : 
+        #     self.assertIfExistThenSuccess(tasks, TaskDropDb)       
 
 
 class StateSuperTableOnly(AnyState):
@@ -1082,7 +1114,7 @@ class StateMechine:
         self._curState = self._findCurrentState()  # starting state
         # transitition target probabilities, indexed with value of STATE_EMPTY,
         # STATE_DB_ONLY, etc.
-        self._stateWeights = [1, 3, 5, 15]
+        self._stateWeights = [1, 2, 10, 40]
 
     def getCurrentState(self):
         return self._curState
@@ -1128,33 +1160,22 @@ class StateMechine:
     def _findCurrentState(self):
         dbc = self._dbConn
         ts = time.time()  # we use this to debug how fast/slow it is to do the various queries to find the current DB state
-        if dbc.query("show databases") == 0:  # no database?!
-            # logger.debug("Found EMPTY state")
-            logger.debug(
-                "[STT] empty database found, between {} and {}".format(
-                    ts, time.time()))
+        if not dbc.hasDatabases():  # no database?!
+            logger.debug( "[STT] empty database found, between {} and {}".format(ts, time.time()))
             return StateEmpty()
         # did not do this when openning connection, and this is NOT the worker
         # thread, which does this on their own
-        dbc.execute("use db")
-        if dbc.query("show tables") == 0:  # no tables
-            # logger.debug("Found DB ONLY state")
-            logger.debug(
-                "[STT] DB_ONLY found, between {} and {}".format(
-                    ts, time.time()))
+        dbc.use("db")
+        if not dbc.hasTables():  # no tables
+            logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time()))
             return StateDbOnly()
-        if dbc.query("SELECT * FROM db.{}".format(DbManager.getFixedSuperTableName())
-                     ) == 0:  # no regular tables
-            # logger.debug("Found TABLE_ONLY state")
-            logger.debug(
-                "[STT] SUPER_TABLE_ONLY found, between {} and {}".format(
-                    ts, time.time()))
+
+        sTable = DbManager.getFixedSuperTable()
+        if sTable.hasRegTables(dbc):  # no regular tables
+            logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time()))
             return StateSuperTableOnly()
         else:  # has actual tables
-            # logger.debug("Found HAS_DATA state")
-            logger.debug(
-                "[STT] HAS_DATA found, between {} and {}".format(
-                    ts, time.time()))
+            logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time()))
             return StateHasData()
 
     def transition(self, tasks):
@@ -1172,7 +1193,8 @@ class StateMechine:
             # case of multiple creation and drops
 
         if self._curState.canDropDb():
-            self._curState.assertIfExistThenSuccess(tasks, TaskDropDb)
+            if gSvcMgr == None: # only if we are running as client-only
+                self._curState.assertIfExistThenSuccess(tasks, TaskDropDb)
             # self.assertAtMostOneSuccess(tasks, DropDbTask) # not really in
             # case of drop-create-drop
 
@@ -1300,13 +1322,17 @@ class DbManager():
     def getFixedSuperTableName(cls):
         return "fs_table"
 
+    @classmethod
+    def getFixedSuperTable(cls):
+        return TdSuperTable(cls.getFixedSuperTableName())
+
     def releaseTable(self, i):  # return the table back, so others can use it
         self.tableNumQueue.release(i)
 
     def getNextTick(self):
         with self._lock:  # prevent duplicate tick
-            if Dice.throw(10) == 0:  # 1 in 10 chance
-                return self._lastTick + datetime.timedelta(0, -100)
+            if Dice.throw(20) == 0:  # 1 in 20 chance
+                return self._lastTick + datetime.timedelta(0, -100) # Go back in time 100 seconds
             else:  # regular
                 # add one second to it
                 self._lastTick += datetime.timedelta(0, 1)
@@ -1322,7 +1348,9 @@ class DbManager():
             self.getNextInt())
 
     def getNextFloat(self):
-        return 0.9 + self.getNextInt()
+        ret = 0.9 + self.getNextInt()
+        # print("Float obtained: {}".format(ret))
+        return ret
 
     def getTableNameToDelete(self):
         tblNum = self.tableNumQueue.pop()  # TODO: race condition!
@@ -1340,33 +1368,35 @@ class TaskExecutor():
         def __init__(self, size=10):
             self._size = size
             self._list = []
+            self._lock = threading.Lock()
 
         def add(self, n: int):
-            if not self._list:  # empty
-                self._list.append(n)
-                return
-            # now we should insert
-            nItems = len(self._list)
-            insPos = 0
-            for i in range(nItems):
-                insPos = i
-                if n <= self._list[i]:  # smaller than this item, time to insert
-                    break  # found the insertion point
-                insPos += 1  # insert to the right
+            with self._lock:
+                if not self._list:  # empty
+                    self._list.append(n)
+                    return
+                # now we should insert
+                nItems = len(self._list)
+                insPos = 0
+                for i in range(nItems):
+                    insPos = i
+                    if n <= self._list[i]:  # smaller than this item, time to insert
+                        break  # found the insertion point
+                    insPos += 1  # insert to the right
 
-            if insPos == 0:  # except for the 1st item, # TODO: elimiate first item as gating item
-                return  # do nothing
+                if insPos == 0:  # except for the 1st item, # TODO: elimiate first item as gating item
+                    return  # do nothing
 
-            # print("Inserting at postion {}, value: {}".format(insPos, n))
-            self._list.insert(insPos, n)  # insert
+                # print("Inserting at postion {}, value: {}".format(insPos, n))
+                self._list.insert(insPos, n)  # insert
 
-            newLen = len(self._list)
-            if newLen <= self._size:
-                return  # do nothing
-            elif newLen == (self._size + 1):
-                del self._list[0]  # remove the first item
-            else:
-                raise RuntimeError("Corrupt Bounded List")
+                newLen = len(self._list)
+                if newLen <= self._size:
+                    return  # do nothing
+                elif newLen == (self._size + 1):
+                    del self._list[0]  # remove the first item
+                else:
+                    raise RuntimeError("Corrupt Bounded List")
 
         def __str__(self):
             return repr(self._list)
@@ -1419,7 +1449,6 @@ class Task():
         # logger.debug("Creating new task {}...".format(self._taskNum))
 
         self._execStats = execStats
-        self._lastSql = ""  # last SQL executed/attempted
 
     def isSuccess(self):
         return self._err is None
@@ -1446,6 +1475,39 @@ class Task():
             "To be implemeted by child classes, class name: {}".format(
                 self.__class__.__name__))
 
+    def _isErrAcceptable(self, errno, msg):
+        if errno in [
+                0x05,  # TSDB_CODE_RPC_NOT_READY
+                # 0x200, # invalid SQL， TODO: re-examine with TD-934
+                0x360, 0x362, 
+                0x369, # tag already exists
+                0x36A, 0x36B, 0x36D,
+                0x381, 
+                0x380, # "db not selected"
+                0x383,
+                0x386,  # DB is being dropped?!
+                0x503,
+                0x510,  # vnode not in ready state
+                0x600,
+                1000  # REST catch-all error
+            ]: 
+            return True # These are the ALWAYS-ACCEPTABLE ones
+        elif (errno in [ 0x0B ]) and gConfig.auto_start_service:
+            return True # We may get "network unavilable" when restarting service
+        elif errno == 0x200 : # invalid SQL, we need to div in a bit more
+            if msg.find("invalid column name") != -1:
+                return True 
+            elif msg.find("tags number not matched") != -1: # mismatched tags after modification
+                return True
+            elif msg.find("duplicated column names") != -1: # also alter table tag issues
+                return True
+        elif (gSvcMgr!=None) and gSvcMgr.isRestarting():
+            logger.info("Ignoring error when service is restarting: errno = {}, msg = {}".format(errno, msg))
+            return True
+        
+        return False # Not an acceptable error
+
+
     def execute(self, wt: WorkerThread):
         wt.verifyThreadSelf()
         self._workerThread = wt  # type: ignore
@@ -1456,36 +1518,25 @@ class Task():
             "[-] executing task {}...".format(self.__class__.__name__))
 
         self._err = None
-        self._execStats.beginTaskType(
-            self.__class__.__name__)  # mark beginning
+        self._execStats.beginTaskType(self.__class__.__name__)  # mark beginning
+        errno2 = None
         try:
             self._executeInternal(te, wt)  # TODO: no return value?
         except taos.error.ProgrammingError as err:
-            errno2 = err.errno if (
-                err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
             if (gConfig.continue_on_exception):  # user choose to continue
-                self.logDebug(
-                    "[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                        errno2, err, self._lastSql))
+                self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
+                        errno2, err, wt.getDbConn().getLastSql()))
                 self._err = err
-            elif (errno2 in [
-                0x05,  # TSDB_CODE_RPC_NOT_READY
-                0x200, 0x360, 0x362, 0x36A, 0x36B, 0x36D,
-                0x381, 0x380, 0x383,
-                0x386,  # DB is being dropped?!
-                0x503,
-                0x510,  # vnode not in ready state
-                0x600,
-                1000  # REST catch-all error
-            ]):  # allowed errors
-                self.logDebug(
-                    "[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                        errno2, err, self._lastSql))
+            elif self._isErrAcceptable(errno2, err.__str__()):
+                self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
+                        errno2, err, wt.getDbConn().getLastSql()))
                 print("_", end="", flush=True)
                 self._err = err
-            else:
-                errMsg = "[=] Unexpected Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format(
-                    errno2, err, self._lastSql)
+            else: # not an acceptable error
+                errMsg = "[=] Unexpected Taos library exception ({}): errno=0x{:X}, msg: {}, SQL: {}".format(
+                    self.__class__.__name__,
+                    errno2, err, wt.getDbConn().getLastSql())
                 self.logDebug(errMsg)
                 if gConfig.debug:
                     # raise # so that we see full stack
@@ -1509,25 +1560,22 @@ class Task():
         except BaseException:
             self.logDebug(
                 "[=] Unexpected exception, SQL: {}".format(
-                    self._lastSql))
+                    wt.getDbConn().getLastSql()))
             raise
         self._execStats.endTaskType(self.__class__.__name__, self.isSuccess())
 
         self.logDebug("[X] task execution completed, {}, status: {}".format(
             self.__class__.__name__, "Success" if self.isSuccess() else "Failure"))
         # TODO: merge with above.
-        self._execStats.incExecCount(self.__class__.__name__, self.isSuccess())
+        self._execStats.incExecCount(self.__class__.__name__, self.isSuccess(), errno2)
 
     def execSql(self, sql):
-        self._lastSql = sql
         return self._dbManager.execute(sql)
 
     def execWtSql(self, wt: WorkerThread, sql):  # execute an SQL on the worker thread
-        self._lastSql = sql
         return wt.execSql(sql)
 
     def queryWtSql(self, wt: WorkerThread, sql):  # execute an SQL on the worker thread
-        self._lastSql = sql
         return wt.querySql(sql)
 
     def getQueryResult(self, wt: WorkerThread):  # execute an SQL on the worker thread
@@ -1542,6 +1590,7 @@ class ExecutionStats:
         self._lock = threading.Lock()
         self._firstTaskStartTime = None
         self._execStartTime = None
+        self._errors = {}
         self._elapsedTime = 0.0  # total elapsed time
         self._accRunTime = 0.0  # accumulated run time
 
@@ -1561,13 +1610,18 @@ class ExecutionStats:
     def endExec(self):
         self._elapsedTime = time.time() - self._execStartTime
 
-    def incExecCount(self, klassName, isSuccess):  # TODO: add a lock here
+    def incExecCount(self, klassName, isSuccess, eno=None):  # TODO: add a lock here
         if klassName not in self._execTimes:
             self._execTimes[klassName] = [0, 0]
         t = self._execTimes[klassName]  # tuple for the data
         t[0] += 1  # index 0 has the "total" execution times
         if isSuccess:
             t[1] += 1  # index 1 has the "success" execution times
+        if eno != None:             
+            if klassName not in self._errors:
+                self._errors[klassName] = {}
+            errors = self._errors[klassName]
+            errors[eno] = errors[eno]+1 if eno in errors else 1
 
     def beginTaskType(self, klassName):
         with self._lock:
@@ -1597,7 +1651,14 @@ class ExecutionStats:
         execTimesAny = 0
         for k, n in self._execTimes.items():
             execTimesAny += n[0]
-            logger.info("|    {0:<24}: {1}/{2}".format(k, n[1], n[0]))
+            errStr = None
+            if k in self._errors:
+                errors = self._errors[k]
+                # print("errors = {}".format(errors))
+                errStrs = ["0x{:X}:{}".format(eno, n) for (eno, n) in errors.items()]
+                # print("error strings = {}".format(errStrs))
+                errStr = ", ".join(errStrs) 
+            logger.info("|    {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr))
 
         logger.info(
             "| Total Tasks Executed (success or not): {} ".format(execTimesAny))
@@ -1649,7 +1710,7 @@ class StateTransitionTask(Task):
 
     @classmethod
     def getRegTableName(cls, i):
-        return "db.reg_table_{}".format(i)
+        return "reg_table_{}".format(i)
 
     def execute(self, wt: WorkerThread):
         super().execute(wt)
@@ -1696,15 +1757,94 @@ class TaskCreateSuperTable(StateTransitionTask):
             logger.debug("Skipping task, no DB yet")
             return
 
-        tblName = self._dbManager.getFixedSuperTableName()
+        sTable = self._dbManager.getFixedSuperTable()
         # wt.execSql("use db")    # should always be in place
-        self.execWtSql(
-            wt,
-            "create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
+        sTable.create(wt.getDbConn(), {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'})
+        # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
         # No need to create the regular tables, INSERT will do that
         # automatically
 
 
+class TdSuperTable:
+    def __init__(self, stName):
+        self._stName = stName
+
+    def create(self, dbc, cols: dict, tags: dict):
+        sql = "CREATE TABLE db.{} ({}) TAGS ({})".format(
+            self._stName,
+            ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]),
+            ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
+            )
+        dbc.execute(sql)        
+
+    def getRegTables(self, dbc: DbConn):
+        try:
+            dbc.query("select TBNAME from db.{}".format(self._stName))  # TODO: analyze result set later            
+        except taos.error.ProgrammingError as err:                    
+            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+            logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
+            raise
+
+        qr = dbc.getQueryResult()
+        return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation
+
+    def hasRegTables(self, dbc: DbConn):
+        return dbc.query("SELECT * FROM db.{}".format(self._stName)) > 0
+
+    def ensureTable(self, dbc: DbConn, regTableName: str):
+        sql = "select tbname from {} where tbname in ('{}')".format(self._stName, regTableName)
+        if dbc.query(sql) >= 1 : # reg table exists already
+            return
+        sql = "CREATE TABLE {} USING {} tags ({})".format(
+            regTableName, self._stName, self._getTagStrForSql(dbc)
+        )
+        dbc.execute(sql)
+
+    def _getTagStrForSql(self, dbc) :
+        tags = self._getTags(dbc)
+        tagStrs = []
+        for tagName in tags: 
+            tagType = tags[tagName]
+            if tagType == 'BINARY':
+                tagStrs.append("'Beijing-Shanghai-LosAngeles'")
+            elif tagType == 'FLOAT':
+                tagStrs.append('9.9')
+            elif tagType == 'INT':
+                tagStrs.append('88')
+            else:
+                raise RuntimeError("Unexpected tag type: {}".format(tagType))
+        return ", ".join(tagStrs)
+
+    def _getTags(self, dbc) -> dict:
+        dbc.query("DESCRIBE {}".format(self._stName))
+        stCols = dbc.getQueryResult()
+        # print(stCols)
+        ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type
+        # print("Tags retrieved: {}".format(ret))
+        return ret
+
+    def addTag(self, dbc, tagName, tagType):
+        if tagName in self._getTags(dbc): # already 
+            return
+        # sTable.addTag("extraTag", "int")
+        sql = "alter table db.{} add tag {} {}".format(self._stName, tagName, tagType)
+        dbc.execute(sql)
+
+    def dropTag(self, dbc, tagName):
+        if not tagName in self._getTags(dbc): # don't have this tag
+            return
+        sql = "alter table db.{} drop tag {}".format(self._stName, tagName)
+        dbc.execute(sql)
+
+    def changeTag(self, dbc, oldTag, newTag):
+        tags = self._getTags(dbc)
+        if not oldTag in tags: # don't have this tag
+            return
+        if newTag in tags: # already have this tag
+            return
+        sql = "alter table db.{} change tag {} {}".format(self._stName, oldTag, newTag)
+        dbc.execute(sql)
+
 class TaskReadData(StateTransitionTask):
     @classmethod
     def getEndState(cls):
@@ -1715,23 +1855,24 @@ class TaskReadData(StateTransitionTask):
         return state.canReadData()
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        sTbName = self._dbManager.getFixedSuperTableName()
-        self.queryWtSql(wt, "select TBNAME from db.{}".format(
-            sTbName))  # TODO: analyze result set later
+        sTable = self._dbManager.getFixedSuperTable()
 
         if random.randrange(
                 5) == 0:  # 1 in 5 chance, simulate a broken connection. TODO: break connection in all situations
             wt.getDbConn().close()
             wt.getDbConn().open()
-        else:
-            # wt.getDbConn().getQueryResult()
-            rTables = self.getQueryResult(wt)
-            # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0])))
-            for rTbName in rTables:  # regular tables
-                self.execWtSql(wt, "select * from db.{}".format(rTbName[0]))
-
-        # tdSql.query(" cars where tbname in ('carzero', 'carone')")
-
+        
+        for rTbName in sTable.getRegTables(wt.getDbConn()):  # regular tables
+            aggExpr = Dice.choice(['*', 'count(*)', 'avg(speed)', 
+                # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable
+                'sum(speed)', 'stddev(speed)', 
+                'min(speed)', 'max(speed)', 'first(speed)', 'last(speed)']) # TODO: add more from 'top'
+            try:
+                self.execWtSql(wt, "select {} from db.{}".format(aggExpr, rTbName))
+            except taos.error.ProgrammingError as err:                    
+                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+                logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, wt.getDbConn().getLastSql()))
+                raise
 
 class TaskDropSuperTable(StateTransitionTask):
     @classmethod
@@ -1789,20 +1930,55 @@ class TaskAlterTags(StateTransitionTask):
         return state.canDropFixedSuperTable()  # if we can drop it, we can alter tags
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        tblName = self._dbManager.getFixedSuperTableName()
+        # tblName = self._dbManager.getFixedSuperTableName()
+        dbc = wt.getDbConn()
+        sTable = self._dbManager.getFixedSuperTable()
         dice = Dice.throw(4)
         if dice == 0:
-            sql = "alter table db.{} add tag extraTag int".format(tblName)
+            sTable.addTag(dbc, "extraTag", "int")
+            # sql = "alter table db.{} add tag extraTag int".format(tblName)
         elif dice == 1:
-            sql = "alter table db.{} drop tag extraTag".format(tblName)
+            sTable.dropTag(dbc, "extraTag")
+            # sql = "alter table db.{} drop tag extraTag".format(tblName)
         elif dice == 2:
-            sql = "alter table db.{} drop tag newTag".format(tblName)
+            sTable.dropTag(dbc, "newTag")
+            # sql = "alter table db.{} drop tag newTag".format(tblName)
         else:  # dice == 3
-            sql = "alter table db.{} change tag extraTag newTag".format(
-                tblName)
+            sTable.changeTag(dbc, "extraTag", "newTag")
+            # sql = "alter table db.{} change tag extraTag newTag".format(tblName)
 
-        self.execWtSql(wt, sql)
+class TaskRestartService(StateTransitionTask):
+    _isRunning = False
+    _classLock = threading.Lock()
 
+    @classmethod
+    def getEndState(cls):
+        return None  # meaning doesn't affect state
+
+    @classmethod
+    def canBeginFrom(cls, state: AnyState):
+        if gConfig.auto_start_service:
+            return state.canDropFixedSuperTable()  # Basicallly when we have the super table
+        return False # don't run this otherwise
+
+    CHANCE_TO_RESTART_SERVICE = 100
+    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
+        if not gConfig.auto_start_service: # only execute when we are in -a mode
+            print("_a", end="", flush=True)
+            return
+
+        with self._classLock:
+            if self._isRunning:
+                print("Skipping restart task, another running already")
+                return
+            self._isRunning = True
+
+        if Dice.throw(self.CHANCE_TO_RESTART_SERVICE) == 0: # 1 in N chance
+            dbc = wt.getDbConn()
+            dbc.execute("show databases") # simple delay, align timing with other workers
+            gSvcMgr.restart()
+
+        self._isRunning = False
 
 class TaskAddData(StateTransitionTask):
     # Track which table is being actively worked on
@@ -1833,39 +2009,31 @@ class TaskAddData(StateTransitionTask):
         return state.canAddData()
 
     def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        ds = self._dbManager
-        # wt.execSql("use db") # TODO: seems to be an INSERT bug to require
-        # this
-        tblSeq = list(
-            range(
+        ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
+        tblSeq = list(range(
                 self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
         random.shuffle(tblSeq)
         for i in tblSeq:
             if (i in self.activeTable):  # wow already active
-                # logger.info("Concurrent data insertion into table: {}".format(i))
-                # print("ct({})".format(i), end="", flush=True) # Concurrent
-                # insertion into table
-                print("x", end="", flush=True)
+                print("x", end="", flush=True) # concurrent insertion
             else:
                 self.activeTable.add(i)  # marking it active
-            # No need to shuffle data sequence, unless later we decide to do
-            # non-increment insertion
-            regTableName = self.getRegTableName(
-                i)  # "db.reg_table_{}".format(i)
-            for j in range(
-                    self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS):  # number of records per table
+            
+            sTable = ds.getFixedSuperTable()
+            regTableName = self.getRegTableName(i)  # "db.reg_table_{}".format(i)
+            sTable.ensureTable(wt.getDbConn(), regTableName)  # Ensure the table exists           
+           
+            for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS):  # number of records per table
                 nextInt = ds.getNextInt()
                 if gConfig.record_ops:
                     self.prepToRecordOps()
-                    self.fAddLogReady.write(
-                        "Ready to write {} to {}\n".format(
-                            nextInt, regTableName))
+                    self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
                     self.fAddLogReady.flush()
                     os.fsync(self.fAddLogReady)
-                sql = "insert into {} using {} tags ('{}', {}) values ('{}', {});".format(
+                sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
                     regTableName,
-                    ds.getFixedSuperTableName(),
-                    ds.getNextBinary(), ds.getNextFloat(),
+                    # ds.getFixedSuperTableName(),
+                    # ds.getNextBinary(), ds.getNextFloat(),
                     ds.getNextTick(), nextInt)
                 self.execWtSql(wt, sql)
                 # Successfully wrote the data into the DB, let's record it
@@ -1912,6 +2080,10 @@ class Dice():
             raise RuntimeError("Cannot throw dice before seeding it")
         return random.randrange(start, stop)
 
+    @classmethod
+    def choice(cls, cList):
+        return random.choice(cList)
+
 
 class LoggingFilter(logging.Filter):
     def filter(self, record: logging.LogRecord):
@@ -1934,14 +2106,16 @@ class MyLoggingAdapter(logging.LoggerAdapter):
 class SvcManager:
     def __init__(self):
         print("Starting TDengine Service Manager")
-        signal.signal(signal.SIGTERM, self.sigIntHandler)
-        signal.signal(signal.SIGINT, self.sigIntHandler)
-        signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
+        # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec
+        # signal.signal(signal.SIGINT, self.sigIntHandler)
+        # signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
 
         self.inSigHandler = False
         # self._status = MainExec.STATUS_RUNNING # set inside
         # _startTaosService()
         self.svcMgrThread = None
+        self._lock = threading.Lock()
+        self._isRestarting = False
 
     def _doMenu(self):
         choice = ""
@@ -1976,23 +2150,22 @@ class SvcManager:
             self.sigHandlerResume()
         elif choice == "2":
             self.stopTaosService()
-        elif choice == "3":
-            self.stopTaosService()
-            self.startTaosService()
+        elif choice == "3": # Restart
+            self.restart()
         else:
             raise RuntimeError("Invalid menu choice: {}".format(choice))
 
         self.inSigHandler = False
 
     def sigIntHandler(self, signalNumber, frame):
-        print("Sig INT Handler starting...")
+        print("SvcManager: INT Signal Handler starting...")
         if self.inSigHandler:
             print("Ignoring repeated SIG_INT...")
             return
         self.inSigHandler = True
 
         self.stopTaosService()
-        print("INT signal handler returning...")
+        print("SvcManager: INT Signal Handler returning...")
         self.inSigHandler = False
 
     def sigHandlerResume(self):
@@ -2005,44 +2178,78 @@ class SvcManager:
                 self.svcMgrThread = None  # no more
 
     def _procIpcAll(self):
-        while self.svcMgrThread:  # for as long as the svc mgr thread is still here
-            self.svcMgrThread.procIpcBatch()  # regular processing,
+        while self.isRunning() or self.isRestarting() :  # for as long as the svc mgr thread is still here
+            if self.isRunning():
+                self.svcMgrThread.procIpcBatch()  # regular processing,
+                self._checkServiceManagerThread()
+            elif self.isRetarting():
+                print("Service restarting...")
             time.sleep(0.5)  # pause, before next round
-            self._checkServiceManagerThread()
         print(
             "Service Manager Thread (with subprocess) has ended, main thread now exiting...")
 
     def startTaosService(self):
-        if self.svcMgrThread:
-            raise RuntimeError(
-                "Cannot start TAOS service when one may already be running")
-        self.svcMgrThread = ServiceManagerThread()  # create the object
-        self.svcMgrThread.start()
-        print("TAOS service started, printing out output...")
-        self.svcMgrThread.procIpcBatch(
-            trimToTarget=10,
-            forceOutput=True)  # for printing 10 lines
-        print("TAOS service started")
+        with self._lock:
+            if self.svcMgrThread:
+                raise RuntimeError("Cannot start TAOS service when one may already be running")
+
+            # Find if there's already a taosd service, and then kill it
+            for proc in psutil.process_iter():
+                if proc.name() == 'taosd':
+                    print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupe")
+                    time.sleep(2.0)
+                    proc.kill()
+                # print("Process: {}".format(proc.name()))
+            
+            self.svcMgrThread = ServiceManagerThread()  # create the object
+            self.svcMgrThread.start()
+            print("Attempting to start TAOS service started, printing out output...")
+            self.svcMgrThread.procIpcBatch(
+                trimToTarget=10,
+                forceOutput=True)  # for printing 10 lines
+            print("TAOS service started")
 
     def stopTaosService(self, outputLines=20):
-        print("Terminating Service Manager Thread (SMT) execution...")
-        if not self.svcMgrThread:
-            raise RuntimeError("Unexpected empty svc mgr thread")
-        self.svcMgrThread.stop()
-        if self.svcMgrThread.isStopped():
-            self.svcMgrThread.procIpcBatch(outputLines)  # one last time
-            self.svcMgrThread = None
-            print("----- End of TDengine Service Output -----\n")
-            print("SMT execution terminated")
-        else:
-            print("WARNING: SMT did not terminate as expected")
+        with self._lock:
+            if not self.isRunning():
+                logger.warning("Cannot stop TAOS service, not running")
+                return
+
+            print("Terminating Service Manager Thread (SMT) execution...")
+            self.svcMgrThread.stop()
+            if self.svcMgrThread.isStopped():
+                self.svcMgrThread.procIpcBatch(outputLines)  # one last time
+                self.svcMgrThread = None
+                print("----- End of TDengine Service Output -----\n")
+                print("SMT execution terminated")
+            else:
+                print("WARNING: SMT did not terminate as expected")
 
     def run(self):
         self.startTaosService()
-        self._procIpcAll()  # pump/process all the messages
-        if self.svcMgrThread:  # if sig handler hasn't destroyed it by now
+        self._procIpcAll()  # pump/process all the messages, may encounter SIG + restart
+        if self.isRunning():  # if sig handler hasn't destroyed it by now
             self.stopTaosService()  # should have started already
 
+    def restart(self):
+        if self._isRestarting:
+            logger.warning("Cannot restart service when it's already restarting")
+            return
+
+        self._isRestarting = True
+        if self.isRunning():
+            self.stopTaosService()
+        else:
+            logger.warning("Service not running when restart requested")
+
+        self.startTaosService()
+        self._isRestarting = False
+
+    def isRunning(self):
+        return self.svcMgrThread != None
+
+    def isRestarting(self):
+        return self._isRestarting
 
 class ServiceManagerThread:
     MAX_QUEUE_SIZE = 10000
@@ -2094,6 +2301,7 @@ class ServiceManagerThread:
                 logger.info("[] TDengine service READY to process requests")
                 return  # now we've started
         # TODO: handle this better?
+        self.procIpcBatch(20, True) # display output before cronking out, trim to last 20 msgs, force output
         raise RuntimeError("TDengine service did not start successfully")
 
     def stop(self):
@@ -2196,7 +2404,10 @@ class ServiceManagerThread:
 
             if self._status == MainExec.STATUS_STARTING:  # we are starting, let's see if we have started
                 if line.find(self.TD_READY_MSG) != -1:  # found
-                    self._status = MainExec.STATUS_RUNNING
+                    logger.info("Waiting for the service to become FULLY READY")
+                    time.sleep(1.0) # wait for the server to truly start. TODO: remove this
+                    logger.info("Service is now FULLY READY")   
+                    self._status = MainExec.STATUS_RUNNING                 
 
             # Trim the queue if necessary: TODO: try this 1 out of 10 times
             self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10)  # trim to 90% size
@@ -2242,6 +2453,21 @@ class TdeSubProcess:
         taosdPath = self.getBuildPath() + "/build/bin/taosd"
         cfgPath = self.getBuildPath() + "/test/cfg"
 
+        # Delete the log files
+        logPath = self.getBuildPath() + "/test/log"
+        # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397
+        # filelist = [ f for f in os.listdir(logPath) ] # if f.endswith(".bak") ]
+        # for f in filelist:
+        #     filePath = os.path.join(logPath, f)
+        #     print("Removing log file: {}".format(filePath))
+        #     os.remove(filePath)        
+        if os.path.exists(logPath):
+            logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S')
+            logger.info("Saving old log files to: {}".format(logPathSaved))
+            os.rename(logPath, logPathSaved)
+        # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms
+            
+
         svcCmd = [taosdPath, '-c', cfgPath]
         # svcCmd = ['vmstat', '1']
         if self.subProcess:  # already there
@@ -2275,16 +2501,46 @@ class TdeSubProcess:
                 print("TDengine service process terminated successfully from SIG_INT")
                 self.subProcess = None
 
+class ThreadStacks: # stack info for all threads
+    def __init__(self):
+        self._allStacks = {}
+        allFrames = sys._current_frames()
+        for th in threading.enumerate():                        
+            stack = traceback.extract_stack(allFrames[th.ident])     
+            self._allStacks[th.native_id] = stack
+
+    def print(self, filteredEndName = None, filterInternal = False):
+        for thNid, stack in self._allStacks.items(): # for each thread            
+            lastFrame = stack[-1]
+            if filteredEndName: # we need to filter out stacks that match this name                
+                if lastFrame.name == filteredEndName : # end did not match
+                    continue
+            if filterInternal:
+                if lastFrame.name in ['wait', 'invoke_excepthook', 
+                    '_wait', # The Barrier exception
+                    'svcOutputReader', # the svcMgr thread
+                    '__init__']: # the thread that extracted the stack
+                    continue # ignore
+            # Now print
+            print("\n<----- Thread Info for ID: {}".format(thNid))
+            for frame in stack:
+                # print(frame)
+                print("File {filename}, line {lineno}, in {name}".format(
+                    filename=frame.filename, lineno=frame.lineno, name=frame.name))
+                print("    {}".format(frame.line))
+            print("-----> End of Thread Info\n")
 
 class ClientManager:
     def __init__(self):
         print("Starting service manager")
-        signal.signal(signal.SIGTERM, self.sigIntHandler)
-        signal.signal(signal.SIGINT, self.sigIntHandler)
+        # signal.signal(signal.SIGTERM, self.sigIntHandler)
+        # signal.signal(signal.SIGINT, self.sigIntHandler)
 
         self._status = MainExec.STATUS_RUNNING
         self.tc = None
 
+        self.inSigHandler = False
+
     def sigIntHandler(self, signalNumber, frame):
         if self._status != MainExec.STATUS_RUNNING:
             print("Repeated SIGINT received, forced exit...")
@@ -2292,9 +2548,50 @@ class ClientManager:
             sys.exit(-1)
         self._status = MainExec.STATUS_STOPPING  # immediately set our status
 
-        print("Terminating program...")
+        print("ClientManager: Terminating program...")
         self.tc.requestToStop()
 
+    def _doMenu(self):
+        choice = ""
+        while True:
+            print("\nInterrupting Client Program, Choose an Action: ")
+            print("1: Resume")
+            print("2: Terminate")
+            print("3: Show Threads")
+            # Remember to update the if range below
+            # print("Enter Choice: ", end="", flush=True)
+            while choice == "":
+                choice = input("Enter Choice: ")
+                if choice != "":
+                    break  # done with reading repeated input
+            if choice in ["1", "2", "3"]:
+                break  # we are done with whole method
+            print("Invalid choice, please try again.")
+            choice = ""  # reset
+        return choice
+
+    def sigUsrHandler(self, signalNumber, frame):
+        print("Interrupting main thread execution upon SIGUSR1")
+        if self.inSigHandler:  # already
+            print("Ignoring repeated SIG_USR1...")
+            return  # do nothing if it's already not running
+        self.inSigHandler = True
+
+        choice = self._doMenu()
+        if choice == "1":
+            print("Resuming execution...")
+            time.sleep(1.0)
+        elif choice == "2":
+            print("Not implemented yet")
+            time.sleep(1.0)
+        elif choice == "3":
+            ts = ThreadStacks()
+            ts.print()
+        else:
+            raise RuntimeError("Invalid menu choice: {}".format(choice))
+
+        self.inSigHandler = False
+
     def _printLastNumbers(self):  # to verify data durability
         dbManager = DbManager(resetDb=False)
         dbc = dbManager.getDbConn()
@@ -2327,21 +2624,17 @@ class ClientManager:
     def prepare(self):
         self._printLastNumbers()
 
-    def run(self):
-        if gConfig.auto_start_service:
-            svcMgr = SvcManager()
-            svcMgr.startTaosService()
-
+    def run(self, svcMgr):    
         self._printLastNumbers()
 
         dbManager = DbManager()  # Regular function
         thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
         self.tc = ThreadCoordinator(thPool, dbManager)
-
+        
         self.tc.run()
         # print("exec stats: {}".format(self.tc.getExecStats()))
         # print("TC failed = {}".format(self.tc.isFailed()))
-        if gConfig.auto_start_service:
+        if svcMgr: # gConfig.auto_start_service:
             svcMgr.stopTaosService()
         # Print exec status, etc., AFTER showing messages from the server
         self.conclude()
@@ -2353,25 +2646,58 @@ class ClientManager:
         self.tc.printStats()
         self.tc.getDbManager().cleanUp()
 
-
 class MainExec:
     STATUS_STARTING = 1
     STATUS_RUNNING = 2
     STATUS_STOPPING = 3
     STATUS_STOPPED = 4
 
-    @classmethod
-    def runClient(cls):
-        clientManager = ClientManager()
-        return clientManager.run()
+    def __init__(self):        
+        self._clientMgr = None
+        self._svcMgr = None
 
-    @classmethod
-    def runService(cls):
-        svcManager = SvcManager()
-        svcManager.run()
+        signal.signal(signal.SIGTERM, self.sigIntHandler)
+        signal.signal(signal.SIGINT,  self.sigIntHandler)
+        signal.signal(signal.SIGUSR1, self.sigUsrHandler)  # different handler!
 
-    @classmethod
-    def runTemp(cls):  # for debugging purposes
+    def sigUsrHandler(self, signalNumber, frame):
+        if self._clientMgr:
+            self._clientMgr.sigUsrHandler(signalNumber, frame)
+        elif self._svcMgr: # Only if no client mgr, we are running alone
+            self._svcMgr.sigUsrHandler(signalNumber, frame)
+        
+    def sigIntHandler(self, signalNumber, frame):
+        if self._svcMgr:
+            self._svcMgr.sigIntHandler(signalNumber, frame)
+        if self._clientMgr:
+            self._clientMgr.sigIntHandler(signalNumber, frame)
+
+    def runClient(self):
+        global gSvcMgr
+        if gConfig.auto_start_service:
+            self._svcMgr = SvcManager()
+            gSvcMgr = self._svcMgr # hack alert
+            self._svcMgr.startTaosService() # we start, don't run
+        
+        self._clientMgr = ClientManager()
+        ret = None
+        try: 
+            ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside
+        except requests.exceptions.ConnectionError as err:
+            logger.warning("Failed to open REST connection to DB")
+            # don't raise
+        return ret
+
+    def runService(self):
+        global gSvcMgr
+        self._svcMgr = SvcManager()
+        gSvcMgr = self._svcMgr # save it in a global variable TODO: hack alert
+
+        self._svcMgr.run() # run to some end state
+        self._svcMgr = None 
+        gSvcMgr = None        
+
+    def runTemp(self):  # for debugging purposes
         # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix
         # dbc = dbState.getDbConn()
         # sTbName = dbState.getFixedSuperTableName()
@@ -2527,10 +2853,11 @@ def main():
     Dice.seed(0)  # initial seeding of dice
 
     # Run server or client
+    mExec = MainExec()
     if gConfig.run_tdengine:  # run server
-        MainExec.runService()
+        mExec.runService()
     else:
-        return MainExec.runClient()
+        return mExec.runClient()
 
 
 if __name__ == "__main__":
diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py
new file mode 100644
index 0000000000..ec3865f4f2
--- /dev/null
+++ b/tests/pytest/util/dnodes-default.py
@@ -0,0 +1,502 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import os.path
+import subprocess
+from util.log import *
+
+
+class TDSimClient:
+    def __init__(self):
+        self.testCluster = False
+
+        self.cfgDict = {
+            "numOfLogLines": "100000000",
+            "numOfThreadsPerCore": "2.0",
+            "locale": "en_US.UTF-8",
+            "charset": "UTF-8",
+            "asyncLog": "0",
+            "minTablesPerVnode": "4",
+            "maxTablesPerVnode": "1000",
+            "tableIncStepPerVnode": "10000",
+            "maxVgroupsPerDb": "1000",
+            "sdbDebugFlag": "143",
+            "rpcDebugFlag": "135",
+            "tmrDebugFlag": "131",
+            "cDebugFlag": "135",
+            "udebugFlag": "135",
+            "jnidebugFlag": "135",
+            "qdebugFlag": "135",
+            }
+    def init(self, path):
+        self.__init__()
+        self.path = path
+
+    def getLogDir(self):
+        self.logDir = "%s/sim/psim/log" % (self.path)
+        return self.logDir
+
+    def getCfgDir(self):
+        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+        return self.cfgDir
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def addExtraCfg(self, option, value):
+        self.cfgDict.update({option: value})
+
+    def cfg(self, option, value):
+        cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def deploy(self):
+        self.logDir = "%s/sim/psim/log" % (self.path)
+        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+        self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path)
+
+        cmd = "rm -rf " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+    
+        cmd = "mkdir -p " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "touch " + self.cfgPath
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        if self.testCluster:
+            self.cfg("masterIp", "192.168.0.1")
+            self.cfg("secondIp", "192.168.0.2")
+        self.cfg("logDir", self.logDir)
+
+        for key, value in self.cfgDict.items():
+            self.cfg(key, value)
+
+        tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
+
+
+class TDDnode:
+    def __init__(self, index):
+        self.index = index
+        self.running = 0
+        self.deployed = 0
+        self.testCluster = False
+        self.valgrind = 0
+
+    def init(self, path):
+        self.path = path
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def setValgrind(self, value):
+        self.valgrind = value
+
+    def getDataSize(self):
+        totalSize = 0
+
+        if (self.deployed == 1):
+            for dirpath, dirnames, filenames in os.walk(self.dataDir):
+                for f in filenames:
+                    fp = os.path.join(dirpath, f)
+
+                    if not os.path.islink(fp):
+                        totalSize = totalSize + os.path.getsize(fp)
+
+        return totalSize
+
+    def deploy(self):
+        self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
+        self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index)
+        self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index)
+        self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (
+            self.path, self.index)
+
+        cmd = "rm -rf " + self.dataDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.dataDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "touch " + self.cfgPath
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        if self.testCluster:
+            self.startIP()
+
+        if self.testCluster:
+            self.cfg("masterIp", "192.168.0.1")
+            self.cfg("secondIp", "192.168.0.2")
+            self.cfg("publicIp", "192.168.0.%d" % (self.index))
+            self.cfg("internalIp", "192.168.0.%d" % (self.index))
+            self.cfg("privateIp", "192.168.0.%d" % (self.index))
+        self.cfg("dataDir", self.dataDir)
+        self.cfg("logDir", self.logDir)
+        self.cfg("numOfLogLines", "100000000")
+        self.cfg("mnodeEqualVnodeNum", "0")
+        self.cfg("walLevel", "2")
+        self.cfg("fsync", "1000")
+        self.cfg("statusInterval", "1")
+        self.cfg("numOfMnodes", "3")
+        self.cfg("numOfThreadsPerCore", "2.0")
+        self.cfg("monitor", "0")
+        self.cfg("maxVnodeConnections", "30000")
+        self.cfg("maxMgmtConnections", "30000")
+        self.cfg("maxMeterConnections", "30000")
+        self.cfg("maxShellConns", "30000")
+        self.cfg("locale", "en_US.UTF-8")
+        self.cfg("charset", "UTF-8")
+        self.cfg("asyncLog", "0")
+        self.cfg("anyIp", "0")
+        self.cfg("dDebugFlag", "135")
+        self.cfg("mDebugFlag", "135")
+        self.cfg("sdbDebugFlag", "135")
+        self.cfg("rpcDebugFlag", "135")
+        self.cfg("tmrDebugFlag", "131")
+        self.cfg("cDebugFlag", "135")
+        self.cfg("httpDebugFlag", "135")
+        self.cfg("monitorDebugFlag", "135")
+        self.cfg("udebugFlag", "135")
+        self.cfg("jnidebugFlag", "135")
+        self.cfg("qdebugFlag", "135")
+        self.deployed = 1
+        tdLog.debug(
+            "dnode:%d is deployed and configured by %s" %
+            (self.index, self.cfgPath))
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def start(self):
+        buildPath = self.getBuildPath()
+
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+
+        binPath = buildPath + "/build/bin/taosd"
+
+        if self.deployed == 0:
+            tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+        if self.valgrind == 0:
+            cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
+                binPath, self.cfgDir)
+        else:
+            valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
+
+            cmd = "nohup %s %s -c %s 2>&1 & " % (
+                valgrindCmdline, binPath, self.cfgDir)
+
+            print(cmd)
+
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+        self.running = 1
+        tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
+
+        tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
+        time.sleep(5)
+
+    def stop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        if self.running != 0:
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+            while(processID):
+                killCmd = "kill -INT %s > /dev/null 2>&1" % processID
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(
+                    psCmd, shell=True).decode("utf-8")
+            for port in range(6030, 6041):
+                fuserCmd = "fuser -k -n tcp %d" % port
+                os.system(fuserCmd)
+            if self.valgrind:
+                time.sleep(2)
+
+            self.running = 0
+            tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
+
+    def forcestop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        if self.running != 0:
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+            while(processID):
+                killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(
+                    psCmd, shell=True).decode("utf-8")
+            for port in range(6030, 6041):
+                fuserCmd = "fuser -k -n tcp %d" % port
+                os.system(fuserCmd)
+            if self.valgrind:
+                time.sleep(2)
+
+            self.running = 0
+            tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
+
+    def startIP(self):
+        cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def stopIP(self):
+        cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
+            self.index, self.index)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def cfg(self, option, value):
+        cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def getDnodeRootDir(self, index):
+        dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index)
+        return dnodeRootDir
+
+    def getDnodesRootDir(self):
+        dnodesRootDir = "%s/sim/psim" % (self.path)
+        return dnodesRootDir
+
+
+class TDDnodes:
+    def __init__(self):
+        self.dnodes = []
+        self.dnodes.append(TDDnode(1))
+        self.dnodes.append(TDDnode(2))
+        self.dnodes.append(TDDnode(3))
+        self.dnodes.append(TDDnode(4))
+        self.dnodes.append(TDDnode(5))
+        self.dnodes.append(TDDnode(6))
+        self.dnodes.append(TDDnode(7))
+        self.dnodes.append(TDDnode(8))
+        self.dnodes.append(TDDnode(9))
+        self.dnodes.append(TDDnode(10))
+        self.simDeployed = False
+
+    def init(self, path):
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        binPath = os.path.dirname(os.path.realpath(__file__))
+        binPath = binPath + "/../../../debug/"
+        tdLog.debug("binPath %s" % (binPath))
+        binPath = os.path.realpath(binPath)
+        tdLog.debug("binPath real path %s" % (binPath))
+
+        # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
+        # tdLog.debug(cmd)
+        # os.system(cmd)
+
+        # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
+        # if os.system(cmd) != 0 :
+        #  tdLog.exit(cmd)
+        # tdLog.debug("execute %s" % (cmd))
+
+        # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+        # tdLog.debug("execute %s" % (cmd))
+
+        if path == "":
+            # self.path = os.path.expanduser('~')
+            self.path = os.path.abspath(binPath + "../../")
+        else:
+            self.path = os.path.realpath(path)
+
+        for i in range(len(self.dnodes)):
+            self.dnodes[i].init(self.path)
+
+        self.sim = TDSimClient()
+        self.sim.init(self.path)
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def setValgrind(self, value):
+        self.valgrind = value
+
+    def deploy(self, index):
+        self.sim.setTestCluster(self.testCluster)
+
+        if (self.simDeployed == False):
+            self.sim.deploy()
+            self.simDeployed = True
+
+        self.check(index)
+        self.dnodes[index - 1].setTestCluster(self.testCluster)
+        self.dnodes[index - 1].setValgrind(self.valgrind)
+        self.dnodes[index - 1].deploy()
+
+    def cfg(self, index, option, value):
+        self.check(index)
+        self.dnodes[index - 1].cfg(option, value)
+
+    def start(self, index):
+        self.check(index)
+        self.dnodes[index - 1].start()
+
+    def stop(self, index):
+        self.check(index)
+        self.dnodes[index - 1].stop()
+
+    def getDataSize(self, index):
+        self.check(index)
+        return self.dnodes[index - 1].getDataSize()
+
+    def forcestop(self, index):
+        self.check(index)
+        self.dnodes[index - 1].forcestop()
+
+    def startIP(self, index):
+        self.check(index)
+
+        if self.testCluster:
+            self.dnodes[index - 1].startIP()
+
+    def stopIP(self, index):
+        self.check(index)
+
+        if self.dnodes[index - 1].testCluster:
+            self.dnodes[index - 1].stopIP()
+
+    def check(self, index):
+        if index < 1 or index > 10:
+            tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
+
+    def stopAll(self):
+        tdLog.info("stop all dnodes")
+        for i in range(len(self.dnodes)):
+            self.dnodes[i].stop()
+
+        psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        if processID:
+            cmd = "sudo systemctl stop taosd"
+            os.system(cmd)
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+
+    def getDnodesRootDir(self):
+        dnodesRootDir = "%s/sim" % (self.path)
+        return dnodesRootDir
+
+    def getSimCfgPath(self):
+        return self.sim.getCfgDir()
+
+    def getSimLogPath(self):
+        return self.sim.getLogDir()
+
+    def addSimExtraCfg(self, option, value):
+        self.sim.addExtraCfg(option, value)
+
+
+tdDnodes = TDDnodes()
diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py
index 150b8f0c81..485c1b4b3f 100644
--- a/tests/pytest/util/dnodes-no-random-fail.py
+++ b/tests/pytest/util/dnodes-no-random-fail.py
@@ -349,7 +349,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -358,7 +358,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -465,7 +465,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -474,7 +474,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py
index 36c0f9d796..7c4edde47f 100644
--- a/tests/pytest/util/dnodes-random-fail.py
+++ b/tests/pytest/util/dnodes-random-fail.py
@@ -349,7 +349,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -358,7 +358,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -465,7 +465,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
@@ -474,7 +474,7 @@ class TDDnodes:
         psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
         processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
         while(processID):
-            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(
diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim
index 46fc3a4ff7..12a49c59b5 100644
--- a/tests/script/general/db/alter_option.sim
+++ b/tests/script/general/db/alter_option.sim
@@ -24,16 +24,16 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 10 then 
+if $data06 != 10 then 
   return -1
 endi
-if $data06 != 20,20,20 then 
+if $data07 != 20,20,20 then 
   return -1
 endi
-if $data07 != 2 then 
+if $data08 != 2 then 
   return -1
 endi
-if $data08 != 4 then 
+if $data09 != 4 then 
   return -1
 endi
 
diff --git a/tests/script/general/db/basic.sim b/tests/script/general/db/basic.sim
index a7c4358aff..1c4939256b 100644
--- a/tests/script/general/db/basic.sim
+++ b/tests/script/general/db/basic.sim
@@ -32,10 +32,10 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 20 then 
+if $data06 != 20 then 
   return -1
 endi
-if $data07 != 16 then 
+if $data08 != 16 then 
   return -1
 endi
  
@@ -72,7 +72,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data05 != 15 then 
+if $data06 != 15 then 
   return -1
 endi
 
diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim
index 8545b8d2cd..cb493c1456 100644
--- a/tests/script/general/http/restful_full.sim
+++ b/tests/script/general/http/restful_full.sim
@@ -81,7 +81,7 @@ print ===============  step2 - no db
 #11
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:6020/rest/sql
 print 11-> $system_content
-if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
+if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
   return -1
 endi
 
diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim
index be8587f2ae..ed6e427fe7 100644
--- a/tests/script/general/parser/create_db.sim
+++ b/tests/script/general/parser/create_db.sim
@@ -122,18 +122,18 @@ endi
 if $data04 != $replica then 
  return -1
 endi
-if $data05 != $days then 
+if $data06 != $days then 
  return -1
 endi
-if $data06 != 365,365,365 then 
+if $data07 != 365,365,365 then 
  return -1
 endi
-print data07 = $data07
-if $data07 != $cache then
- print expect $cache, actual:$data07
+print data08 = $data07
+if $data08 != $cache then
+ print expect $cache, actual:$data08
  return -1
 endi
-if $data08 != 4 then
+if $data09 != 4 then
  return -1
 endi
 
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index c6981d2902..1e73893793 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -93,6 +93,8 @@ run general/parser/groupby.sim
 sleep 2000
 run general/parser/tags_filter.sim
 sleep 2000
+run general/parser/topbot.sim
+sleep 2000
 run general/parser/union.sim
 sleep 2000
 run general/parser/sliding.sim
diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim
new file mode 100644
index 0000000000..a0c46dbc65
--- /dev/null
+++ b/tests/script/general/parser/topbot.sim
@@ -0,0 +1,74 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+$dbPrefix = tb_db
+$tbPrefix = tb_tb
+$stbPrefix = tb_stb
+$tbNum = 10
+$rowNum = 1000
+$totalNum = $tbNum * $rowNum
+$loops = 200000
+$log = 10000
+$ts0 = 1537146000000
+$delta = 600000
+print ========== topbot.sim
+$i = 0
+$db = $dbPrefix . $i
+$stb = $stbPrefix . $i
+
+sql drop database $db -x step1
+step1:
+sql create database $db cache 16 maxtables 200
+print ====== create tables
+sql use $db
+sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int)
+
+$i = 0
+$ts = $ts0
+$halfNum = $tbNum / 2
+while $i < $halfNum
+  $tbId = $i + $halfNum
+  $tb = $tbPrefix . $i
+  $tb1 = $tbPrefix . $tbId
+  sql create table $tb using $stb tags( $i )
+  sql create table $tb1 using $stb tags( $tbId )
+
+  $x = 0
+  while $x < $rowNum
+    $xs = $x * $delta
+    $ts = $ts0 + $xs
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+    $binary = 'binary . $c
+    $binary = $binary . '
+    $nchar = 'nchar . $c
+    $nchar = $nchar . '
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
+    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+endw
+print ====== tables created
+
+sql use $db
+##### select from table
+print ====== select top/bot from table and check num of rows returned
+sql select top(c1, 100) from tb_stb0
+if $row != 100 then
+  return -1
+endi
+
+sql select last(c2) from tb_tb9
+if $row != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/sh/exec-default.sh b/tests/script/sh/exec-default.sh
new file mode 100755
index 0000000000..1c84a6b10e
--- /dev/null
+++ b/tests/script/sh/exec-default.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# if [ $# != 4 || $# != 5 ]; then 
+  # echo "argument list need input : "
+  # echo "  -n nodeName"
+  # echo "  -s start/stop"
+  # echo "  -c clear"
+  # exit 1
+# fi
+
+NODE_NAME=
+EXEC_OPTON=
+CLEAR_OPTION="false"
+while getopts "n:s:u:x:ct" arg 
+do
+  case $arg in
+    n)
+      NODE_NAME=$OPTARG
+      ;;
+    s)
+      EXEC_OPTON=$OPTARG
+      ;;
+    c)
+      CLEAR_OPTION="clear"
+      ;;
+    t)
+      SHELL_OPTION="true"
+      ;;
+    u)
+      USERS=$OPTARG
+      ;;
+    x)
+      SIGNAL=$OPTARG
+      ;;
+    ?)
+      echo "unkown argument"
+      ;;
+  esac
+done
+
+SCRIPT_DIR=`dirname $0`
+cd $SCRIPT_DIR/../
+SCRIPT_DIR=`pwd`
+
+IN_TDINTERNAL="community"
+if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../../..
+else
+  cd ../../
+fi
+
+TAOS_DIR=`pwd`
+TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
+
+if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+else
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+fi
+
+BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
+
+SIM_DIR=$TAOS_DIR/sim
+NODE_DIR=$SIM_DIR/$NODE_NAME
+EXE_DIR=$BUILD_DIR/bin
+CFG_DIR=$NODE_DIR/cfg
+LOG_DIR=$NODE_DIR/log
+DATA_DIR=$NODE_DIR/data
+MGMT_DIR=$NODE_DIR/data/mgmt
+TSDB_DIR=$NODE_DIR/data/tsdb
+
+TAOS_CFG=$NODE_DIR/cfg/taos.cfg
+
+echo ------------ $EXEC_OPTON $NODE_NAME
+
+TAOS_FLAG=$SIM_DIR/tsim/flag
+if [ -f "$TAOS_FLAG" ]; then 
+  EXE_DIR=/usr/local/bin/taos
+fi
+
+if [ "$CLEAR_OPTION" = "clear" ]; then 
+  echo rm -rf $MGMT_DIR $TSDB_DIR  
+  rm -rf $TSDB_DIR
+  rm -rf $MGMT_DIR
+fi
+
+if [ "$EXEC_OPTON" = "start" ]; then 
+  echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR
+  
+  if [ "$SHELL_OPTION" = "true" ]; then 
+    TT=`date +%s`
+    mkdir ${LOG_DIR}/${TT}
+    nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes   $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &   
+  else
+    nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & 
+  fi
+  
+else
+  #relative path
+  RCFG_DIR=sim/$NODE_NAME/cfg
+  PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
+  while [ -n "$PID" ]
+  do
+    if [ "$SIGNAL" = "SIGKILL" ]; then
+      echo try to kill by signal SIGKILL
+      kill -9 $PID
+    else
+      echo try to kill by signal SIGINT
+      kill -SIGINT $PID
+    fi
+    sleep 1
+    PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
+  done 
+fi
+
diff --git a/tests/script/unique/clusterSimCase/client-01.sim b/tests/script/unique/clusterSimCase/client-01.sim
new file mode 100644
index 0000000000..b35790d69e
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-01.sim
@@ -0,0 +1,69 @@
+# create database db1
+# create table tb ()
+# loop insert seq data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-01-
+###############################################################
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-01 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-01 insert data into  $tb  error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-02.sim b/tests/script/unique/clusterSimCase/client-02.sim
new file mode 100644
index 0000000000..30a2ceb089
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-02.sim
@@ -0,0 +1,70 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-02-
+###############################################################
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-02 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-02 insert data into  $tb  error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-03.sim b/tests/script/unique/clusterSimCase/client-03.sim
new file mode 100644
index 0000000000..78a25e26a5
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-03.sim
@@ -0,0 +1,83 @@
+# create database db1
+# create table stb ()
+# loop dynamic create sub table and insert disorder data
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-03-
+###############################################################
+
+sql connect
+
+print ================ create db and super-table
+$db  = db
+$stb = stb
+sql create database if not exists $db replica 2 
+sql use $db
+sql create table if not exists $stb ( ts timestamp , c1 int , c2 binary(16) ) tags ( t1 int , t2 binary(16) )
+
+#print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+#$i = $tblStart
+#while $i < $tblEnd
+#  $tb = tb . $i
+##  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+#  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+#  $i = $i + 1
+#endw 
+
+$totalRows   = 0
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loop_cnt = 0
+loop_run:
+print ================ client-03 start loop insert data ( loop_cnt: $loop_cnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+
+  $t2Binary = $preBinary . $i
+  $t2Binary = $t2Binary . ' 
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb using $stb tags ( $i ,  $t2Binary ) values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb using $stb tags ( $i ,  $t2Binary ) values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ****** client-3 insert data into $tb error ***
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+sql select count(*) from $stb
+print data00 $data00 totalRows $totalRows
+if $data00 != $totalRows then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ***** client-3 insert data into $stbl error ********
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-04.sim b/tests/script/unique/clusterSimCase/client-04.sim
new file mode 100644
index 0000000000..5248e9d7bb
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-04.sim
@@ -0,0 +1,81 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop database, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-04-
+###############################################################
+
+$totalRows   = 0
+
+sql connect
+
+init_lable:
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:    
+print ================ client-04 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-04 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 100 then
+  sql drop database if exists $db
+  sleep 20000
+  $totalRows   = 0
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-05.sim b/tests/script/unique/clusterSimCase/client-05.sim
new file mode 100644
index 0000000000..a2deaba9b2
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-05.sim
@@ -0,0 +1,86 @@
+# create database db1
+# loop create table ,  insert disorder data, drop table, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-05-
+###############################################################
+
+$totalRows   = 0
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table $tb (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-05 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-05 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 100 then
+
+  $i = $tblStart
+  while $i < $tblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 20000
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-06.sim b/tests/script/unique/clusterSimCase/client-06.sim
new file mode 100644
index 0000000000..209b615e64
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-06.sim
@@ -0,0 +1,95 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop some tables and create new tables, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-06-
+$numPerUpdateTbl = 2000
+###############################################################
+$totalRows    = 0  
+$dropTblStart = $tblStart
+$dropTblEnd   = $dropTblStart
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table [ from $tblStart to $tblEnd ] (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$dropTblStart = $dropTblEnd
+$dropTblEnd   = $dropTblStart + $numPerUpdateTbl
+
+$tblStart = $tblEnd
+$tblEnd   = $tblEnd + $numPerUpdateTbl
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 160
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-06 start loop insert data from $dropTblStart to $tblStart ( loopCnt: $loopCnt )
+$i = $dropTblStart
+while $i < $tblStart
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $dropTblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-06 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $loopCnt > 2 then
+  print ================ client-06 drop table from $dropTblStart to $dropTblEnd
+  $i = $dropTblStart
+  while $i < $dropTblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 10000
+  goto init_lable 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-07.sim b/tests/script/unique/clusterSimCase/client-07.sim
new file mode 100644
index 0000000000..3bf3585205
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-07.sim
@@ -0,0 +1,101 @@
+# create database db1
+# loop create table ,  insert disorder data, drop table, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-07-
+###############################################################
+$alterCnt = 0
+
+$columnA = c . $alterCnt
+$columnB = d . $alterCnt
+
+$totalRows   = 0
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+
+print ================ create table $tb (ts timestamp, $columnA int, $columnB binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , $columnA int , $columnB binary(16) )
+  $i = $i + 1
+endw 
+
+$rowsPerLoop = 160
+
+$loopCnt = 0
+$alterStep = 0
+ 
+loop_run:
+print ================ client-07 start loop insert data ( loopCnt: $loopCnt )
+$i  = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $tblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-07 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+if $alterStep > 2 then
+  $alterStep = 0
+  
+  $alterCnt = $alterCnt + 1
+  $i = $tblStart
+  while $i < $tblEnd
+    $tb = tb . $i
+    $newColumnA = c . $alterCnt
+    $newcolumnB = d . $alterCnt
+
+    sql alter table $tb add column $newColumnA double
+    sql alter table $tb add column $newcolumnB binary( 16 )
+    sql alter table $tb drop column $columnA
+    sql alter table $tb drop column $columnB
+    $i = $i + 1
+  endw
+  $columnA = $newColumnA
+  $columnB = $newcolumnB
+    
+  sleep 10000
+  goto loop_run 
+endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/client-08.sim b/tests/script/unique/clusterSimCase/client-08.sim
new file mode 100644
index 0000000000..d9d64834cb
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/client-08.sim
@@ -0,0 +1,95 @@
+# create database db1
+# create table tb ()
+# loop insert disorder data, drop some tables and create new tables, sleep , 
+###############################################################
+################ set script parameters ################
+$tblStart = 0
+$tblEnd   = 10000
+$tsStart  = 1325347200000  # 2012-01-01 00:00:00.000
+$preBinary = ' . client-08-
+$numPerUpdateTbl = 2000
+###############################################################
+$totalRows    = 0  
+$dropTblStart = $tblStart
+$dropTblEnd   = $dropTblStart
+
+sql connect
+
+print ================ create db
+$db = db
+sql create database if not exists $db replica 2 
+sql use $db
+
+init_lable:
+print ================ create table [ from $tblStart to $tblEnd ] (ts timestamp, c1 int, c2 binary(16))
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) )
+  $i = $i + 1
+endw 
+
+$dropTblStart = $dropTblEnd
+$dropTblEnd   = $dropTblStart + $numPerUpdateTbl
+
+$tblStart = $tblEnd
+$tblEnd   = $tblEnd + $numPerUpdateTbl
+
+$rowsPerTbl  = 0
+$ts          = $tsStart
+$rowsPerLoop = 40
+
+$loopCnt = 0
+ 
+loop_run:
+print ================ client-08 start loop insert data from $dropTblStart to $tblStart ( loopCnt: $loopCnt )
+$i = $dropTblStart
+while $i < $tblStart
+  $tb = tb . $i
+  
+  $c2Binary = $preBinary . $i
+  $c2Binary = $c2Binary . '
+
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary )
+    sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary )
+    $x = $x + 40
+    $ts = $ts + 40a
+  endw 
+  if $i == $dropTblStart then
+    $rowsPerTbl = $rowsPerTbl + $x   
+  endi
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+endw 
+
+sql select count(*) from $tb
+print data00 $data00 rowsPerTbl $rowsPerTbl tb: $tb
+if $data00 != $rowsPerTbl then
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client-08 insert data into $tb error ***** *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  return -1
+endi
+
+$loopCnt = $loopCnt + 1
+
+#if $loopCnt > 3 then
+  print ================ client-08 drop table from $dropTblStart to $dropTblEnd
+  $i = $dropTblStart
+  while $i < $dropTblEnd
+    $tb = tb . $i
+    sql drop table if exists $tb
+    $i = $i + 1
+    $totalRows = $totalRows - $rowsPerTbl
+  endw  
+  sleep 10000
+  goto init_lable 
+#endi
+
+goto loop_run
diff --git a/tests/script/unique/clusterSimCase/cluster_main.sim b/tests/script/unique/clusterSimCase/cluster_main.sim
new file mode 100644
index 0000000000..39607c2915
--- /dev/null
+++ b/tests/script/unique/clusterSimCase/cluster_main.sim
@@ -0,0 +1,248 @@
+# scp -r root@114.116.252.220:/home/ubuntu/clusterSimCase unique/.
+
+#taos> show dnodes;
+#   id   |           end_point            | vnodes | cores  |     status     |   role   |       create_time       |
+#==================================================================================================================
+#      1 | ubuntu-OptiPlex-7060:7100      |      0 |     12 | ready          | mnode    | 2020-07-22 06:25:31.677 |
+#      2 | ubuntu-OptiPlex-7060:7200      |      4 |     12 | ready          | vnode    | 2020-07-22 06:25:34.795 |
+#      3 | ubuntu-OptiPlex-7060:7300      |      4 |     12 | ready          | vnode    | 2020-07-22 06:25:34.803 |
+#Query OK, 3 row(s) in set (0.002448s)
+#taos> show vgroups;
+#    vgId     |   tables    |   status   | onlineVnodes | dnode  |   vstatus   | dnode  |   vstatus   |
+#======================================================================================================
+#           2 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           3 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           4 |        1024 | ready      |            2 |      3 | master      |      2 | slave       |
+#           5 |         718 | ready      |            2 |      3 | master      |      2 | slave       |
+#Query OK, 4 row(s) in set (0.002749s)
+#
+#taos> show mnodes
+#   -> ;
+#   id   |           end_point            |      role      |       create_time       |
+#=====================================================================================
+#      1 | ubuntu-OptiPlex-7060:7100      | master         | 2020-07-22 06:25:31.677 |
+#Query OK, 1 row(s) in set (0.002126s)
+
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256
+
+#system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1
+#system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1
+
+#system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 256  
+#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 256
+
+system sh/cfg.sh -n dnode1 -c role -v 1
+system sh/cfg.sh -n dnode2 -c role -v 2
+system sh/cfg.sh -n dnode3 -c role -v 2
+system sh/cfg.sh -n dnode4 -c role -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator and dnode1
+system sh/exec_tarbitrator.sh -s start
+system sh/exec.sh -n dnode1 -s start
+
+print ============== step2: start dnode2/dnode3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep 1000
+sql connect
+
+print ============== step1: add dnode2/dnode3 into cluster
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+sleep 3000
+print ============== step3: start back client-01.sim
+#run_back unique/clusterSimCase/client-01.sim
+#run_back unique/clusterSimCase/client-02.sim
+#run_back unique/clusterSimCase/client-03.sim
+#run_back unique/clusterSimCase/client-04.sim
+#run_back unique/clusterSimCase/client-05.sim
+#run_back unique/clusterSimCase/client-06.sim
+#run_back unique/clusterSimCase/client-07.sim
+run_back unique/clusterSimCase/client-08.sim
+#run_back unique/clusterSimCase/client-01.sim
+#run_back unique/clusterSimCase/client-01.sim
+sleep 20000
+
+$db = db
+sql use $db
+
+$loop_cnt = 0
+loop_cluster_do:
+print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
+
+print ============== step4: stop dnode3
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+
+$checkCnt = 0
+wait_vgroup_chang_0:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_0
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != offline then
+  sleep 2000
+  goto wait_vgroup_chang_0
+endi
+if $vg2Dnode2Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_0
+endi
+
+
+sleep 3000
+print ============== step3: restart dnode3
+system sh/exec.sh -n dnode3 -s start
+
+$checkCnt = 0
+wait_vgroup_chang_1:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_1
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != slave then
+  sleep 2000
+  goto wait_vgroup_chang_1
+endi
+if $vg2Dnode2Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_1
+endi
+
+sleep 3000
+print ============== step4: stop dnode2
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+
+$checkCnt = 0
+wait_vgroup_chang_2:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_2
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode3Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_2
+endi
+if $vg2Dnode2Status != offline then
+  sleep 2000
+  goto wait_vgroup_chang_2
+endi
+
+
+sleep 3000
+print ============== step5: restart dnode2
+system sh/exec.sh -n dnode2 -s start
+
+$checkCnt = 0
+wait_vgroup_chang_3:
+$checkCnt = $checkCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+print show vgroups
+sql show vgroups
+#if $rows != 4 then
+#  print rows: $rows
+#  sleep 2000
+#  goto wait_vgroup_chang_3
+#endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$vg2Dnode3Status = $data5_2
+$vg2Dnode2Status = $data7_2
+
+if $vg2Dnode2Status != slave then
+  sleep 2000
+  goto wait_vgroup_chang_3
+endi
+if $vg2Dnode3Status != master then
+  sleep 2000
+  goto wait_vgroup_chang_3
+endi
+
+sleep 3000
+print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** ****
+$loop_cnt = $loop_cnt + 1
+if $loop_cnt == 50 then
+  return 0
+endi
+goto loop_cluster_do
diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 8686a23f65..4736c1ebed 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   #add_executable(insertPerTable insertPerTable.c)
diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt
index 14ef791a4e..0a11a8f14e 100644
--- a/tests/tsim/CMakeLists.txt
+++ b/tests/tsim/CMakeLists.txt
@@ -1,11 +1,11 @@
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF (TD_WINDOWS_64)