Merge remote-tracking branch 'upstream/develop' into odbc

2020-10-17 06:25:25 +08:00 · 2020-10-17 06:25:25 +08:00 · 1df2739281
parent 2436485e9b b4472552e6
commit 1df2739281
64 changed files with 1995 additions and 375 deletions
--- a/50
+++ b/50
@ -5,30 +5,33 @@ pipeline {
      WKC= '/var/lib/jenkins/workspace/TDinternal/community'
  }
  stages {
-    stage('build TDengine') {
-      agent{label 'master'}
-      steps {
-        sh '''
-        cd ${WKC}
-        git checkout develop
-        git pull
-        git submodule update
-        cd ${WK}
-        git checkout develop
-        git pull
-        export TZ=Asia/Harbin
-        date
-        rm -rf ${WK}/debug
-        mkdir debug
-        cd debug
-        #cmake .. > /dev/null
-        #make > /dev/null
-        '''
-      }
-    }
-
-    stage('Parallel test stage') {
+      stage('Parallel test stage') {
      parallel {
+        stage('pytest') {
+          agent{label 'master'}
+          steps {
+            sh '''
+            date
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests
+            #./test-all.sh smoke
+            ./test-all.sh pytest
+            date'''
+          }
+        }
        stage('test_b1') {
          agent{label '184'}
          steps {
@ -118,4 +121,5 @@ pipeline {
    }

  }
+  
 }
--- a/cmake/env.inc
+++ b/cmake/env.inc
@ -45,7 +45,7 @@ IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
 ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
  MESSAGE(STATUS "Build Release Version")
 ELSE ()
-  IF (TD_WINDOWS_64)
+  IF (TD_WINDOWS)
    SET(CMAKE_BUILD_TYPE "Release")
    MESSAGE(STATUS "Build Release Version in Windows as default")
  ELSE ()
--- a/cmake/install.inc
+++ b/cmake/install.inc
@ -16,6 +16,7 @@ ELSEIF (TD_WINDOWS)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
  INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
+  INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include)
  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) 
--- a/deps/zlib-1.2.11/inc/zconf.h
+++ b/deps/zlib-1.2.11/inc/zconf.h
@ -472,7 +472,7 @@ typedef uint64_t  z_crc_t;
 #endif
 #ifndef Z_SOLO
 #if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else
--- a/deps/zlib-1.2.11/src/zconf.h
+++ b/deps/zlib-1.2.11/src/zconf.h
@ -472,7 +472,7 @@ typedef uLong FAR uLongf;
 #endif
 #ifndef Z_SOLO
 #  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else
--- a/documentation20/webdocs/markdowndocs/connector-java-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md
@ -72,38 +72,34 @@ maven 项目中使用如下 pom.xml 配置即可：

 ### 获取连接

-如下所示配置即可获取 TDengine Connection：
+#### 通过JdbcUrl获取连接
+
+通过指定的jdbcUrl获取连接，如下所示：
 ```java
 Class.forName("com.taosdata.jdbc.TSDBDriver");
-String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata";
+String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
 Connection conn = DriverManager.getConnection(jdbcUrl);
 ```
-> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。
+以上示例，建立了到hostname为taosdemo.com，端口为6030（TDengine的默认端口），数据库名为test的连接。这个url中指定用户名（user）为root，密码（password）为taosdata。

 TDengine 的 JDBC URL 规范格式为：
-`jdbc:TAOS://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
-
-其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下：
-
+`jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
+url中的配置参数如下：
 * user：登录 TDengine 用户名，默认值 root。
 * password：用户登录密码，默认值 taosdata。
-* charset：客户端使用的字符集，默认值为系统字符集。
 * cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* charset：客户端使用的字符集，默认值为系统字符集。
 * locale：客户端语言环境，默认值系统当前 locale。
 * timezone：客户端使用的时区，默认值为系统当前时区。

-以上参数可以在 3 处配置，`优先级由高到低`分别如下：
-1. JDBC URL 参数
-    如上所述，可以在 JDBC URL 的参数中指定。
-2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
+#### 使用JdbcUrl和Properties获取连接
+
+除了通过指定的jdbcUrl获取连接，还可以使用Properties指定建立连接时的参数，如下所示：
 ```java
 public Connection getConn() throws Exception{
  Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata";
+  String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
  Properties connProps = new Properties();
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
@ -111,16 +107,39 @@ public Connection getConn() throws Exception{
  return conn;
 }
 ```
+以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库名为test的连接。这个连接在url中指定了用户名(user)为root，密码（password）为taosdata，并在connProps中指定了使用的字符集、语言环境、时区等信息。

-3. 客户端配置文件 taos.cfg
+properties中的配置参数如下：
+* TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。
+* TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。
+* TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
+* TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。
+* TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。

-    linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
-```properties
-# client default username
-# defaultUser           root
+#### 使用客户端配置文件建立连接
+当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、secondEp参数。
+如下所示：
+1. 在java中不指定hostname和port
+```java
+public Connection getConn() throws Exception{
+  Class.forName("com.taosdata.jdbc.TSDBDriver");
+  String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
+  Properties connProps = new Properties();
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+  return conn;
+}
+```
+2. 在配置文件中指定firstEp和secondEp
+```
+# first fully qualified domain name (FQDN) for TDengine system
+firstEp               cluster_node1:6030

-# client default password
-# defaultPass           taosdata
+# second fully qualified domain name (FQDN) for TDengine system, for cluster only
+secondEp              cluster_node2:6030

 # default system charset
 # charset               UTF-8
@ -128,6 +147,19 @@ public Connection getConn() throws Exception{
 # system locale
 # locale                en_US.UTF-8
 ```
+
+以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030，数据库名为test的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。
+TDengine中，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。
+
+> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
+
+#### 配置参数的优先级
+通过以上3种方式获取连接，如果配置参数在url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下：
+1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。
+2. Properties connProps
+3. 客户端配置文件 taos.cfg
+例如：在url中指定了password为taosdata，在Properties中指定了password为taosdemo，那么，JDBC会使用url中的password建立连接。
+
 > 更多详细配置请参考[客户端配置][13]

 ### 创建数据库和表
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@ -1,4 +1,4 @@
-FROM centos:7
+FROM ubuntu:16

 WORKDIR /root

--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: 'RELEASE_VERSION'
+version: '2.0.5.1'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@ -72,7 +72,7 @@ parts:
      - usr/bin/taosd
      - usr/bin/taos
      - usr/bin/taosdemo
-      - usr/lib/libtaos.so.RELEASE_VERSION
+      - usr/lib/libtaos.so.2.0.5.1
      - usr/lib/libtaos.so.1
      - usr/lib/libtaos.so

--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@ -126,6 +126,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {

  balanceAccquireDnodeList();

+  mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
+         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
@ -135,17 +137,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
        pVnodeGid->pDnode = pDnode;
        dnode++;
        vnodes++;
+        mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i);
        break;
+      } else {
+        mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
+               mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
+               pDnode->alternativeRole);
      }
    }
  }

  if (vnodes != pVgroup->numOfVnodes) {
-    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
-           pVgroup->numOfVnodes, vnodes);
    balanceReleaseDnodeList();
    balanceUnLock();

+    mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
+
+    void *     pIter = NULL;
+    SDnodeObj *pDnode = NULL;
+    while (1) {
+      pIter = mnodeGetNextDnode(pIter, &pDnode);
+      if (pDnode == NULL) break;
+      mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
+             pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
+      mnodeDecDnodeRef(pDnode);
+    }
+    sdbFreeIter(pIter);
+
    if (mnodeGetOnlineDnodesNum() == 0) {
      return TSDB_CODE_MND_NOT_READY;
    } else {
@ -553,7 +571,8 @@ static void balanceCheckDnodeAccess() {
      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
        pDnode->status = TAOS_DN_STATUS_OFFLINE;
        pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
-        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
+              pDnode->lastAccess);
        balanceSetVgroupOffline(pDnode);
      }
    }
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@ -80,6 +80,8 @@ enum {
  DATA_FROM_DATA_FILE = 2,
 };

+typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
+
 typedef struct STableComInfo {
  uint8_t numOfTags;
  uint8_t precision;
@ -226,7 +228,7 @@ typedef struct STableDataBlocks {
 typedef struct SQueryInfo {
  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
  uint32_t         type;          // query/insert type
-  // TODO refactor
+
  STimeWindow      window;        // query time window
  SInterval        interval;

@ -440,19 +442,20 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql);
 * @param pObj
 */
 void tscFreeSqlObj(SSqlObj *pSql);
-
-void tscFreeSqlObjInCache(void *pSql);
+void tscFreeRegisteredSqlObj(void *pSql);

 void tscCloseTscObj(STscObj *pObj);

+// todo move to taos? or create a new file: taos_internal.h
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                     void *param, void **taos);
-void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res);

-void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
+void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
+
+void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);

 void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
-void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
 bool tscIsUpdateQuery(SSqlObj* pSql);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
@ -517,8 +520,6 @@ extern SRpcCorEpSet tscMgmtEpSet;

 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

-typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
-
 int32_t tscCompareTidTags(const void* p1, const void* p2);
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);

--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
 static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);

-void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
+void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) {
  SSqlCmd* pCmd = &pSql->cmd;

  pSql->signature = pSql;
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@ -2445,8 +2445,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
  // in the first round, get the min-max value of all involved data
  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
  SPercentileInfo *pInfo = pResInfo->interResultBuf;
-  pInfo->minval = DBL_MAX;
-  pInfo->maxval = -DBL_MAX;
+  SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
+  SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
  pInfo->numOfElems = 0;

  return true;
@ -2461,12 +2461,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
  // the first stage, only acquire the min/max value
  if (pInfo->stage == 0) {
    if (pCtx->preAggVals.isSet) {
-      if (pInfo->minval > pCtx->preAggVals.statis.min) {
-        pInfo->minval = (double)pCtx->preAggVals.statis.min;
+      if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
+        SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
      }

-      if (pInfo->maxval < pCtx->preAggVals.statis.max) {
-        pInfo->maxval = (double)pCtx->preAggVals.statis.max;
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
+        SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
      }

      pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
@ -2500,12 +2500,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
            break;
        }

-        if (v < pInfo->minval) {
-          pInfo->minval = v;
+        if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+          SET_DOUBLE_VAL(&pInfo->minval, v);
        }

-        if (v > pInfo->maxval) {
-          pInfo->maxval = v;
+        if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+          SET_DOUBLE_VAL(&pInfo->maxval, v);
        }

        pInfo->numOfElems += 1;
@ -2564,12 +2564,12 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
        break;
    }

-    if (v < pInfo->minval) {
-      pInfo->minval = v;
+    if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+      SET_DOUBLE_VAL(&pInfo->minval, v);
    }

-    if (v > pInfo->maxval) {
-      pInfo->maxval = v;
+    if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+      SET_DOUBLE_VAL(&pInfo->maxval, v);
    }

    pInfo->numOfElems += 1;
@ -2609,7 +2609,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
    }

    pInfo->stage += 1;
-    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
  } else {
    pResInfo->complete = true;
  }
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@ -742,6 +742,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
                               int32_t numOfVnodes) {
  destroyColumnModel(pFinalModel);
  tOrderDescDestroy(pDesc);
+
  for (int32_t i = 0; i < numOfVnodes; ++i) {
    pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
  }
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@ -151,10 +151,12 @@ void tscKillQuery(STscObj *pObj, uint32_t killId) {

  pthread_mutex_unlock(&pObj->mutex);

-  if (pSql == NULL) return;
-
-  tscDebug("%p query is killed, queryId:%d", pSql, killId);
-  taos_stop_query(pSql);
+  if (pSql == NULL) {
+    tscError("failed to kill query, id:%d, it may have completed/terminated", killId);
+  } else {
+    tscDebug("%p query is killed, queryId:%d", pSql, killId);
+    taos_stop_query(pSql);
+  }
 }

 void tscAddIntoStreamList(SSqlStream *pStream) {
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@ -467,44 +467,6 @@ int tscProcessSql(SSqlObj *pSql) {
  return doProcessSql(pSql);
 }

-void tscKillSTableQuery(SSqlObj *pSql) {
-  SSqlCmd* pCmd = &pSql->cmd;
-  
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    return;
-  }
-
-  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-
-  for (int i = 0; i < pSql->subState.numOfSub; ++i) {
-    // NOTE: pSub may have been released already here
-    SSqlObj *pSub = pSql->pSubs[i];
-    if (pSub == NULL) {
-      continue;
-    }
-
-    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
-    if (p == NULL) {
-      continue;
-    }
-
-    SSqlObj* pSubObj = (SSqlObj*) (*p);
-    assert(pSubObj->self == (SSqlObj**) p);
-
-    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    if (pSubObj->pRpcCtx != NULL) {
-      rpcCancelRequest(pSubObj->pRpcCtx);
-      pSubObj->pRpcCtx = NULL;
-    }
-
-    tscQueueAsyncRes(pSubObj); // async res? not other functions?
-    taosCacheRelease(tscObjCache, (void**) &p, false);
-  }
-
-  tscDebug("%p super table query cancelled", pSql);
-}
-
 int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
  pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@ -28,6 +28,7 @@
 #include "tutil.h"
 #include "ttimer.h"
 #include "tscProfile.h"
+#include "ttimer.h"

 static bool validImpl(const char* str, size_t maxsize) {
  if (str == NULL) {
@ -257,10 +258,21 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port,
 void taos_close(TAOS *taos) {
  STscObj *pObj = (STscObj *)taos;

-  if (pObj == NULL || pObj->signature != pObj)  {
+  if (pObj == NULL) {
+    tscDebug("(null) try to free tscObj and close dnodeConn");
    return;
  }

+  tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
+  if (pObj->signature != pObj) {
+    tscDebug("%p already closed or invalid tscObj", pObj);
+    return;
+  }
+
+  // make sure that the close connection can only be executed once.
+  pObj->signature = NULL;
+  taosTmrStopA(&(pObj->pTimer));
+
  SSqlObj* pHb = pObj->pHb;
  if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
    if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode
@ -296,7 +308,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
  tsem_post(&pSql->rspSem);
 }

-TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
+TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) {
  STscObj *pObj = (STscObj *)taos;
  if (pObj == NULL || pObj->signature != pObj) {
    terrno = TSDB_CODE_TSC_DISCONNECTED;
@ -321,12 +333,20 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
  tsem_init(&pSql->rspSem, 0, 0);
  doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);

+  if (res != NULL) {
+    *res = pSql;
+  }
+
  tsem_wait(&pSql->rspSem);
  return pSql; 
 }

 TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
-  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr));
+  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL);
+}
+
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) {
+  return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res);
 }

 int taos_result_precision(TAOS_RES *res) {
@ -678,6 +698,45 @@ int* taos_fetch_lengths(TAOS_RES *res) {

 char *taos_get_client_info() { return version; }

+static void tscKillSTableQuery(SSqlObj *pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    return;
+  }
+
+  // set the master sqlObj flag to cancel query
+  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
+  for (int i = 0; i < pSql->subState.numOfSub; ++i) {
+    // NOTE: pSub may have been released already here
+    SSqlObj *pSub = pSql->pSubs[i];
+    if (pSub == NULL) {
+      continue;
+    }
+
+    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
+    if (p == NULL) {
+      continue;
+    }
+
+    SSqlObj* pSubObj = (SSqlObj*) (*p);
+    assert(pSubObj->self == (SSqlObj**) p);
+
+    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+    if (pSubObj->pRpcCtx != NULL) {
+      rpcCancelRequest(pSubObj->pRpcCtx);
+      pSubObj->pRpcCtx = NULL;
+    }
+
+    tscQueueAsyncRes(pSubObj);
+    taosCacheRelease(tscObjCache, (void**) &p, false);
+  }
+
+  tscDebug("%p super table query cancelled", pSql);
+}
+
 void taos_stop_query(TAOS_RES *res) {
  SSqlObj *pSql = (SSqlObj *)res;
  if (pSql == NULL || pSql->signature != pSql) {
@ -687,23 +746,26 @@ void taos_stop_query(TAOS_RES *res) {
  tscDebug("%p start to cancel query", res);
  SSqlCmd *pCmd = &pSql->cmd;

-  // TODO there are multi-thread problem.
-  // It may have been released by the other thread already.
-  // The ref count may fix this problem.
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-
  // set the error code for master pSqlObj firstly
  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;

+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
    assert(pSql->pRpcCtx == NULL);
    tscKillSTableQuery(pSql);
  } else {
    if (pSql->cmd.command < TSDB_SQL_LOCAL) {
+      /*
+       * There is multi-thread problem here, since pSql->pRpcCtx may have been
+       * reset and freed in the processMsgFromServer function, and causes the invalid
+       * write problem for rpcCancelRequest.
+       */
      if (pSql->pRpcCtx != NULL) {
        rpcCancelRequest(pSql->pRpcCtx);
        pSql->pRpcCtx = NULL;
      }
+
+      tscQueueAsyncRes(pSql);
    }
  }

@ -832,6 +894,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
  pSql->fp = asyncCallback;
  pSql->fetchFp = asyncCallback;
  pSql->param = pSql;
+
+  registerSqlObj(pSql);
  int code = tsParseSql(pSql, true);
  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
    tsem_wait(&pSql->rspSem);
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@ -258,11 +258,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
  assert(numOfSub > 0);
  
  // scan all subquery, if one sub query has only ts, ignore it
-  tscDebug("%p start to launch secondary subqueries, total:%d, only:%d needs to query", pSql, pSql->subState.numOfSub, numOfSub);
+  tscDebug("%p start to launch secondary subqueries, %d out of %d needs to query", pSql, numOfSub, pSql->subState.numOfSub);

  //the subqueries that do not actually launch the secondary query to virtual node is set as completed.
  SSubqueryState* pState = &pSql->subState;
-  pState->numOfRemain = pState->numOfSub;
+  pState->numOfRemain = numOfSub;

  bool success = true;
  
@ -1491,9 +1491,16 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
  return TSDB_CODE_SUCCESS;
 }

-static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
-  tscDebug("%p start to free subquery obj", pSql);
+static void tscFreeRetrieveSup(SSqlObj *pSql) {
+  SRetrieveSupport *trsupport = pSql->param;

+  void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0);
+  if (p == NULL) {
+    tscDebug("%p retrieve supp already released", pSql);
+    return;
+  }
+
+  tscDebug("%p start to free subquery supp obj:%p", pSql, trsupport);
 //  int32_t  index = trsupport->subqueryIndex;
 //  SSqlObj *pParentSql = trsupport->pParentSql;

@ -1556,17 +1563,18 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 }

 void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
+  // it has been freed already
+  if (pSql->param != trsupport || pSql->param == NULL) {
+    return;
+  }
+
  SSqlObj *pParentSql = trsupport->pParentSql;
  int32_t  subqueryIndex = trsupport->subqueryIndex;
  
  assert(pSql != NULL);
-  SSubqueryState* pState = &pParentSql->subState;
-  int32_t remain = pState->numOfRemain;
-  int32_t sub = pState->numOfSub;
-  UNUSED(remain);
-  UNUSED(sub);

-  assert(pParentSql->subState.numOfRemain <= pState->numOfSub && pParentSql->subState.numOfRemain >= 0);
+  SSubqueryState* pState = &pParentSql->subState;
+  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);

  // retrieved in subquery failed. OR query cancelled in retrieve phase.
  if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) {
@ -1597,12 +1605,12 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
    }
  }

-  remain = -1;
-  if ((remain = atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1)) > 0) {
+  int32_t remain = -1;
+  if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
    tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
        pState->numOfSub - remain);

-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
    return;
  }
  
@ -1614,7 +1622,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
  tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel,
                            pState->numOfSub);
  
-  tscFreeSubSqlObj(trsupport, pSql);
+  tscFreeRetrieveSup(pSql);

  // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes
  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
@ -1674,7 +1682,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
    tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
        pState->numOfSub - remain);

-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
    return;
  }
  
@ -1694,7 +1702,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
  pParentSql->res.numOfRows = 0;
  pParentSql->res.row = 0;
  
-  tscFreeSubSqlObj(trsupport, pSql);
+  tscFreeRetrieveSup(pSql);

  // set the command flag must be after the semaphore been correctly set.
  pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE;
@ -1706,14 +1714,21 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
 }

 static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
+  SSqlObj *pSql = (SSqlObj *)tres;
+  assert(pSql != NULL);
+
+  // this query has been freed already
  SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
+  if (pSql->param == NULL || param == NULL) {
+    tscDebug("%p already freed in dnodecallback", pSql);
+    assert(pSql->res.code == TSDB_CODE_TSC_QUERY_CANCELLED);
+    return;
+  }
+
  tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
  int32_t           idx   = trsupport->subqueryIndex;
  SSqlObj *         pParentSql = trsupport->pParentSql;

-  assert(tres != NULL);
-  SSqlObj *pSql = (SSqlObj *)tres;
-
  SSubqueryState* pState = &pParentSql->subState;
  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
  
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@ -141,7 +141,7 @@ void taos_init_imp(void) {
  int64_t refreshTime = 10; // 10 seconds by default
  if (tscMetaCache == NULL) {
    tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeSqlObjInCache, "sqlObj");
+    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
  }

  tscDebug("client is initialized successfully");
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@ -389,7 +389,7 @@ static void tscFreeSubobj(SSqlObj* pSql) {
 *
 * @param pSql
 */
-void tscFreeSqlObjInCache(void *pSql) {
+void tscFreeRegisteredSqlObj(void *pSql) {
  assert(pSql != NULL);

  SSqlObj** p = (SSqlObj**)pSql;
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@ -210,12 +210,12 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {

 static void *dnodeProcessWriteQueue(void *param) {
  SWriteWorker *pWorker = (SWriteWorker *)param;
-  SWriteMsg    *pWrite;
-  SWalHead     *pHead;
+  SWriteMsg *   pWrite;
+  SWalHead *    pHead;
  int32_t       numOfMsgs;
  int           type;
-  void         *pVnode, *item;
-  SRspRet      *pRspRet;
+  void *        pVnode, *item;
+  SRspRet *     pRspRet;

  dDebug("write worker:%d is running", pWorker->workerId);

@ -237,13 +237,18 @@ static void *dnodeProcessWriteQueue(void *param) {
        pHead->msgType = pWrite->rpcMsg.msgType;
        pHead->version = 0;
        pHead->len = pWrite->contLen;
-        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]);
+        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
+               taosMsg[pWrite->rpcMsg.msgType]);
      } else {
        pHead = (SWalHead *)item;
-        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version);
+        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
+               pHead->version);
      }

      int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
+      dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
+             pHead->version, tstrerror(code));
+
      if (pWrite) {
        pWrite->rpcMsg.code = code;
        if (code <= 0) pWrite->processedCount = 1;
@ -279,13 +284,13 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
  int32_t num = taosGetQueueNumber(pWorker->qset);

  if (num > 0) {
-     usleep(30000);
-     sched_yield();
+    usleep(30000);
+    sched_yield();
  } else {
-     taosFreeQall(pWorker->qall);
-     taosCloseQset(pWorker->qset);
-     pWorker->qset = NULL;
-     dDebug("write worker:%d is released", pWorker->workerId);
-     pthread_exit(NULL);
+    taosFreeQall(pWorker->qall);
+    taosCloseQset(pWorker->qset);
+    pWorker->qset = NULL;
+    dDebug("write worker:%d is released", pWorker->workerId);
+    pthread_exit(NULL);
  }
 }
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@ -194,9 +194,9 @@ static FORCE_INLINE bool isNull(const char *val, int32_t type) {
    case TSDB_DATA_TYPE_DOUBLE:
      return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
    case TSDB_DATA_TYPE_NCHAR:
-      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
+      return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
    case TSDB_DATA_TYPE_BINARY:
-      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
+      return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
    default:
      return false;
  };
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@ -193,6 +193,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "No write p
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "Missing data file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "Out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "Unexpected generic error in vnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE,     0, 0x050A, "Invalid version file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")

@ -247,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores
 // sync
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sync version")

 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
--- a/src/inc/twal.h
+++ b/src/inc/twal.h
@ -51,6 +51,7 @@ int     walWrite(twalh, SWalHead *);
 void    walFsync(twalh);
 int     walRestore(twalh, void *pVnode, FWalWrite writeFp);
 int     walGetWalFile(twalh, char *name, uint32_t *index);
+int64_t walGetVersion(twalh);

 extern int wDebugFlag;

--- a/src/kit/shell/inc/shell.h
+++ b/src/kit/shell/inc/shell.h
@ -20,6 +20,7 @@
 #include "taos.h"
 #include "taosdef.h"
 #include "stdbool.h"
+#include "tsclient.h"

 #define MAX_USERNAME_SIZE      64
 #define MAX_DBNAME_SIZE        64
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@ -294,9 +294,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {

  st = taosGetTimestampUs();

-  TAOS_RES* pSql = taos_query(con, command);
-  atomic_store_ptr(&result, pSql);  // set the global TAOS_RES pointer
-
+  TAOS_RES* pSql = taos_query_h(con, command, &result);
  if (taos_errno(pSql)) {
    taos_error(pSql);
    return;
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@ -648,8 +648,12 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void

  while (numOfRows < rows) {
    pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb);
+
    if (pDb == NULL) break;
-    if (pDb->pAcct != pUser->pAcct) continue;
+    if (pDb->pAcct != pUser->pAcct) {
+      mnodeDecDbRef(pDb);
+      continue;
+    }

    cols = 0;

--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@ -471,7 +471,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
             mnodeGetClusterId());
      return TSDB_CODE_MND_INVALID_CLUSTER_ID;
    } else {
-      mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
+      mTrace("dnode:%d, status received, access times %d openVnodes:%d:%d", pDnode->dnodeId, pDnode->lastAccess,
+             htons(pStatus->openVnodes), pDnode->openVnodes);
    }
  }

--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@ -594,7 +594,7 @@ static int sdbWrite(void *param, void *data, int type) {
      pthread_mutex_unlock(&tsSdbObj.mutex);
      sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
               pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
-      return TSDB_CODE_MND_APP_ERROR;
+      return TSDB_CODE_SYN_INVALID_VERSION;
    } else {
      tsSdbObj.version = pHead->version;
    }
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@ -131,6 +131,8 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
      mnodeAddTableIntoStable(pTable->superTable, pTable);
      grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
      if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
+    } else {
+      mError("table:%s:%p, correspond stable not found suid:%" PRIu64, pTable->info.tableId, pTable, pTable->suid);
    }
  } else {
    grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
@ -839,10 +841,11 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
    return TSDB_CODE_MND_OUT_OF_MEMORY;
  }

+  int64_t us = taosGetTimestampUs();
  pStable->info.tableId = strdup(pCreate->tableId);
  pStable->info.type    = TSDB_SUPER_TABLE;
  pStable->createdTime  = taosGetTimestampMs();
-  pStable->uid          = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
+  pStable->uid          = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
  pStable->sversion     = 0;
  pStable->tversion     = 0;
  pStable->numOfColumns = htons(pCreate->numOfColumns);
@ -1496,7 +1499,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
      continue;
    }
    if (pTable->vgHash == NULL) {
-      mError("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
+      mDebug("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
             stableName);
      mnodeDecTableRef(pTable);

@ -1707,7 +1710,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
  pTable->vgId         = pVgroup->vgId;

  if (pTable->info.type == TSDB_CHILD_TABLE) {
-    STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
+    STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
    if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
    if (pMsg->pSTable == NULL) {
      mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
@ -2252,7 +2255,8 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
  int32_t numOfTables = 0;
  SChildTableObj *pTable = NULL;

-  mInfo("stable:%s, all child tables:%d will dropped from sdb", pStable->info.tableId, pStable->numOfTables);
+  mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid,
+        pStable->numOfTables);

  while (1) {
    pIter = mnodeGetNextChildTable(pIter, &pTable);
@ -2405,14 +2409,16 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
    }
  } else {
    if (mnodeMsg->retry++ < 10) {
-      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
-             mnodeMsg->rpcMsg.handle);
+      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
+             " result:%s thandle:%p",
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->sid,
+             pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);

      dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
    } else {
-      mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-             pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid,
+             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);

      SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
      sdbDeleteRow(&oper);
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@ -270,31 +270,34 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
  Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode 
 */
 void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
-  SVnodeLoad *pNextV = NULL;
-
  void *pIter = NULL;
  while (1) {
    SVgObj *pVgroup;
    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
    if (pVgroup == NULL) break;

-    pNextV = pVloads;
-    int32_t i;
-    for (i = 0; i < openVnodes; ++i) {
-      if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
-        break;
-      }
-      pNextV++;
-    }
+    for (int v = 0; v < pVgroup->numOfVnodes; ++v) {
+      if (pVgroup->vnodeGid[v].dnodeId == pDnode->dnodeId) {
+        // vgroup should have a vnode on this dnode
+        bool have = false;
+        for (int32_t i = 0; i < openVnodes; ++i) {
+          SVnodeLoad *pVload = pVloads + i;
+          if (pVgroup->vgId == pVload->vgId) {
+            have = true;
+            break;
+          }
+        }

-    if (i == openVnodes) {
-      if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
-        mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
-               vgroupStatus[pVgroup->status]);
-      } else {
-        mDebug("vgId:%d, not exist in dnode:%d and status is %s, send create msg", pVgroup->vgId, pDnode->dnodeId,
-               vgroupStatus[pVgroup->status]);
-        mnodeSendCreateVgroupMsg(pVgroup, NULL);
+        if (have) continue;
+
+        if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
+          mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
+                 vgroupStatus[pVgroup->status]);
+        } else {
+          mDebug("vgId:%d, not exist in dnode:%d and status is %s, send create msg", pVgroup->vgId, pDnode->dnodeId,
+                 vgroupStatus[pVgroup->status]);
+          mnodeSendCreateVgroupMsg(pVgroup, NULL);
+        }
      }
    }

@ -302,7 +305,6 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
  }

  sdbFreeIter(pIter);
-  return;
 }

 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
@ -723,8 +725,16 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
  while (numOfRows < rows) {
    pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup);
    if (pVgroup == NULL) break;
-    if (pVgroup->pDb != pDb) continue;
-    if (!mnodeFilterVgroups(pVgroup, pTable)) continue;
+
+    if (pVgroup->pDb != pDb) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }
+
+    if (!mnodeFilterVgroups(pVgroup, pTable)) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }

    cols = 0;

--- a/src/os/src/linux/CMakeLists.txt
+++ b/src/os/src/linux/CMakeLists.txt
@ -4,4 +4,4 @@ PROJECT(TDengine)
 AUX_SOURCE_DIRECTORY(. SRC)

 ADD_LIBRARY(os ${SRC})
-TARGET_LINK_LIBRARIES(os m rt)
+TARGET_LINK_LIBRARIES(os m rt z)
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@ -1051,6 +1051,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
    if (code != 0) { // parsing error
      if (rpcIsReq(pHead->msgType)) {
        rpcSendErrorMsgToPeer(pRecv, code);
+        if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) {
+          rpcCloseConn(pConn);
+        }
        tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
      } 
    } else { // msg is passed to app only parsing is ok 
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@ -311,9 +311,20 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {

  if (pNode == NULL) return 0;

+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
+    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
+           pWalHead->version, nodeVersion);
+    for (int i = 0; i < pNode->replica; ++i) {
+      pPeer = pNode->peerInfo[i];
+      syncRestartConnection(pPeer);
+    }
+    return TSDB_CODE_SYN_INVALID_VERSION;
+  }
+
  // always update version
  nodeVersion = pWalHead->version;
-  sDebug("replica:%d nodeRole:%d qtype:%d", pNode->replica, nodeRole, qtype);
+  sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
+         qtype, pWalHead->version);

  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;

@ -883,7 +894,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
  SSyncNode *   pNode = pPeer->pSyncNode;
  SPeersStatus *pPeersStatus = (SPeersStatus *)cont;

-  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+  sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
         syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);

  pPeer->version = pPeersStatus->version;
@ -970,7 +981,8 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {

  int retLen = write(pPeer->peerFd, msg, statusMsgLen);
  if (retLen == statusMsgLen) {
-    sDebug("%s, status msg is sent", pPeer->id);
+    sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
+           pPeersStatus->version, pPeersStatus->ack);
  } else {
    sDebug("%s, failed to send status msg, restart", pPeer->id);
    syncRestartConnection(pPeer);
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@ -2284,7 +2284,7 @@ void filterPrepare(void* expr, void* param) {
  if (pInfo->optr == TSDB_RELATION_IN) {
    pInfo->q = (char*) pCond->arr;
  } else {
-    pInfo->q = calloc(1, pSchema->bytes);
+    pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE);   // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
    tVariantDump(pCond, pInfo->q, pSchema->type, true);
  }
 }
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@ -558,8 +558,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
  pCacheObj->numOfElemsInTrash++;
  __cache_unlock(pCacheObj);

-  uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data,
-      pCacheObj->numOfElemsInTrash);
+  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
+      pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }

 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@ -239,8 +239,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {

  code = vnodeReadVersion(pVnode);
  if (code != TSDB_CODE_SUCCESS) {
-    vnodeCleanUp(pVnode);
-    return code;
+    vError("vgId:%d, failed to read version, generate it from data file", pVnode->vgId);
+    // Allow vnode start even when read version fails, set version as walVersion or zero
+    // vnodeCleanUp(pVnode);
+    // return code;
  }

  pVnode->fversion = pVnode->version;
@ -292,6 +294,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
  }

  walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
+  if (pVnode->version == 0) {
+    pVnode->version = walGetVersion(pVnode->wal);
+  }

  SSyncInfo syncInfo;
  syncInfo.vgId = pVnode->vgId;
@ -947,6 +952,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
  len += snprintf(content + len, maxLen - len, "}\n");

  fwrite(content, 1, len, fp);
+  fflush(fp);
  fclose(fp);

  vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion);
@ -960,7 +966,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
  cJSON  *root = NULL;
  int     maxLen = 100;

-  terrno = TSDB_CODE_VND_APP_ERROR;
+  terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
  sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
  FILE *fp = fopen(versionFile, "r");
  if (!fp) {
@ -974,7 +980,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
  }

  content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
+  int len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
    goto PARSE_OVER;
@ -999,6 +1005,6 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
 PARSE_OVER:
  taosTFree(content);
  cJSON_Delete(root);
-  if(fp) fclose(fp);
+  if (fp) fclose(fp);
  return terrno;
 }
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@ -54,13 +54,13 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {

  // tsdb may be in reset state
  if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING)
-    return TSDB_CODE_APP_NOT_READY;
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;

  // TODO: Later, let slave to support query
  // if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
  if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role);
+    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[msgType],
+           pVnode->syncCfg.replica, syncRole[pVnode->role]);
    return TSDB_CODE_APP_NOT_READY;
  }

@ -80,7 +80,7 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) {
  taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }

-static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle, bool* freeHandle) {
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle) {
  bool continueExec = false;

  int32_t code = TSDB_CODE_SUCCESS;
@ -106,55 +106,56 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle,
  return code;
 }

-static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
  pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
  pRet->len = sizeof(SRetrieveTableRsp);

  memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  SRetrieveTableRsp* pRsp = pRet->rsp;
+  SRetrieveTableRsp *pRsp = pRet->rsp;

  pRsp->completed = true;
 }

 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
-  void    *pCont = pReadMsg->pCont;
+  void *   pCont = pReadMsg->pCont;
  int32_t  contLen = pReadMsg->contLen;
  SRspRet *pRet = &pReadMsg->rspRet;

-  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont;
+  SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
  memset(pRet, 0, sizeof(SRspRet));

  // qHandle needs to be freed correctly
  if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-    SRetrieveTableMsg* killQueryMsg = (SRetrieveTableMsg*) pReadMsg->pCont;
+    SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
    killQueryMsg->free = htons(killQueryMsg->free);
    killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);

-    vWarn("QInfo:%p connection %p broken, kill query", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
    assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);

-    void** qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) killQueryMsg->qhandle);
+    void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
    if (qhandle == NULL || *qhandle == NULL) {
-      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
+            pReadMsg->rpcMsg.handle);
    } else {
-      assert(*qhandle == (void*) killQueryMsg->qhandle);
+      assert(*qhandle == (void *)killQueryMsg->qhandle);

      qKillQuery(*qhandle);
-      qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true);
    }

    return TSDB_CODE_TSC_QUERY_CANCELLED;
  }

  int32_t code = TSDB_CODE_SUCCESS;
-  void**  handle = NULL;
+  void ** handle = NULL;

  if (contLen != 0) {
    qinfo_t pQInfo = NULL;
    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);

-    SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
-    pRsp->code    = code;
+    SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
+    pRsp->code = code;
    pRsp->qhandle = 0;

    pRet->len = sizeof(SQueryTableRsp);
@ -163,7 +164,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {

    // current connect is broken
    if (code == TSDB_CODE_SUCCESS) {
-      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo);
+      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t)pQInfo);
      if (handle == NULL) {  // failed to register qhandle, todo add error test case
        vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
               tstrerror(pRsp->code));
@ -171,13 +172,15 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
        qDestroyQueryInfo(pQInfo);  // destroy it directly
      } else {
        assert(*handle == pQInfo);
-        pRsp->qhandle = htobe64((uint64_t) pQInfo);
+        pRsp->qhandle = htobe64((uint64_t)pQInfo);
      }

-      if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+      if (handle != NULL &&
+          vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
+               pReadMsg->rpcMsg.handle);
        pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-        qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+        qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
        return pRsp->code;
      }
    } else {
@ -190,12 +193,12 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
    }
  } else {
    assert(pCont != NULL);
-    void** qhandle = (void**) pCont;
+    void **qhandle = (void **)pCont;

    vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);

    bool freehandle = false;
-    bool buildRes = qTableQuery(*qhandle); // do execute query
+    bool buildRes = qTableQuery(*qhandle);  // do execute query

    // build query rsp, the retrieve request has reached here already
    if (buildRes) {
@ -233,15 +236,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
  pRetrieve->free = htons(pRetrieve->free);
  pRetrieve->qhandle = htobe64(pRetrieve->qhandle);

-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
+         pRetrieve->free, pReadMsg->rpcMsg.handle);

  memset(pRet, 0, sizeof(SRspRet));

  int32_t code = TSDB_CODE_SUCCESS;
-  void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
-  if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
+  void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
+  if (handle == NULL || (*handle) != (void *)pRetrieve->qhandle) {
    code = TSDB_CODE_QRY_INVALID_QHANDLE;
-    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
+    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void *)pRetrieve->qhandle);

    vnodeBuildNoResultQueryRsp(pRet);
    return code;
@ -250,7 +254,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
  if (pRetrieve->free == 1) {
    vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
    qKillQuery(*handle);
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);

    vnodeBuildNoResultQueryRsp(pRet);
    code = TSDB_CODE_TSC_QUERY_CANCELLED;
@ -259,26 +263,27 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {

  // register the qhandle to connect to quit query immediate if connection is broken
  if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
+           pReadMsg->rpcMsg.handle);
    code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
    qKillQuery(*handle);
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
    return code;
  }

  bool freeHandle = true;
-  bool buildRes   = false;
+  bool buildRes = false;

  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
  if (code != TSDB_CODE_SUCCESS) {
-    //TODO handle malloc failure
+    // TODO handle malloc failure
    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
    pRet->len = sizeof(SRetrieveTableRsp);
    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
    freeHandle = true;
-  } else { // result is not ready, return immediately
+  } else {  // result is not ready, return immediately
    if (!buildRes) {
-      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
      return TSDB_CODE_QRY_NOT_READY;
    }

@ -288,7 +293,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
  // If qhandle is not added into vread queue, the query should be completed already or paused with error.
  // Here free qhandle immediately
  if (freeHandle) {
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
  }

  return code;
@ -296,13 +301,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {

 // notify connection(handle) that current qhandle is created, if current connection from
 // client is broken, the query needs to be killed immediately.
-int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
-  SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
-  killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
+int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) {
+  SRetrieveTableMsg *killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
+  killQueryMsg->qhandle = htobe64((uint64_t)qhandle);
  killQueryMsg->free = htons(1);
  killQueryMsg->header.vgId = htonl(vgId);
  killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));

  vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle);
-  return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
+  return rpcReportProgress(handle, (char *)killQueryMsg, sizeof(SRetrieveTableMsg));
 }
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@ -47,7 +47,7 @@ void vnodeInitWriteFp(void) {
 int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
  int32_t    code = 0;
  SVnodeObj *pVnode = (SVnodeObj *)param1;
-  SWalHead  *pHead = param2;
+  SWalHead * pHead = param2;

  if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
    vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
@ -61,41 +61,41 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {

  // tsdb may be in reset state
  if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING) 
-    return TSDB_CODE_APP_NOT_READY;
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;

-  if (pHead->version == 0) { // from client or CQ 
+  if (pHead->version == 0) {  // from client or CQ
    if (pVnode->status != TAOS_VN_STATUS_READY) {
-      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->status);
+      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->status);
      return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
    }

    if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->syncCfg.replica, pVnode->role);
+      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->syncCfg.replica, syncRole[pVnode->role]);
      return TSDB_CODE_APP_NOT_READY;
    }

    // assign version
-    pVnode->version++;
-    pHead->version = pVnode->version;
-    if (pVnode->delay) usleep(pVnode->delay*1000);
+    pHead->version = pVnode->version + 1;
+    if (pVnode->delay) usleep(pVnode->delay * 1000);

-  } else { // from wal or forward 
+  } else {  // from wal or forward
    // for data from WAL or forward, version may be smaller
    if (pHead->version <= pVnode->version) return 0;
  }

-  pVnode->version = pHead->version;
-
-  // write into WAL
-  code = walWrite(pVnode->wal, pHead);
-  if (code < 0) return code;
-
  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
  int32_t syncCode = 0;
  syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
  if (syncCode < 0) return syncCode;

+  // write into WAL
+  code = walWrite(pVnode->wal, pHead);
+  if (code < 0) return code;
+
+  pVnode->version = pHead->version;
+
  // write data locally
  code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
  if (code < 0) return code;
@ -191,7 +191,7 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR

 int vnodeWriteToQueue(void *param, void *data, int type) {
  SVnodeObj *pVnode = param;
-  SWalHead *pHead = data;
+  SWalHead * pHead = data;

  int size = sizeof(SWalHead) + pHead->len;
  SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
@ -204,4 +204,3 @@ int vnodeWriteToQueue(void *param, void *data, int type) {

  return 0;
 }
-
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@ -120,10 +120,9 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {

  if (pCfg->keep == 1) return pWal;

-  if (walHandleExistingFiles(path) == 0) 
-    walRenew(pWal);
+  if (walHandleExistingFiles(path) == 0) walRenew(pWal);

-  if (pWal && pWal->fd <0) {
+  if (pWal && pWal->fd < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    wError("wal:%s, failed to open(%s)", path, strerror(errno));
    walRelease(pWal);
@ -154,7 +153,7 @@ int walAlter(twalh wal, const SWalCfg *pCfg) {
  pWal->fsyncPeriod = pCfg->fsyncPeriod;
  if (walNeedFsyncTimer(pWal)) {
    wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
-    taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer,walTmrCtrl);
+    taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl);
  } else {
    wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
    taosTmrStop(pWal->timer);
@ -174,9 +173,9 @@ void walClose(void *handle) {

  if (pWal->keep == 0) {
    // remove all files in the directory
-    for (int i=0; i<pWal->num; ++i) {
-      snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
-      if (remove(pWal->name) <0) {
+    for (int i = 0; i < pWal->num; ++i) {
+      snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i);
+      if (remove(pWal->name) < 0) {
        wError("wal:%s, failed to remove", pWal->name);
      } else {
        wDebug("wal:%s, it is removed", pWal->name);
@ -197,7 +196,7 @@ int walRenew(void *handle) {

  pthread_mutex_lock(&pWal->mutex);

-  if (pWal->fd >=0) {
+  if (pWal->fd >= 0) {
    close(pWal->fd);
    pWal->id++;
    wDebug("wal:%s, it is closed", pWal->name);
@ -218,7 +217,7 @@ int walRenew(void *handle) {
      // remove the oldest wal file
      char name[TSDB_FILENAME_LEN * 3];
      snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
-      if (remove(name) <0) {
+      if (remove(name) < 0) {
        wError("wal:%s, failed to remove(%s)", name, strerror(errno));
      } else {
        wDebug("wal:%s, it is removed", name);
@ -247,7 +246,7 @@ int walWrite(void *handle, SWalHead *pHead) {
  taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
  int contLen = pHead->len + sizeof(SWalHead);

-  if(taosTWrite(pWal->fd, pHead, contLen) != contLen) {
+  if (taosTWrite(pWal->fd, pHead, contLen) != contLen) {
    wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
    terrno = TAOS_SYSTEM_ERROR(errno);
  } else {
@ -258,7 +257,6 @@ int walWrite(void *handle, SWalHead *pHead) {
 }

 void walFsync(void *handle) {
-
  SWal *pWal = handle;
  if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return;

@ -276,12 +274,11 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
  uint32_t maxId = 0, minId = -1, index =0;

  terrno = 0;
-  int   plen = strlen(walPrefix);
-  char  opath[TSDB_FILENAME_LEN+5];
+  int  plen = strlen(walPrefix);
+  char opath[TSDB_FILENAME_LEN + 5];

  int slen = snprintf(opath, sizeof(opath), "%s", pWal->path);
-  if ( pWal->keep == 0) 
-    strcpy(opath+slen, "/old");
+  if (pWal->keep == 0) strcpy(opath + slen, "/old");

  DIR *dir = opendir(opath);
  if (dir == NULL && errno == ENOENT) return 0;
@ -290,8 +287,8 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
    return terrno;
  }

-  while ((ent = readdir(dir))!= NULL) {
-    if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+  while ((ent = readdir(dir)) != NULL) {
+    if (strncmp(ent->d_name, walPrefix, plen) == 0) {
      index = atol(ent->d_name + plen);
      if (index > maxId) maxId = index;
      if (index < minId) minId = index;
@ -306,13 +303,13 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
    return terrno;
  }

-  if ( count != (maxId-minId+1) ) {
+  if (count != (maxId - minId + 1)) {
    wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
    terrno = TSDB_CODE_WAL_APP_ERROR;
  } else {
    wDebug("wal:%s, %d files will be restored", opath, count);

-    for (index = minId; index<=maxId; ++index) {
+    for (index = minId; index <= maxId; ++index) {
      snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
      terrno = walRestoreWalFile(pWal, pVnode, writeFp);
      if (terrno < 0) break;
@ -345,7 +342,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
 }

 int walGetWalFile(void *handle, char *name, uint32_t *index) {
-  SWal   *pWal = handle;
+  SWal *  pWal = handle;
  int     code = 1;
  int32_t first = 0;

@ -361,7 +358,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
    code = -1;  // index out of range
  } else {
    sprintf(name, "wal/%s%d", walPrefix, *index);
-    code = (*index == pWal->id) ? 0:1;
+    code = (*index == pWal->id) ? 0 : 1;
  }

  pthread_mutex_unlock(&(pWal->mutex));
@ -370,7 +367,6 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
 }

 static void walRelease(SWal *pWal) {
-
  pthread_mutex_destroy(&pWal->mutex);
  pWal->signature = NULL;
  free(pWal);
@ -385,7 +381,7 @@ static void walRelease(SWal *pWal) {

 static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
  char *name = pWal->name;
-  int size = 1024 * 1024; // default 1M buffer size
+  int   size = 1024 * 1024;  // default 1M buffer size

  terrno = 0;
  char *buffer = malloc(size);
@ -487,8 +483,8 @@ int walHandleExistingFiles(const char *path) {
  } else {
    // move all files to old directory
    int count = 0;
-    while ((ent = readdir(dir))!= NULL) {  
-      if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+    while ((ent = readdir(dir)) != NULL) {
+      if (strncmp(ent->d_name, walPrefix, plen) == 0) {
        snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
        snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
        if (taosMkDir(opath, 0755) != 0) {
@ -515,23 +511,23 @@ int walHandleExistingFiles(const char *path) {
 }

 static int walRemoveWalFiles(const char *path) {
-  int    plen = strlen(walPrefix);
-  char   name[TSDB_FILENAME_LEN * 3];
+  int  plen = strlen(walPrefix);
+  char name[TSDB_FILENAME_LEN * 3];

  terrno = 0;

  struct dirent *ent;
-  DIR   *dir = opendir(path);
+  DIR *dir = opendir(path);
  if (dir == NULL && errno == ENOENT) return 0;
  if (dir == NULL) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return terrno;
  }

-  while ((ent = readdir(dir))!= NULL) {
-    if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+  while ((ent = readdir(dir)) != NULL) {
+    if (strncmp(ent->d_name, walPrefix, plen) == 0) {
      snprintf(name, sizeof(name), "%s/%s", path, ent->d_name);
-      if (remove(name) <0) {
+      if (remove(name) < 0) {
        wError("wal:%s, failed to remove(%s)", name, strerror(errno));
        terrno = TAOS_SYSTEM_ERROR(errno);
      }
@ -561,3 +557,10 @@ static void walProcessFsyncTimer(void *param, void *tmrId) {
    pWal->timer = NULL;
  }
 }
+
+int64_t walGetVersion(twalh param) {
+  SWal *pWal = param;
+  if (pWal == 0) return 0;
+
+  return pWal->version;
+}
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@ -172,23 +172,23 @@ python3 testNoCompress.py
 python3 testMinTablesPerVnode.py

 # functions
-python3 ./test.py -f functions/function_avg.py
-python3 ./test.py -f functions/function_bottom.py
-python3 ./test.py -f functions/function_count.py
-python3 ./test.py -f functions/function_diff.py
-python3 ./test.py -f functions/function_first.py
-python3 ./test.py -f functions/function_last.py
-python3 ./test.py -f functions/function_last_row.py
-python3 ./test.py -f functions/function_leastsquares.py
-python3 ./test.py -f functions/function_max.py
-python3 ./test.py -f functions/function_min.py
-python3 ./test.py -f functions/function_operations.py
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
 python3 ./test.py -f functions/function_percentile.py
-python3 ./test.py -f functions/function_spread.py
-python3 ./test.py -f functions/function_stddev.py
-python3 ./test.py -f functions/function_sum.py
-python3 ./test.py -f functions/function_top.py
-#python3 ./test.py -f functions/function_twa.py
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+#python3 ./test.py -f functions/function_twa.py -r 1
 python3 queryCount.py
 python3 ./test.py -f query/queryGroupbyWithInterval.py
 python3 client/twoClients.py
--- a/tests/pytest/functions/function_avg_restart.py
+++ b/tests/pytest/functions/function_avg_restart.py
@ -0,0 +1,73 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        for i in range(self.rowNum):
+            #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+            #            % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # average verifacation 
+        tdSql.error("select avg(ts) from test")
+        tdSql.error("select avg(ts) from test1")
+        tdSql.error("select avg(col7) from test")
+        tdSql.error("select avg(col7) from test1")
+        tdSql.error("select avg(col8) from test")
+        tdSql.error("select avg(col8) from test1")
+        tdSql.error("select avg(col9) from test")
+        tdSql.error("select avg(col9) from test1")
+
+        tdSql.query("select avg(col1) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col2) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col3) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col4) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col5) from test")
+        tdSql.checkData(0, 0, np.average(floatData))              
+        tdSql.query("select avg(col6) from test")
+        tdSql.checkData(0, 0, np.average(floatData))              
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_bottom_restart.py
+++ b/tests/pytest/functions/function_bottom_restart.py
@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")    
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                            
+
+        # bottom verifacation 
+        tdSql.error("select bottom(ts, 10) from test")
+        tdSql.error("select bottom(col1, 0) from test")
+        tdSql.error("select bottom(col1, 101) from test")
+        tdSql.error("select bottom(col2, 0) from test")
+        tdSql.error("select bottom(col2, 101) from test")
+        tdSql.error("select bottom(col3, 0) from test")
+        tdSql.error("select bottom(col3, 101) from test")
+        tdSql.error("select bottom(col4, 0) from test")
+        tdSql.error("select bottom(col4, 101) from test")
+        tdSql.error("select bottom(col5, 0) from test")
+        tdSql.error("select bottom(col5, 101) from test")
+        tdSql.error("select bottom(col6, 0) from test")
+        tdSql.error("select bottom(col6, 101) from test")        
+        tdSql.error("select bottom(col7, 10) from test")        
+        tdSql.error("select bottom(col8, 10) from test")
+        tdSql.error("select bottom(col9, 10) from test")
+
+        tdSql.query("select bottom(col1, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col2, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col3, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col4, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col5, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 0.1)
+        tdSql.checkData(1, 1, 1.1)
+
+        tdSql.query("select bottom(col6, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 0.1)
+        tdSql.checkData(1, 1, 1.1)
+                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_count_restart.py
+++ b/tests/pytest/functions/function_count_restart.py
@ -0,0 +1,79 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+        
+        # Count verifacation
+        tdSql.query("select count(*) from test")        
+        tdSql.checkData(0, 0, 11)
+
+        tdSql.query("select count(ts) from test")        
+        tdSql.checkData(0, 0, 11)                        
+        tdSql.query("select count(col1) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col2) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col3) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col4) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col5) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col6) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col7) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col8) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col9) from test")        
+        tdSql.checkData(0, 0, 11)
+
+        #tdSql.execute("alter table test add column col10 int")
+        #tdSql.query("select count(col10) from test")        
+        #tdSql.checkRows(0)
+
+        ##tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)")
+        tdSql.query("select count(col10) from test")        
+        tdSql.checkData(0, 0, 1)        
+              
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_diff_restart.py
+++ b/tests/pytest/functions/function_diff_restart.py
@ -0,0 +1,99 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1))
+        
+        # diff verifacation 
+        #tdSql.query("select diff(col1) from test1")
+        #tdSql.checkRows(0)
+        #
+        #tdSql.query("select diff(col2) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col3) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col4) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col5) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col6) from test1")
+        #tdSql.checkRows(0)
+
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                                
+        
+        tdSql.error("select diff(ts) from test")
+        tdSql.error("select diff(ts) from test1")
+        tdSql.error("select diff(col1) from test")
+        tdSql.error("select diff(col2) from test")
+        tdSql.error("select diff(col3) from test")
+        tdSql.error("select diff(col4) from test")
+        tdSql.error("select diff(col5) from test")
+        tdSql.error("select diff(col6) from test")
+        tdSql.error("select diff(col7) from test")     
+        tdSql.error("select diff(col7) from test1")               
+        tdSql.error("select diff(col8) from test") 
+        tdSql.error("select diff(col8) from test1")
+        tdSql.error("select diff(col9) from test")        
+        tdSql.error("select diff(col9) from test1")
+        
+        tdSql.query("select diff(col1) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col2) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col3) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col4) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col5) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col6) from test1")
+        tdSql.checkRows(10)
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_first_restart.py
+++ b/tests/pytest/functions/function_first_restart.py
@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select first(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+
+        tdSql.query("select first(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 0.1)
+
+        tdSql.query("select first(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 0.1)
+
+        tdSql.query("select first(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)
+
+        tdSql.query("select first(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata1')
+
+        tdSql.query("select first(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据1')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_last_restart.py
+++ b/tests/pytest/functions/function_last_restart.py
@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select last(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 10)
+
+        tdSql.query("select last(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+
+        tdSql.query("select last(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata10')
+
+        tdSql.query("select last(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据10')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_last_row_restart.py
+++ b/tests/pytest/functions/function_last_row_restart.py
@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select last_row(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 10)
+
+        tdSql.query("select last_row(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last_row(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last_row(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+
+        tdSql.query("select last_row(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata10')
+
+        tdSql.query("select last_row(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据10')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_leastsquares_restart.py
+++ b/tests/pytest/functions/function_leastsquares_restart.py
@ -0,0 +1,68 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        # leastsquares verifacation 
+        tdSql.error("select leastsquares(ts, 1, 1) from test1")
+        tdSql.error("select leastsquares(col1, 1, 1) from test")
+        tdSql.error("select leastsquares(col2, 1, 1) from test")
+        tdSql.error("select leastsquares(col3, 1, 1) from test")
+        tdSql.error("select leastsquares(col4, 1, 1) from test")
+        tdSql.error("select leastsquares(col5, 1, 1) from test")
+        tdSql.error("select leastsquares(col6, 1, 1) from test")
+        tdSql.error("select leastsquares(col7, 1, 1) from test1")
+        tdSql.error("select leastsquares(col8, 1, 1) from test1")
+        tdSql.error("select leastsquares(col9, 1, 1) from test1")
+                
+        tdSql.query("select leastsquares(col1, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col2, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col3, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col4, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col5, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}')
+
+        tdSql.query("select leastsquares(col6, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_max_restart.py
+++ b/tests/pytest/functions/function_max_restart.py
@ -0,0 +1,76 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db") 
+
+        intData = []        
+        floatData = []
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #             col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # max verifacation 
+        tdSql.error("select max(ts) from test")
+        tdSql.error("select max(ts) from test1")
+        tdSql.error("select max(col7) from test")
+        tdSql.error("select max(col7) from test1")
+        tdSql.error("select max(col8) from test")
+        tdSql.error("select max(col8) from test1")
+        tdSql.error("select max(col9) from test")
+        tdSql.error("select max(col9) from test1")
+        
+        tdSql.query("select max(col1) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col2) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col3) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col4) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+        
+        tdSql.query("select max(col5) from test1")
+        tdSql.checkData(0, 0, np.max(floatData))
+
+        tdSql.query("select max(col6) from test1")
+        tdSql.checkData(0, 0, np.max(floatData))
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_min_restart.py
+++ b/tests/pytest/functions/function_min_restart.py
@ -0,0 +1,73 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # min verifacation 
+        tdSql.error("select min(ts) from test")
+        tdSql.error("select min(ts) from test1")
+        tdSql.error("select min(col7) from test")
+        tdSql.error("select min(col7) from test1")
+        tdSql.error("select min(col8) from test")
+        tdSql.error("select min(col8) from test1")
+        tdSql.error("select min(col9) from test")
+        tdSql.error("select min(col9) from test1")
+        
+        tdSql.query("select min(col1) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col2) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col3) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col4) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+        
+        tdSql.query("select min(col5) from test1")
+        tdSql.checkData(0, 0, np.min(floatData))
+
+        tdSql.query("select min(col6) from test1")
+        tdSql.checkData(0, 0, np.min(floatData))
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_operations_restart.py
+++ b/tests/pytest/functions/function_operations_restart.py
@ -0,0 +1,70 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        # min verifacation 
+        tdSql.error("select ts + col1 from test")
+        tdSql.error("select ts + col1 from test1")
+        tdSql.error("select col1 + col7 from test")
+        tdSql.error("select col1 + col7 from test1")
+        tdSql.error("select col1 + col8 from test")
+        tdSql.error("select col1 + col8 from test1")
+        tdSql.error("select col1 + col9 from test")
+        tdSql.error("select col1 + col9 from test1")
+
+        tdSql.query("select col1 + col2 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0, 0, 2.0)        
+
+        tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0, 0, 3.2)
+
+        #tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11))
+        tdSql.query("select col1 + col2 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+        
+        tdSql.query("select col1 + col2 * col3 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+
+        tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_percentile_restart.py
+++ b/tests/pytest/functions/function_percentile_restart.py
@ -0,0 +1,136 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # percentile verifacation 
+        tdSql.error("select percentile(ts 20) from test")
+        tdSql.error("select apercentile(ts 20) from test")
+        tdSql.error("select percentile(col7 20) from test")
+        tdSql.error("select apercentile(col7 20) from test")
+        tdSql.error("select percentile(col8 20) from test")        
+        tdSql.error("select apercentile(col8 20) from test")
+        tdSql.error("select percentile(col9 20) from test")
+        tdSql.error("select apercentile(col9 20) from test")        
+
+        tdSql.query("select percentile(col1, 0) from test")        
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col1, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col1, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col1, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col1, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100))
+        tdSql.query("select apercentile(col1, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))  
+
+        tdSql.query("select percentile(col2, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col2, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col2, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col2, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col2, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100)) 
+        tdSql.query("select apercentile(col2, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col3, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col3, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col3, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col3, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col3, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100))
+        tdSql.query("select apercentile(col3, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col4, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col4, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col4, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col4, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col4, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100)) 
+        tdSql.query("select apercentile(col4, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col5, 0) from test")        
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 0))
+        tdSql.query("select apercentile(col5, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col5, 50) from test")
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 50))
+        tdSql.query("select apercentile(col5, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))        
+        tdSql.query("select percentile(col5, 100) from test")
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 100)) 
+        tdSql.query("select apercentile(col5, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))     
+
+        tdSql.query("select percentile(col6, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 0))
+        tdSql.query("select apercentile(col6, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col6, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 50))
+        tdSql.query("select apercentile(col6, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col6, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 100)) 
+        tdSql.query("select apercentile(col6, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))       
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_spread_restart.py
+++ b/tests/pytest/functions/function_spread_restart.py
@ -0,0 +1,69 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")        
+       
+        tdSql.error("select spread(col7) from test")     
+        tdSql.error("select spread(col7) from test1")               
+        tdSql.error("select spread(col8) from test") 
+        tdSql.error("select spread(col8) from test1")
+        tdSql.error("select spread(col9) from test")        
+        tdSql.error("select spread(col9) from test1")
+        
+        tdSql.query("select spread(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select spread(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_stddev_restart.py
+++ b/tests/pytest/functions/function_stddev_restart.py
@ -0,0 +1,75 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # stddev verifacation 
+        tdSql.error("select stddev(ts) from test1")
+        tdSql.error("select stddev(col1) from test")
+        tdSql.error("select stddev(col2) from test")
+        tdSql.error("select stddev(col3) from test")
+        tdSql.error("select stddev(col4) from test")
+        tdSql.error("select stddev(col5) from test")
+        tdSql.error("select stddev(col6) from test")
+        tdSql.error("select stddev(col7) from test1")
+        tdSql.error("select stddev(col8) from test1")
+        tdSql.error("select stddev(col9) from test1")
+
+        tdSql.query("select stddev(col1) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col2) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col3) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col4) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col5) from test1")
+        tdSql.checkData(0, 0, np.std(floatData))
+
+        tdSql.query("select stddev(col6) from test1")
+        tdSql.checkData(0, 0, np.std(floatData))
+            
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_sum_restart.py
+++ b/tests/pytest/functions/function_sum_restart.py
@ -0,0 +1,64 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # sum verifacation 
+        tdSql.error("select sum(ts) from test")
+        tdSql.error("select sum(col7) from test")
+        tdSql.error("select sum(col8) from test")
+        tdSql.error("select sum(col9) from test")
+
+        tdSql.query("select sum(col1) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col2) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col3) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col4) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col5) from test")
+        tdSql.checkData(0, 0, np.sum(floatData)) 
+        tdSql.query("select sum(col6) from test")
+        tdSql.checkData(0, 0, np.sum(floatData))              
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_top_restart.py
+++ b/tests/pytest/functions/function_top_restart.py
@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # top verifacation 
+        tdSql.error("select top(ts, 10) from test")
+        tdSql.error("select top(col1, 0) from test")
+        tdSql.error("select top(col1, 101) from test")
+        tdSql.error("select top(col2, 0) from test")
+        tdSql.error("select top(col2, 101) from test")
+        tdSql.error("select top(col3, 0) from test")
+        tdSql.error("select top(col3, 101) from test")
+        tdSql.error("select top(col4, 0) from test")
+        tdSql.error("select top(col4, 101) from test")
+        tdSql.error("select top(col5, 0) from test")
+        tdSql.error("select top(col5, 101) from test")
+        tdSql.error("select top(col6, 0) from test")
+        tdSql.error("select top(col6, 101) from test")        
+        tdSql.error("select top(col7, 10) from test")        
+        tdSql.error("select top(col8, 10) from test")
+        tdSql.error("select top(col9, 10) from test")
+
+        tdSql.query("select top(col1, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col2, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col3, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col4, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col5, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 8.1)
+        tdSql.checkData(1, 1, 9.1)
+
+        tdSql.query("select top(col6, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 8.1)
+        tdSql.checkData(1, 1, 9.1)
+                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/functions/function_twa_restart.py
+++ b/tests/pytest/functions/function_twa_restart.py
@ -0,0 +1,130 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # twa verifacation 
+        tdSql.error("select twa(ts) from test")
+        tdSql.error("select twa(ts) from test1")
+
+        tdSql.error("select twa(col1) from test")
+        tdSql.error("select twa(col1) from test1")
+
+        tdSql.error("select twa(col2) from test")
+        tdSql.error("select twa(col2) from test1")
+
+        tdSql.error("select twa(col3) from test")
+        tdSql.error("select twa(col3) from test1")
+
+        tdSql.error("select twa(col4) from test")
+        tdSql.error("select twa(col4) from test1")
+
+        tdSql.error("select twa(col5) from test")
+        tdSql.error("select twa(col5) from test1")
+
+        tdSql.error("select twa(col6) from test")
+        tdSql.error("select twa(col6) from test1")        
+
+        tdSql.error("select twa(col7) from test")
+        tdSql.error("select twa(col7) from test1")
+
+        tdSql.error("select twa(col8) from test")
+        tdSql.error("select twa(col8) from test1")
+
+        tdSql.error("select twa(col9) from test")
+        tdSql.error("select twa(col9) from test1")
+
+        tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
+            
+        tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
+        
+        tdSql.query("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col1) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col2) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col3) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.query("select twa(col4) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.query("select twa(col5) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
+        tdSql.query("select twa(col6) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/pytest/handle_val_log.sh
+++ b/tests/pytest/handle_val_log.sh
--- a/tests/pytest/test.py
+++ b/tests/pytest/test.py
@ -16,6 +16,7 @@
 import sys
 import getopt
 import subprocess
+import time
 from distutils.log import warn as printf

 from util.log import *
@ -33,7 +34,8 @@ if __name__ == "__main__":
    valgrind = 0
    logSql = True
    stop = 0
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scgh', [
+    restart = False
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [
        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help'])
    for key, value in opts:
        if key in ['-h', '--help']:
@ -46,8 +48,12 @@ if __name__ == "__main__":
            tdLog.printNoPrefix('-s stop All dnodes')
            tdLog.printNoPrefix('-c Test Cluster Flag')
            tdLog.printNoPrefix('-g valgrind Test Flag')
+            tdLog.printNoPrefix('-r taosd restart test')
            sys.exit(0)

+        if key in ['-r', '--restart']: 
+            restart = True
+
        if key in ['-f', '--file']:
            fileName = value

@ -138,5 +144,19 @@ if __name__ == "__main__":
            tdCases.runAllLinux(conn)
        else:
            tdCases.runOneLinux(conn, fileName)
-
+    if restart:
+        if fileName == "all":
+            tdLog.info("not need to query ")
+        else:    
+            sp = fileName.rsplit(".", 1)
+            if len(sp) == 2 and sp[1] == "py":
+                tdDnodes.stopAll()
+                tdDnodes.start(1)
+                time.sleep(1)            
+                conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                tdLog.info("query test after taosd restart")
+                tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+            else:
+                tdLog.info("not need to query")
    conn.close()
--- a/tests/script/general/insert/query_block1_memory.sim
+++ b/tests/script/general/insert/query_block1_memory.sim
@ -30,9 +30,11 @@ print =============== step 1
 $x = $N
 $y = $N / 2
 while $x > $y
-  $ms = $x . m 
+  $z = $x * 60000
+  $ms = 1601481600000 - $z
+
  $xt = - . $x 
-  sql insert into $tb values (now - $ms , -$x ) 
+  sql insert into $tb values ($ms , -$x ) 
  $x = $x - 1
 endw 

@ -45,8 +47,10 @@ endi
 $x = $N / 2
 $y = $N
 while $x < $y
-  $ms = $x . m 
-  sql insert into $tb values (now + $ms , $x ) 
+  $z = $x * 60000
+  $ms = 1601481600000 + $z
+
+  sql insert into $tb values ($ms , $x ) 
  $x = $x + 1
 endw 
 sql select * from $tb
@ -60,14 +64,14 @@ print =============== step 2
 $N1 = $N + 1
 $result1 = $N / 2
 $result2 = $N
-$step = $N1 . m
+$step = $N1 * 60000

-$start1 = now- . $step
-$start2 = now
-$start3 = now+ . $step
-$end1 = now- . $step
-$end2 = now
-$end3 = now+ . $step
+$start1 = 1601481600000 -  $step
+$start2 = 1601481600000
+$start3 = 1601481600000 + $step
+$end1 = 1601481600000 -  $step
+$end2 = 1601481600000
+$end3 = 1601481600000 +  $step

 sql select * from $tb where ts < $start1 and ts > $end1
 if $rows != 0 then 
--- a/tests/script/general/parser/where.sim
+++ b/tests/script/general/parser/where.sim
@ -33,7 +33,8 @@ while $i < $tbNum

  $x = 0
  while $x < $rowNum
-    $ms = $x . m
+    $y = $x * 60000
+    $ms = 1600099200000 + $y
    $c = $x / 100
    $c = $c * 100
    $c = $x - $c
@ -41,7 +42,7 @@ while $i < $tbNum
    $binary = $binary . '
    $nchar = 'nchar . $c
    $nchar = $nchar . '
-    sql insert into $tb values (now + $ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+    sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
    $x = $x + 1
  endw

@ -299,7 +300,8 @@ while $i < 1

  $x = 0
  while $x < 10000
-    $ms = $x . m
+    $y = $x * 60000
+    $ms = 1601481600000 + $y
    $c = $x / 100
    $c = $c * 100
    $c = $x - $c
@ -307,7 +309,7 @@ while $i < 1
    $binary = $binary . '
    $nchar = 'nchar . $c
    $nchar = $nchar . '
-    sql insert into $tb values (now + $ms , null , null , null , null , null , null , null , null , null )
+    sql insert into $tb values ($ms , null , null , null , null , null , null , null , null , null )
    $x = $x + 1
  endw

--- a/tests/script/general/stream/table_n.sim
+++ b/tests/script/general/stream/table_n.sim
@ -289,3 +289,5 @@ endi
 if $data09 != 20 then 
  return -1
 endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
--- a/tests/script/unique/cluster/vgroup100.sim
+++ b/tests/script/unique/cluster/vgroup100.sim
@ -29,6 +29,9 @@ system sh/exec.sh -n dnode3 -s start

 sleep 3000

+$maxNum = 102
+$maxNum = 12
+
 $x = 0
 show2: 
 	$x = $x + 1
@ -58,7 +61,7 @@ endi

 print ============================== step3
 $count = 2
-while $count < 102
+while $count < $maxNum
  $db = d . $count
  $tb = $db . .t
  $tb2 = $db . .t2
@ -73,7 +76,7 @@ endw
 print ============================== step4

 $count = 2
-while $count < 102
+while $count < $maxNum
  $db = d . $count
  $tb = $db . .t
  sql select * from $tb
@ -131,7 +134,7 @@ show8:
 	endi

 $count = 2
-while $count < 102
+while $count < $maxNum
  $db = d . $count
  $tb = $db . .t
  sql select * from $tb
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@ -121,7 +121,7 @@ if [ "$2" != "sim" ]; then
  elif [ "$1" == "full" ]; then
    echo "### run Python full test ###"
    runPyCaseOneByOne fulltest.sh
-  elif [ "$1" == "b1" ]; then
+  elif [ "$1" == "pytest" ]; then
    echo "### run Python full test ###"
    runPyCaseOneByOne fulltest.sh
  elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then