From 58dafe2de274d4f3855d66a3a3414df3f6785b18 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Tue, 13 Oct 2020 12:24:38 +0000
Subject: [PATCH 01/30] TD-1711

---
 src/client/src/tscSql.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index ef8625da06..484de20d16 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -832,6 +832,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
   pSql->fp = asyncCallback;
   pSql->fetchFp = asyncCallback;
   pSql->param = pSql;
+
+  registerSqlObj(pSql);
   int code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     tsem_wait(&pSql->rspSem);

From 90c2b1f527d0cedf821adcde5a00ec59ee0bb100 Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Wed, 14 Oct 2020 11:11:41 +0800
Subject: [PATCH 02/30] include file for static packaging

---
 packaging/docker/Dockerfile     | 2 +-
 snap/snapcraft.yaml             | 4 ++--
 src/os/src/linux/CMakeLists.txt | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index b4497e210f..0f138be4d4 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM centos:7
+FROM ubuntu:16
 
 WORKDIR /root
 
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index d55be5b8c0..b0d8d8983b 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: 'RELEASE_VERSION'
+version: '2.0.5.1'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.RELEASE_VERSION
+      - usr/lib/libtaos.so.2.0.5.1
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 
diff --git a/src/os/src/linux/CMakeLists.txt b/src/os/src/linux/CMakeLists.txt
index 752326cc1d..b1a7ebf54e 100644
--- a/src/os/src/linux/CMakeLists.txt
+++ b/src/os/src/linux/CMakeLists.txt
@@ -4,4 +4,4 @@ PROJECT(TDengine)
 AUX_SOURCE_DIRECTORY(. SRC)
 
 ADD_LIBRARY(os ${SRC})
-TARGET_LINK_LIBRARIES(os m rt)
+TARGET_LINK_LIBRARIES(os m rt z)

From 0066dc9ac08748cfd81c92ca7fe0c4089f4b015f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 12:43:45 +0800
Subject: [PATCH 03/30] [td-1716]

---
 src/client/src/tscSql.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index ef8625da06..f73b4abb15 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -257,10 +257,21 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port,
 void taos_close(TAOS *taos) {
   STscObj *pObj = (STscObj *)taos;
 
-  if (pObj == NULL || pObj->signature != pObj)  {
+  if (pObj == NULL) {
+    tscDebug("(null) try to free tscObj and close dnodeConn");
     return;
   }
 
+  tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
+  if (pObj->signature != pObj) {
+    tscDebug("%p already closed or invalid tscObj", pObj);
+    return;
+  }
+
+  // make sure that the close connection can only be executed once.
+  pObj->signature = NULL;
+  taosTmrStopA(&(pObj->pTimer));
+
   SSqlObj* pHb = pObj->pHb;
   if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
     if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode

From 2a9855bbaf52c513bcb4b028ae9b764cd2a70501 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 13:02:23 +0800
Subject: [PATCH 04/30] [td-225] fix bugs found by tsim

---
 src/client/src/tscSubquery.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index fbbd810a19..abfe62c72c 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -258,11 +258,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
   assert(numOfSub > 0);
   
   // scan all subquery, if one sub query has only ts, ignore it
-  tscDebug("%p start to launch secondary subqueries, total:%d, only:%d needs to query", pSql, pSql->subState.numOfSub, numOfSub);
+  tscDebug("%p start to launch secondary subqueries, %d out of %d needs to query", pSql, numOfSub, pSql->subState.numOfSub);
 
   //the subqueries that do not actually launch the secondary query to virtual node is set as completed.
   SSubqueryState* pState = &pSql->subState;
-  pState->numOfRemain = pState->numOfSub;
+  pState->numOfRemain = numOfSub;
 
   bool success = true;
   

From 7f70eff949ccafad2992f04af9364663cb8ae7cc Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 13:24:41 +0800
Subject: [PATCH 05/30] [td-1716] enable repeatably close connection to avoid
 close the connection twice in c# driver #3737

---
 src/client/src/tscSql.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index acc38ac8ed..c5a46e2185 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -27,6 +27,7 @@
 #include "ttokendef.h"
 #include "tutil.h"
 #include "tscProfile.h"
+#include "ttimer.h"
 
 static bool validImpl(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -256,10 +257,21 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port,
 void taos_close(TAOS *taos) {
   STscObj *pObj = (STscObj *)taos;
 
-  if (pObj == NULL || pObj->signature != pObj)  {
+  if (pObj == NULL) {
+    tscDebug("(null) try to free tscObj and close dnodeConn");
     return;
   }
 
+  tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
+  if (pObj->signature != pObj) {
+    tscDebug("%p already closed or invalid tscObj", pObj);
+    return;
+  }
+
+  // make sure that the close connection can only be executed once.
+  pObj->signature = NULL;
+  taosTmrStopA(&(pObj->pTimer));
+
   SSqlObj* pHb = pObj->pHb;
   if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
     if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode

From aed51263c8538b22613191b310567921784e3223 Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Wed, 14 Oct 2020 14:28:49 +0800
Subject: [PATCH 06/30] package taoserror.h into windows client

---
 cmake/install.inc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/install.inc b/cmake/install.inc
index f5e01e2f1d..c7fbd6df79 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -16,6 +16,7 @@ ELSEIF (TD_WINDOWS)
   INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
   INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
   INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
+  INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) 

From 4c22b72d0663b3aa9b85f2ac0efaceb31d64e00e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 14:31:46 +0800
Subject: [PATCH 07/30] [td-225]

---
 src/util/src/tcache.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 2896481574..6e20c1708d 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -558,8 +558,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   pCacheObj->numOfElemsInTrash++;
   __cache_unlock(pCacheObj);
 
-  uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data,
-      pCacheObj->numOfElemsInTrash);
+  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
+      pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }
 
 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {

From e9d283db163316150dcb2192c2233d835f010dce Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 14 Oct 2020 15:06:21 +0800
Subject: [PATCH 08/30] TD-1667

---
 src/mnode/src/mnodeDb.c     |  6 +++++-
 src/mnode/src/mnodeVgroup.c | 12 ++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 37a626c6cc..8d7c267ab7 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -648,8 +648,12 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
 
   while (numOfRows < rows) {
     pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb);
+
     if (pDb == NULL) break;
-    if (pDb->pAcct != pUser->pAcct) continue;
+    if (pDb->pAcct != pUser->pAcct) {
+      mnodeDecDbRef(pDb);
+      continue;
+    }
 
     cols = 0;
 
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 1f4132544b..69b9224d4f 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -723,8 +723,16 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
   while (numOfRows < rows) {
     pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup);
     if (pVgroup == NULL) break;
-    if (pVgroup->pDb != pDb) continue;
-    if (!mnodeFilterVgroups(pVgroup, pTable)) continue;
+
+    if (pVgroup->pDb != pDb) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }
+
+    if (!mnodeFilterVgroups(pVgroup, pTable)) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }
 
     cols = 0;
 

From db6236a66b848ffd01d44c1ce5e665102d585346 Mon Sep 17 00:00:00 2001
From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com>
Date: Wed, 14 Oct 2020 18:31:14 +0800
Subject: [PATCH 09/30] Update connector-java-ch.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

更新jdbc connector文档
---
 .../webdocs/markdowndocs/connector-java-ch.md | 79 +++++++++++++------
 1 file changed, 56 insertions(+), 23 deletions(-)

diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md
index f6da7ff403..8f508a21b0 100644
--- a/documentation20/webdocs/markdowndocs/connector-java-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md
@@ -72,38 +72,35 @@ maven 项目中使用如下 pom.xml 配置即可：
 
 ### 获取连接
 
-如下所示配置即可获取 TDengine Connection：
+#### 通过JdbcUrl获取连接
+
+通过指定的jdbcUrl获取连接，如下所示：
 ```java
 Class.forName("com.taosdata.jdbc.TSDBDriver");
-String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata";
+String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/log?user=root&password=taosdata";
 Connection conn = DriverManager.getConnection(jdbcUrl);
 ```
-> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。
+以上示例，建立了到hostname为taosdemo.com，端口为6030，database为log的连接。
+> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库，user和password参数指定了用户名和密码
 
 TDengine 的 JDBC URL 规范格式为：
-`jdbc:TAOS://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
-
-其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下：
-
+`jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
+url中的配置参数如下：
 * user：登录 TDengine 用户名，默认值 root。
 * password：用户登录密码，默认值 taosdata。
-* charset：客户端使用的字符集，默认值为系统字符集。
 * cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* charset：客户端使用的字符集，默认值为系统字符集。
 * locale：客户端语言环境，默认值系统当前 locale。
 * timezone：客户端使用的时区，默认值为系统当前时区。
 
-以上参数可以在 3 处配置，`优先级由高到低`分别如下：
-1. JDBC URL 参数
-    如上所述，可以在 JDBC URL 的参数中指定。
-2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
+#### 使用JdbcUrl和Properties获取连接
+
+除了通过指定的jdbcUrl获取连接，还可以使用Properties指定建立连接时的参数，如下所示：
 ```java
 public Connection getConn() throws Exception{
   Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata";
+  String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/log?user=root&password=taosdata";
   Properties connProps = new Properties();
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
@@ -111,16 +108,39 @@ public Connection getConn() throws Exception{
   return conn;
 }
 ```
+以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库为log的连接。这个连接在jdbcUrl中指定了用户名和密码，并在connProps中指定了使用的字符集、语言环境、时区等信息。
 
-3. 客户端配置文件 taos.cfg
+properties中的配置参数如下：
+* TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。
+* TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。
+* TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
+* TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。
+* TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。
 
-    linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
-```properties
-# client default username
-# defaultUser           root
+#### 使用客户端配置文件建立连接
+当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、SecondEp参数。
+如下所示：
+1. 在java中不指定hostname和port
+```java
+public Connection getConn() throws Exception{
+  Class.forName("com.taosdata.jdbc.TSDBDriver");
+  String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata";
+  Properties connProps = new Properties();
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+  return conn;
+}
+```
+2. 在配置文件中指定firstEp和secondEp
+```
+# first fully qualified domain name (FQDN) for TDengine system
+firstEp               cluster_node1:6030
 
-# client default password
-# defaultPass           taosdata
+# second fully qualified domain name (FQDN) for TDengine system, for cluster only
+secondEp              cluster_node2:6030
 
 # default system charset
 # charset               UTF-8
@@ -128,6 +148,19 @@ public Connection getConn() throws Exception{
 # system locale
 # locale                en_US.UTF-8
 ```
+
+以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。
+TDengine中，只要firstEp和secondEp中任何一个节点为有效连接，TDengine就可以通过该节点，找到集群的master。因此，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。
+
+> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在的机器上的配置文件，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+
+#### 配置参数的优先级
+通过以上3种方式获取连接，如果配置参数在url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下：
+1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。
+2. Properties connProps
+3. 客户端配置文件 taos.cfg
+例如：在url中指定了password为taosdata，在Properties中指定了password为taosdemo，那么，JDBC会使用url中的password建立连接。
+
 > 更多详细配置请参考[客户端配置][13]
 
 ### 创建数据库和表

From d838bacdbe270d9359cb72292f4466c014dea3d7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 18:48:24 +0800
Subject: [PATCH 10/30] [td-1637]

---
 src/client/src/tscLocalMerge.c |  1 +
 src/client/src/tscServer.c     |  3 ++-
 src/client/src/tscSubquery.c   | 33 ++++++++++++++++++---------------
 3 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 16f208da98..d2f74bdd59 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -742,6 +742,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
                                int32_t numOfVnodes) {
   destroyColumnModel(pFinalModel);
   tOrderDescDestroy(pDesc);
+
   for (int32_t i = 0; i < numOfVnodes; ++i) {
     pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
   }
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 5cddaa1c4d..1161bae424 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -475,6 +475,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
     return;
   }
 
+  // set the master sqlObj flag to cancel query
   pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
 
   for (int i = 0; i < pSql->subState.numOfSub; ++i) {
@@ -498,7 +499,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
       pSubObj->pRpcCtx = NULL;
     }
 
-    tscQueueAsyncRes(pSubObj); // async res? not other functions?
+//    tscQueueAsyncRes(pSubObj); // async res? not other functions?
     taosCacheRelease(tscObjCache, (void**) &p, false);
   }
 
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index abfe62c72c..185d6784d1 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1491,9 +1491,16 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
-  tscDebug("%p start to free subquery obj", pSql);
+static void tscFreeRetrieveSup(SSqlObj *pSql) {
+  SRetrieveSupport *trsupport = pSql->param;
 
+  void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0);
+  if (p == NULL) {
+    tscDebug("%p retrieve supp already released", pSql);
+    return;
+  }
+
+  tscDebug("%p start to free subquery supp obj:%p", pSql, trsupport);
 //  int32_t  index = trsupport->subqueryIndex;
 //  SSqlObj *pParentSql = trsupport->pParentSql;
 
@@ -1560,13 +1567,9 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   int32_t  subqueryIndex = trsupport->subqueryIndex;
   
   assert(pSql != NULL);
-  SSubqueryState* pState = &pParentSql->subState;
-  int32_t remain = pState->numOfRemain;
-  int32_t sub = pState->numOfSub;
-  UNUSED(remain);
-  UNUSED(sub);
 
-  assert(pParentSql->subState.numOfRemain <= pState->numOfSub && pParentSql->subState.numOfRemain >= 0);
+  SSubqueryState* pState = &pParentSql->subState;
+  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
 
   // retrieved in subquery failed. OR query cancelled in retrieve phase.
   if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) {
@@ -1597,12 +1600,12 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
     }
   }
 
-  remain = -1;
-  if ((remain = atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1)) > 0) {
+  int32_t remain = -1;
+  if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
     tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
         pState->numOfSub - remain);
 
-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
     return;
   }
   
@@ -1614,7 +1617,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel,
                             pState->numOfSub);
   
-  tscFreeSubSqlObj(trsupport, pSql);
+  tscFreeRetrieveSup(pSql);
 
   // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
@@ -1674,7 +1677,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
     tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
         pState->numOfSub - remain);
 
-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
     return;
   }
   
@@ -1694,7 +1697,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   pParentSql->res.numOfRows = 0;
   pParentSql->res.row = 0;
   
-  tscFreeSubSqlObj(trsupport, pSql);
+  tscFreeRetrieveSup(pSql);
 
   // set the command flag must be after the semaphore been correctly set.
   pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE;
@@ -1711,8 +1714,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
   int32_t           idx   = trsupport->subqueryIndex;
   SSqlObj *         pParentSql = trsupport->pParentSql;
 
-  assert(tres != NULL);
   SSqlObj *pSql = (SSqlObj *)tres;
+  assert(pSql != NULL && trsupport == pSql->param);
 
   SSubqueryState* pState = &pParentSql->subState;
   assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);

From f14734c2617595f59dbe60543d141958134e502f Mon Sep 17 00:00:00 2001
From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com>
Date: Wed, 14 Oct 2020 19:08:17 +0800
Subject: [PATCH 11/30] Update connector-java-ch.md

---
 .../webdocs/markdowndocs/connector-java-ch.md | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md
index 8f508a21b0..e3743f4f9d 100644
--- a/documentation20/webdocs/markdowndocs/connector-java-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md
@@ -77,11 +77,10 @@ maven 项目中使用如下 pom.xml 配置即可：
 通过指定的jdbcUrl获取连接，如下所示：
 ```java
 Class.forName("com.taosdata.jdbc.TSDBDriver");
-String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/log?user=root&password=taosdata";
+String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
 Connection conn = DriverManager.getConnection(jdbcUrl);
 ```
-以上示例，建立了到hostname为taosdemo.com，端口为6030，database为log的连接。
-> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库，user和password参数指定了用户名和密码
+以上示例，建立了到hostname为taosdemo.com，端口为6030（TDengine的默认端口），数据库名为test的连接。这个url中指定用户名（user）为root，密码（password）为taosdata。
 
 TDengine 的 JDBC URL 规范格式为：
 `jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
@@ -99,7 +98,7 @@ url中的配置参数如下：
 ```java
 public Connection getConn() throws Exception{
   Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/log?user=root&password=taosdata";
+  String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
   Properties connProps = new Properties();
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
@@ -108,7 +107,7 @@ public Connection getConn() throws Exception{
   return conn;
 }
 ```
-以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库为log的连接。这个连接在jdbcUrl中指定了用户名和密码，并在connProps中指定了使用的字符集、语言环境、时区等信息。
+以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库名为test的连接。这个连接在url中指定了用户名(user)为root，密码（password）为taosdata，并在connProps中指定了使用的字符集、语言环境、时区等信息。
 
 properties中的配置参数如下：
 * TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。
@@ -119,13 +118,13 @@ properties中的配置参数如下：
 * TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。
 
 #### 使用客户端配置文件建立连接
-当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、SecondEp参数。
+当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、secondEp参数。
 如下所示：
 1. 在java中不指定hostname和port
 ```java
 public Connection getConn() throws Exception{
   Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata";
+  String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
   Properties connProps = new Properties();
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
@@ -149,10 +148,10 @@ secondEp              cluster_node2:6030
 # locale                en_US.UTF-8
 ```
 
-以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。
-TDengine中，只要firstEp和secondEp中任何一个节点为有效连接，TDengine就可以通过该节点，找到集群的master。因此，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。
+以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030，数据库名为test的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。
+TDengine中，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。
 
-> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在的机器上的配置文件，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
 
 #### 配置参数的优先级
 通过以上3种方式获取连接，如果配置参数在url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下：

From 181fadd74d08176183721cbb723b3a0f41b9bc78 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Wed, 14 Oct 2020 19:30:06 +0800
Subject: [PATCH 12/30] [TD-1676] modify Jenkinsfile

---
 Jenkinsfile                    | 3 ++-
 tests/pytest/handle_val_log.sh | 0
 2 files changed, 2 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 tests/pytest/handle_val_log.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index 5a8f6a551e..55fa27cf28 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -118,4 +118,5 @@ pipeline {
     }
 
   }
-}
+  
+}
\ No newline at end of file
diff --git a/tests/pytest/handle_val_log.sh b/tests/pytest/handle_val_log.sh
old mode 100644
new mode 100755

From d0175e6ea086d6219989342994c38d385e5b2753 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 14 Oct 2020 19:38:15 +0800
Subject: [PATCH 13/30] TD-1719

---
 src/mnode/src/mnodeTable.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index b2f1436dba..ca9d6cff96 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -131,6 +131,8 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
       mnodeAddTableIntoStable(pTable->superTable, pTable);
       grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
       if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
+    } else {
+      mError("table:%s:%p, correspond stable not found suid:%" PRIu64, pTable->info.tableId, pTable, pTable->suid);
     }
   } else {
     grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
@@ -839,10 +841,11 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
+  int64_t us = taosGetTimestampUs();
   pStable->info.tableId = strdup(pCreate->tableId);
   pStable->info.type    = TSDB_SUPER_TABLE;
   pStable->createdTime  = taosGetTimestampMs();
-  pStable->uid          = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
+  pStable->uid          = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
   pStable->sversion     = 0;
   pStable->tversion     = 0;
   pStable->numOfColumns = htons(pCreate->numOfColumns);
@@ -1496,7 +1499,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
       continue;
     }
     if (pTable->vgHash == NULL) {
-      mError("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
+      mDebug("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
              stableName);
       mnodeDecTableRef(pTable);
 
@@ -1705,9 +1708,9 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   pTable->createdTime  = taosGetTimestampMs();
   pTable->sid          = tid;
   pTable->vgId         = pVgroup->vgId;
-    
+
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
+    STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
     if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
     if (pMsg->pSTable == NULL) {
       mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
@@ -2252,7 +2255,8 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
   int32_t numOfTables = 0;
   SChildTableObj *pTable = NULL;
 
-  mInfo("stable:%s, all child tables:%d will dropped from sdb", pStable->info.tableId, pStable->numOfTables);
+  mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid,
+        pStable->numOfTables);
 
   while (1) {
     pIter = mnodeGetNextChildTable(pIter, &pTable);

From 8623f145139af5e5a7ea233b397fbcfb3f2c8dec Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 14 Oct 2020 22:20:26 +0800
Subject: [PATCH 14/30] TD-1652

---
 src/dnode/src/dnodeVWrite.c |  37 +++++++------
 src/inc/taoserror.h         |   1 +
 src/mnode/src/mnodeSdb.c    |   2 +-
 src/sync/src/syncMain.c     |   8 +--
 src/vnode/src/vnodeRead.c   |  93 +++++++++++++++++---------------
 src/vnode/src/vnodeWrite.c  |  34 ++++++------
 src/wal/src/walMain.c       | 105 ++++++++++++++++++------------------
 7 files changed, 146 insertions(+), 134 deletions(-)

diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index 3f2c9df222..311bebc788 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -210,12 +210,12 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
 
 static void *dnodeProcessWriteQueue(void *param) {
   SWriteWorker *pWorker = (SWriteWorker *)param;
-  SWriteMsg    *pWrite;
-  SWalHead     *pHead;
+  SWriteMsg *   pWrite;
+  SWalHead *    pHead;
   int32_t       numOfMsgs;
   int           type;
-  void         *pVnode, *item;
-  SRspRet      *pRspRet;
+  void *        pVnode, *item;
+  SRspRet *     pRspRet;
 
   dDebug("write worker:%d is running", pWorker->workerId);
 
@@ -237,16 +237,21 @@ static void *dnodeProcessWriteQueue(void *param) {
         pHead->msgType = pWrite->rpcMsg.msgType;
         pHead->version = 0;
         pHead->len = pWrite->contLen;
-        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]);
+        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
+               taosMsg[pWrite->rpcMsg.msgType]);
       } else {
         pHead = (SWalHead *)item;
-        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version);
+        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
+               pHead->version);
       }
 
       int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
-      if (pWrite) { 
+      dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
+             pHead->version, tstrerror(code));
+
+      if (pWrite) {
         pWrite->rpcMsg.code = code;
-        if (code <= 0) pWrite->processedCount = 1; 
+        if (code <= 0) pWrite->processedCount = 1;
       }
     }
 
@@ -258,7 +263,7 @@ static void *dnodeProcessWriteQueue(void *param) {
       taosGetQitem(pWorker->qall, &type, &item);
       if (type == TAOS_QTYPE_RPC) {
         pWrite = (SWriteMsg *)item;
-        dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code); 
+        dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code);
       } else if (type == TAOS_QTYPE_FWD) {
         pHead = (SWalHead *)item;
         vnodeConfirmForward(pVnode, pHead->version, 0);
@@ -279,13 +284,13 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
   int32_t num = taosGetQueueNumber(pWorker->qset);
 
   if (num > 0) {
-     usleep(30000);
-     sched_yield();
+    usleep(30000);
+    sched_yield();
   } else {
-     taosFreeQall(pWorker->qall);
-     taosCloseQset(pWorker->qset);
-     pWorker->qset = NULL;
-     dDebug("write worker:%d is released", pWorker->workerId);
-     pthread_exit(NULL);
+    taosFreeQall(pWorker->qall);
+    taosCloseQset(pWorker->qset);
+    pWorker->qset = NULL;
+    dDebug("write worker:%d is released", pWorker->workerId);
+    pthread_exit(NULL);
   }
 }
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index 786342b5a6..2600bd2c55 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -247,6 +247,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores
 // sync
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sync version")
 
 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 895aa0400a..14558485aa 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -594,7 +594,7 @@ static int sdbWrite(void *param, void *data, int type) {
       pthread_mutex_unlock(&tsSdbObj.mutex);
       sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
                pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
-      return TSDB_CODE_MND_APP_ERROR;
+      return TSDB_CODE_SYN_INVALID_VERSION;
     } else {
       tsSdbObj.version = pHead->version;
     }
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index 4d5e19af77..f5f1ff5853 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -313,7 +313,8 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
 
   // always update version
   nodeVersion = pWalHead->version;
-  sDebug("replica:%d nodeRole:%d qtype:%d", pNode->replica, nodeRole, qtype);
+  sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
+         qtype, pWalHead->version);
 
   if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
 
@@ -883,7 +884,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
   SSyncNode *   pNode = pPeer->pSyncNode;
   SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
 
-  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+  sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
          syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
 
   pPeer->version = pPeersStatus->version;
@@ -970,7 +971,8 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
 
   int retLen = write(pPeer->peerFd, msg, statusMsgLen);
   if (retLen == statusMsgLen) {
-    sDebug("%s, status msg is sent", pPeer->id);
+    sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
+           pPeersStatus->version, pPeersStatus->ack);
   } else {
     sDebug("%s, failed to send status msg, restart", pPeer->id);
     syncRestartConnection(pPeer);
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index a3ed080f24..baaeae2a81 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -49,18 +49,18 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
 
   if (pVnode->status != TAOS_VN_STATUS_READY) {
     vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[msgType], pVnode->status);
-    return TSDB_CODE_APP_NOT_READY; 
+    return TSDB_CODE_APP_NOT_READY;
   }
 
-  // tsdb may be in reset state  
+  // tsdb may be in reset state
   if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING)
-    return TSDB_CODE_APP_NOT_READY;
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;
 
   // TODO: Later, let slave to support query
   // if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-  if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { 
-    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role);
+  if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
+    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[msgType],
+           pVnode->syncCfg.replica, syncRole[pVnode->role]);
     return TSDB_CODE_APP_NOT_READY;
   }
 
@@ -80,7 +80,7 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) {
   taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }
 
-static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle, bool* freeHandle) {
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle) {
   bool continueExec = false;
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -106,55 +106,56 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle,
   return code;
 }
 
-static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
   pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
   pRet->len = sizeof(SRetrieveTableRsp);
 
   memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  SRetrieveTableRsp* pRsp = pRet->rsp;
+  SRetrieveTableRsp *pRsp = pRet->rsp;
 
   pRsp->completed = true;
 }
 
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
-  void    *pCont = pReadMsg->pCont;
+  void *   pCont = pReadMsg->pCont;
   int32_t  contLen = pReadMsg->contLen;
   SRspRet *pRet = &pReadMsg->rspRet;
 
-  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont;
+  SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
   memset(pRet, 0, sizeof(SRspRet));
 
   // qHandle needs to be freed correctly
   if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-    SRetrieveTableMsg* killQueryMsg = (SRetrieveTableMsg*) pReadMsg->pCont;
+    SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
     killQueryMsg->free = htons(killQueryMsg->free);
     killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
 
-    vWarn("QInfo:%p connection %p broken, kill query", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
 
-    void** qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) killQueryMsg->qhandle);
+    void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
     if (qhandle == NULL || *qhandle == NULL) {
-      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
+            pReadMsg->rpcMsg.handle);
     } else {
-      assert(*qhandle == (void*) killQueryMsg->qhandle);
+      assert(*qhandle == (void *)killQueryMsg->qhandle);
 
       qKillQuery(*qhandle);
-      qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true);
     }
 
     return TSDB_CODE_TSC_QUERY_CANCELLED;
   }
 
   int32_t code = TSDB_CODE_SUCCESS;
-  void**  handle = NULL;
+  void ** handle = NULL;
 
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
     code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
 
-    SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
-    pRsp->code    = code;
+    SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
+    pRsp->code = code;
     pRsp->qhandle = 0;
 
     pRet->len = sizeof(SQueryTableRsp);
@@ -163,7 +164,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
     // current connect is broken
     if (code == TSDB_CODE_SUCCESS) {
-      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo);
+      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t)pQInfo);
       if (handle == NULL) {  // failed to register qhandle, todo add error test case
         vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
                tstrerror(pRsp->code));
@@ -171,13 +172,15 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
         qDestroyQueryInfo(pQInfo);  // destroy it directly
       } else {
         assert(*handle == pQInfo);
-        pRsp->qhandle = htobe64((uint64_t) pQInfo);
+        pRsp->qhandle = htobe64((uint64_t)pQInfo);
       }
 
-      if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+      if (handle != NULL &&
+          vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
+               pReadMsg->rpcMsg.handle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-        qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+        qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
         return pRsp->code;
       }
     } else {
@@ -190,12 +193,12 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     }
   } else {
     assert(pCont != NULL);
-    void** qhandle = (void**) pCont;
+    void **qhandle = (void **)pCont;
 
     vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
 
     bool freehandle = false;
-    bool buildRes = qTableQuery(*qhandle); // do execute query
+    bool buildRes = qTableQuery(*qhandle);  // do execute query
 
     // build query rsp, the retrieve request has reached here already
     if (buildRes) {
@@ -233,16 +236,17 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->free = htons(pRetrieve->free);
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
+         pRetrieve->free, pReadMsg->rpcMsg.handle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
   int32_t code = TSDB_CODE_SUCCESS;
-  void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
-  if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
+  void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
+  if (handle == NULL || (*handle) != (void *)pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
-    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
-    
+    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void *)pRetrieve->qhandle);
+
     vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
@@ -250,7 +254,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   if (pRetrieve->free == 1) {
     vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
     qKillQuery(*handle);
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
 
     vnodeBuildNoResultQueryRsp(pRet);
     code = TSDB_CODE_TSC_QUERY_CANCELLED;
@@ -259,26 +263,27 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   // register the qhandle to connect to quit query immediate if connection is broken
   if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
+           pReadMsg->rpcMsg.handle);
     code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     qKillQuery(*handle);
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
     return code;
   }
 
   bool freeHandle = true;
-  bool buildRes   = false;
+  bool buildRes = false;
 
   code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
   if (code != TSDB_CODE_SUCCESS) {
-    //TODO handle malloc failure
+    // TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     pRet->len = sizeof(SRetrieveTableRsp);
     memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
     freeHandle = true;
-  } else { // result is not ready, return immediately
+  } else {  // result is not ready, return immediately
     if (!buildRes) {
-      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
       return TSDB_CODE_QRY_NOT_READY;
     }
 
@@ -288,7 +293,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   // If qhandle is not added into vread queue, the query should be completed already or paused with error.
   // Here free qhandle immediately
   if (freeHandle) {
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
   }
 
   return code;
@@ -296,13 +301,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
 // notify connection(handle) that current qhandle is created, if current connection from
 // client is broken, the query needs to be killed immediately.
-int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
-  SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
-  killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
+int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) {
+  SRetrieveTableMsg *killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
+  killQueryMsg->qhandle = htobe64((uint64_t)qhandle);
   killQueryMsg->free = htons(1);
   killQueryMsg->header.vgId = htonl(vgId);
   killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
   vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle);
-  return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
+  return rpcReportProgress(handle, (char *)killQueryMsg, sizeof(SRetrieveTableMsg));
 }
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 0c310439bb..b1f4539a6c 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -47,11 +47,11 @@ void vnodeInitWriteFp(void) {
 int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   int32_t    code = 0;
   SVnodeObj *pVnode = (SVnodeObj *)param1;
-  SWalHead  *pHead = param2;
+  SWalHead * pHead = param2;
 
   if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
     vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
-    return TSDB_CODE_VND_MSG_NOT_PROCESSED; 
+    return TSDB_CODE_VND_MSG_NOT_PROCESSED;
   }
 
   if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
@@ -59,28 +59,29 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     return TSDB_CODE_VND_NO_WRITE_AUTH;
   }
 
-  // tsdb may be in reset state 
+  // tsdb may be in reset state
   if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING) 
-    return TSDB_CODE_APP_NOT_READY;
-  
-  if (pHead->version == 0) { // from client or CQ 
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;
+
+  if (pHead->version == 0) {  // from client or CQ
     if (pVnode->status != TAOS_VN_STATUS_READY) {
-      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->status);
+      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->status);
       return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
     }
 
     if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->syncCfg.replica, pVnode->role);
+      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->syncCfg.replica, syncRole[pVnode->role]);
       return TSDB_CODE_APP_NOT_READY;
     }
 
     // assign version
     pVnode->version++;
     pHead->version = pVnode->version;
-    if (pVnode->delay) usleep(pVnode->delay*1000);
+    if (pVnode->delay) usleep(pVnode->delay * 1000);
 
-  } else { // from wal or forward 
+  } else {  // from wal or forward
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
   }
@@ -91,12 +92,12 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   code = walWrite(pVnode->wal, pHead);
   if (code < 0) return code;
 
-  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync 
+  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
   int32_t syncCode = 0;
   syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
   if (syncCode < 0) return syncCode;
 
-  // write data locally 
+  // write data locally
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
   if (code < 0) return code;
 
@@ -115,14 +116,14 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
 
   // save insert result into item
   SShellSubmitRspMsg *pRsp = NULL;
-  if (pRet) {  
+  if (pRet) {
     pRet->len = sizeof(SShellSubmitRspMsg);
     pRet->rsp = rpcMallocCont(pRet->len);
     pRsp = pRet->rsp;
   }
 
   if (tsdbInsertData(pVnode->tsdb, pCont, pRsp) < 0) code = terrno;
-  
+
   return code;
 }
 
@@ -191,7 +192,7 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR
 
 int vnodeWriteToQueue(void *param, void *data, int type) {
   SVnodeObj *pVnode = param;
-  SWalHead *pHead = data;
+  SWalHead * pHead = data;
 
   int size = sizeof(SWalHead) + pHead->len;
   SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
@@ -204,4 +205,3 @@ int vnodeWriteToQueue(void *param, void *data, int type) {
 
   return 0;
 }
-
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
index 5d7f8ee20b..7e0f987213 100644
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@@ -63,7 +63,7 @@ static void walRelease(SWal *pWal);
 
 static void walModuleInitFunc() {
   walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL");
-  if (walTmrCtrl == NULL) 
+  if (walTmrCtrl == NULL)
     walModuleInit = PTHREAD_ONCE_INIT;
   else
     wDebug("WAL module is initialized");
@@ -90,7 +90,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     return NULL;
   }
 
-  atomic_add_fetch_32(&tsWalNum, 1);    
+  atomic_add_fetch_32(&tsWalNum, 1);
   pWal->fd = -1;
   pWal->max = pCfg->wals;
   pWal->id = 0;
@@ -117,18 +117,17 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     walRelease(pWal);
     pWal = NULL;
   }
-     
+
   if (pCfg->keep == 1) return pWal;
 
-  if (walHandleExistingFiles(path) == 0) 
-    walRenew(pWal);
+  if (walHandleExistingFiles(path) == 0) walRenew(pWal);
 
-  if (pWal && pWal->fd <0) {
+  if (pWal && pWal->fd < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to open(%s)", path, strerror(errno));
     walRelease(pWal);
     pWal = NULL;
-  } 
+  }
 
   if (pWal) wDebug("wal:%s, it is open, level:%d fsyncPeriod:%d", path, pWal->level, pWal->fsyncPeriod);
   return pWal;
@@ -154,7 +153,7 @@ int walAlter(twalh wal, const SWalCfg *pCfg) {
   pWal->fsyncPeriod = pCfg->fsyncPeriod;
   if (walNeedFsyncTimer(pWal)) {
     wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
-    taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer,walTmrCtrl);
+    taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl);
   } else {
     wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
     taosTmrStop(pWal->timer);
@@ -167,16 +166,16 @@ int walAlter(twalh wal, const SWalCfg *pCfg) {
 
 void walClose(void *handle) {
   if (handle == NULL) return;
-  
-  SWal *pWal = handle;  
+
+  SWal *pWal = handle;
   taosClose(pWal->fd);
   if (pWal->timer) taosTmrStopA(&pWal->timer);
 
   if (pWal->keep == 0) {
     // remove all files in the directory
-    for (int i=0; i<pWal->num; ++i) {
-      snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id-i);
-      if (remove(pWal->name) <0) {
+    for (int i = 0; i < pWal->num; ++i) {
+      snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i);
+      if (remove(pWal->name) < 0) {
         wError("wal:%s, failed to remove", pWal->name);
       } else {
         wDebug("wal:%s, it is removed", pWal->name);
@@ -197,7 +196,7 @@ int walRenew(void *handle) {
 
   pthread_mutex_lock(&pWal->mutex);
 
-  if (pWal->fd >=0) {
+  if (pWal->fd >= 0) {
     close(pWal->fd);
     pWal->id++;
     wDebug("wal:%s, it is closed", pWal->name);
@@ -218,7 +217,7 @@ int walRenew(void *handle) {
       // remove the oldest wal file
       char name[TSDB_FILENAME_LEN * 3];
       snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
-      if (remove(name) <0) {
+      if (remove(name) < 0) {
         wError("wal:%s, failed to remove(%s)", name, strerror(errno));
       } else {
         wDebug("wal:%s, it is removed", name);
@@ -226,8 +225,8 @@ int walRenew(void *handle) {
 
       pWal->num--;
     }
-  }  
-  
+  }
+
   pthread_mutex_unlock(&pWal->mutex);
 
   return terrno;
@@ -239,15 +238,18 @@ int walWrite(void *handle, SWalHead *pHead) {
 
   terrno = 0;
 
-  // no wal  
+  // no wal
   if (pWal->level == TAOS_WAL_NOLOG) return 0;
-  if (pHead->version <= pWal->version) return 0;
+  if (pHead->version <= pWal->version) {
+    wError("wal:%s, failed to write ver:%" PRIu64 ", last ver:%" PRIu64, pWal->name, pHead->version, pWal->version);
+    return 0;
+  }
 
   pHead->signature = walSignature;
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(taosTWrite(pWal->fd, pHead, contLen) != contLen) {
+  if (taosTWrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -258,7 +260,6 @@ int walWrite(void *handle, SWalHead *pHead) {
 }
 
 void walFsync(void *handle) {
-
   SWal *pWal = handle;
   if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return;
 
@@ -276,12 +277,11 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
   uint32_t maxId = 0, minId = -1, index =0;
 
   terrno = 0;
-  int   plen = strlen(walPrefix);
-  char  opath[TSDB_FILENAME_LEN+5];
-   
+  int  plen = strlen(walPrefix);
+  char opath[TSDB_FILENAME_LEN + 5];
+
   int slen = snprintf(opath, sizeof(opath), "%s", pWal->path);
-  if ( pWal->keep == 0) 
-    strcpy(opath+slen, "/old");
+  if (pWal->keep == 0) strcpy(opath + slen, "/old");
 
   DIR *dir = opendir(opath);
   if (dir == NULL && errno == ENOENT) return 0;
@@ -290,8 +290,8 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
     return terrno;
   }
 
-  while ((ent = readdir(dir))!= NULL) {
-    if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+  while ((ent = readdir(dir)) != NULL) {
+    if (strncmp(ent->d_name, walPrefix, plen) == 0) {
       index = atol(ent->d_name + plen);
       if (index > maxId) maxId = index;
       if (index < minId) minId = index;
@@ -306,13 +306,13 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
     return terrno;
   }
 
-  if ( count != (maxId-minId+1) ) {
+  if (count != (maxId - minId + 1)) {
     wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
     terrno = TSDB_CODE_WAL_APP_ERROR;
   } else {
     wDebug("wal:%s, %d files will be restored", opath, count);
 
-    for (index = minId; index<=maxId; ++index) {
+    for (index = minId; index <= maxId; ++index) {
       snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
       terrno = walRestoreWalFile(pWal, pVnode, writeFp);
       if (terrno < 0) break;
@@ -328,7 +328,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
           terrno = TAOS_SYSTEM_ERROR(errno);
         }
       }
-    } else { 
+    } else {
       // open the existing WAL file in append mode
       pWal->num = count;
       pWal->id = maxId;
@@ -345,9 +345,9 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
 }
 
 int walGetWalFile(void *handle, char *name, uint32_t *index) {
-  SWal   *pWal = handle;
+  SWal *  pWal = handle;
   int     code = 1;
-  int32_t first = 0; 
+  int32_t first = 0;
 
   name[0] = 0;
   if (pWal == NULL || pWal->num == 0) return 0;
@@ -359,18 +359,17 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
 
   if (*index < first && *index > pWal->id) {
     code = -1;  // index out of range
-  } else { 
+  } else {
     sprintf(name, "wal/%s%d", walPrefix, *index);
-    code = (*index == pWal->id) ? 0:1;
+    code = (*index == pWal->id) ? 0 : 1;
   }
 
   pthread_mutex_unlock(&(pWal->mutex));
 
   return code;
-}  
+}
 
 static void walRelease(SWal *pWal) {
-
   pthread_mutex_destroy(&pWal->mutex);
   pWal->signature = NULL;
   free(pWal);
@@ -385,12 +384,12 @@ static void walRelease(SWal *pWal) {
 
 static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   char *name = pWal->name;
-  int size = 1024 * 1024; // default 1M buffer size
+  int   size = 1024 * 1024;  // default 1M buffer size
 
   terrno = 0;
   char *buffer = malloc(size);
   if (buffer == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);   
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return terrno;
   }
 
@@ -487,21 +486,21 @@ int walHandleExistingFiles(const char *path) {
   } else {
     // move all files to old directory
     int count = 0;
-    while ((ent = readdir(dir))!= NULL) {  
-      if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+    while ((ent = readdir(dir)) != NULL) {
+      if (strncmp(ent->d_name, walPrefix, plen) == 0) {
         snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
         snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
         if (taosMkDir(opath, 0755) != 0) {
           wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
           terrno = TAOS_SYSTEM_ERROR(errno);
-          break; 
+          break;
         }
 
         if (rename(oname, nname) < 0) {
           wError("wal:%s, failed to move to new:%s", oname, nname);
           terrno = TAOS_SYSTEM_ERROR(errno);
           break;
-        } 
+        }
 
         count++;
       }
@@ -509,34 +508,34 @@ int walHandleExistingFiles(const char *path) {
 
     wDebug("wal:%s, %d files are moved for restoration", path, count);
   }
-  
+
   closedir(dir);
   return terrno;
 }
 
 static int walRemoveWalFiles(const char *path) {
-  int    plen = strlen(walPrefix);
-  char   name[TSDB_FILENAME_LEN * 3];
- 
+  int  plen = strlen(walPrefix);
+  char name[TSDB_FILENAME_LEN * 3];
+
   terrno = 0;
 
   struct dirent *ent;
-  DIR   *dir = opendir(path);
+  DIR *dir = opendir(path);
   if (dir == NULL && errno == ENOENT) return 0;
   if (dir == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return terrno;
-  }  
+  }
 
-  while ((ent = readdir(dir))!= NULL) {
-    if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
+  while ((ent = readdir(dir)) != NULL) {
+    if (strncmp(ent->d_name, walPrefix, plen) == 0) {
       snprintf(name, sizeof(name), "%s/%s", path, ent->d_name);
-      if (remove(name) <0) {
+      if (remove(name) < 0) {
         wError("wal:%s, failed to remove(%s)", name, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
       }
     }
-  } 
+  }
 
   closedir(dir);
 

From 6f004e4c7a759b9fc4d3ecfab8725aca10e87ea5 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 14 Oct 2020 22:27:19 +0800
Subject: [PATCH 15/30] TD-1652

---
 src/sync/src/syncMain.c    | 10 ++++++++++
 src/vnode/src/vnodeWrite.c | 13 ++++++-------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index f5f1ff5853..ef635e6efc 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -311,6 +311,16 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
 
   if (pNode == NULL) return 0;
 
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
+    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
+           pWalHead->version, nodeVersion);
+    for (int i = 0; i < pNode->replica; ++i) {
+      pPeer = pNode->peerInfo[i];
+      syncRestartConnection(pPeer);
+    }
+    return TSDB_CODE_SYN_INVALID_VERSION;
+  }
+
   // always update version
   nodeVersion = pWalHead->version;
   sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index b1f4539a6c..70b08b9669 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -77,8 +77,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     }
 
     // assign version
-    pVnode->version++;
-    pHead->version = pVnode->version;
+    pHead->version = pVnode->version + 1;
     if (pVnode->delay) usleep(pVnode->delay * 1000);
 
   } else {  // from wal or forward
@@ -86,16 +85,16 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
     if (pHead->version <= pVnode->version) return 0;
   }
 
-  pVnode->version = pHead->version;
+  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
+  int32_t syncCode = 0;
+  syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
+  if (syncCode < 0) return syncCode;
 
   // write into WAL
   code = walWrite(pVnode->wal, pHead);
   if (code < 0) return code;
 
-  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
-  int32_t syncCode = 0;
-  syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
-  if (syncCode < 0) return syncCode;
+  pVnode->version = pHead->version;
 
   // write data locally
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);

From b6786bc66183e5f24d3dfc43cbe16340229a073c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 23:09:16 +0800
Subject: [PATCH 16/30] [td-1637]

---
 src/client/inc/tsclient.h       | 17 ++++-----
 src/client/src/tscAsync.c       |  2 +-
 src/client/src/tscProfile.c     | 10 +++---
 src/client/src/tscServer.c      | 39 ---------------------
 src/client/src/tscSql.c         | 62 +++++++++++++++++++++++++++++----
 src/client/src/tscSubquery.c    | 18 ++++++++--
 src/kit/shell/inc/shell.h       |  1 +
 src/kit/shell/src/shellEngine.c |  4 +--
 8 files changed, 88 insertions(+), 65 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index b6ab3702c9..8621f9d28b 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -80,6 +80,8 @@ enum {
   DATA_FROM_DATA_FILE = 2,
 };
 
+typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
+
 typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
@@ -226,7 +228,7 @@ typedef struct STableDataBlocks {
 typedef struct SQueryInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
   uint32_t         type;          // query/insert type
-  // TODO refactor
+
   STimeWindow      window;        // query time window
   SInterval        interval;
 
@@ -440,19 +442,20 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql);
  * @param pObj
  */
 void tscFreeSqlObj(SSqlObj *pSql);
-
-void tscFreeSqlObjInCache(void *pSql);
+void tscFreeRegisteredSqlObj(void *pSql);
 
 void tscCloseTscObj(STscObj *pObj);
 
+// todo move to taos? or create a new file: taos_internal.h
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos);
-void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res);
 
-void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
+void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
+
+void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
 
 void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
-void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
 bool tscIsUpdateQuery(SSqlObj* pSql);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
@@ -517,8 +520,6 @@ extern SRpcCorEpSet tscMgmtEpSet;
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 
-typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
-
 int32_t tscCompareTidTags(const void* p1, const void* p2);
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
 
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index a76b77bb86..639de294e6 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
 static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
 
-void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
+void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) {
   SSqlCmd* pCmd = &pSql->cmd;
 
   pSql->signature = pSql;
diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c
index bcd52d3a42..bae0f91dcc 100644
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@@ -151,10 +151,12 @@ void tscKillQuery(STscObj *pObj, uint32_t killId) {
 
   pthread_mutex_unlock(&pObj->mutex);
 
-  if (pSql == NULL) return;
-
-  tscDebug("%p query is killed, queryId:%d", pSql, killId);
-  taos_stop_query(pSql);
+  if (pSql == NULL) {
+    tscError("failed to kill query, id:%d, it may have completed/terminated", killId);
+  } else {
+    tscDebug("%p query is killed, queryId:%d", pSql, killId);
+    taos_stop_query(pSql);
+  }
 }
 
 void tscAddIntoStreamList(SSqlStream *pStream) {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 1161bae424..494a8a9c30 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -467,45 +467,6 @@ int tscProcessSql(SSqlObj *pSql) {
   return doProcessSql(pSql);
 }
 
-void tscKillSTableQuery(SSqlObj *pSql) {
-  SSqlCmd* pCmd = &pSql->cmd;
-  
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    return;
-  }
-
-  // set the master sqlObj flag to cancel query
-  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-
-  for (int i = 0; i < pSql->subState.numOfSub; ++i) {
-    // NOTE: pSub may have been released already here
-    SSqlObj *pSub = pSql->pSubs[i];
-    if (pSub == NULL) {
-      continue;
-    }
-
-    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
-    if (p == NULL) {
-      continue;
-    }
-
-    SSqlObj* pSubObj = (SSqlObj*) (*p);
-    assert(pSubObj->self == (SSqlObj**) p);
-
-    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    if (pSubObj->pRpcCtx != NULL) {
-      rpcCancelRequest(pSubObj->pRpcCtx);
-      pSubObj->pRpcCtx = NULL;
-    }
-
-//    tscQueueAsyncRes(pSubObj); // async res? not other functions?
-    taosCacheRelease(tscObjCache, (void**) &p, false);
-  }
-
-  tscDebug("%p super table query cancelled", pSql);
-}
-
 int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
   pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index f73b4abb15..e7369e002b 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -307,7 +307,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
   tsem_post(&pSql->rspSem);
 }
 
-TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
+TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
@@ -332,12 +332,20 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
   tsem_init(&pSql->rspSem, 0, 0);
   doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
 
+  if (res != NULL) {
+    *res = pSql;
+  }
+
   tsem_wait(&pSql->rspSem);
   return pSql; 
 }
 
 TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
-  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr));
+  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL);
+}
+
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) {
+  return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res);
 }
 
 int taos_result_precision(TAOS_RES *res) {
@@ -689,6 +697,45 @@ int* taos_fetch_lengths(TAOS_RES *res) {
 
 char *taos_get_client_info() { return version; }
 
+static void tscKillSTableQuery(SSqlObj *pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    return;
+  }
+
+  // set the master sqlObj flag to cancel query
+  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
+  for (int i = 0; i < pSql->subState.numOfSub; ++i) {
+    // NOTE: pSub may have been released already here
+    SSqlObj *pSub = pSql->pSubs[i];
+    if (pSub == NULL) {
+      continue;
+    }
+
+    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
+    if (p == NULL) {
+      continue;
+    }
+
+    SSqlObj* pSubObj = (SSqlObj*) (*p);
+    assert(pSubObj->self == (SSqlObj**) p);
+
+    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+    if (pSubObj->pRpcCtx != NULL) {
+      rpcCancelRequest(pSubObj->pRpcCtx);
+      pSubObj->pRpcCtx = NULL;
+    }
+
+    tscQueueAsyncRes(pSubObj);
+    taosCacheRelease(tscObjCache, (void**) &p, false);
+  }
+
+  tscDebug("%p super table query cancelled", pSql);
+}
+
 void taos_stop_query(TAOS_RES *res) {
   SSqlObj *pSql = (SSqlObj *)res;
   if (pSql == NULL || pSql->signature != pSql) {
@@ -698,19 +745,20 @@ void taos_stop_query(TAOS_RES *res) {
   tscDebug("%p start to cancel query", res);
   SSqlCmd *pCmd = &pSql->cmd;
 
-  // TODO there are multi-thread problem.
-  // It may have been released by the other thread already.
-  // The ref count may fix this problem.
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-
   // set the error code for master pSqlObj firstly
   pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
 
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
     assert(pSql->pRpcCtx == NULL);
     tscKillSTableQuery(pSql);
   } else {
     if (pSql->cmd.command < TSDB_SQL_LOCAL) {
+      /*
+       * There is multi-thread problem here, since pSql->pRpcCtx may have been
+       * reset and freed in the processMsgFromServer function, and causes the invalid
+       * write problem for rpcCancelRequest.
+       */
       if (pSql->pRpcCtx != NULL) {
         rpcCancelRequest(pSql->pRpcCtx);
         pSql->pRpcCtx = NULL;
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 185d6784d1..5ccb2aee8c 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1563,6 +1563,11 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 }
 
 void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
+  // it has been freed already
+  if (pSql->param != trsupport || pSql->param == NULL) {
+    return;
+  }
+
   SSqlObj *pParentSql = trsupport->pParentSql;
   int32_t  subqueryIndex = trsupport->subqueryIndex;
   
@@ -1709,14 +1714,21 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
 }
 
 static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
+  SSqlObj *pSql = (SSqlObj *)tres;
+  assert(pSql != NULL);
+
+  // this query has been freed already
   SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
+  if (pSql->param == NULL || param == NULL) {
+    tscDebug("%p already freed in dnodecallback", pSql);
+    assert(pSql->res.code == TSDB_CODE_TSC_QUERY_CANCELLED);
+    return;
+  }
+
   tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
   int32_t           idx   = trsupport->subqueryIndex;
   SSqlObj *         pParentSql = trsupport->pParentSql;
 
-  SSqlObj *pSql = (SSqlObj *)tres;
-  assert(pSql != NULL && trsupport == pSql->param);
-
   SSubqueryState* pState = &pParentSql->subState;
   assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
   
diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h
index dd62df170a..765181dbba 100644
--- a/src/kit/shell/inc/shell.h
+++ b/src/kit/shell/inc/shell.h
@@ -20,6 +20,7 @@
 #include "taos.h"
 #include "taosdef.h"
 #include "stdbool.h"
+#include "tsclient.h"
 
 #define MAX_USERNAME_SIZE      64
 #define MAX_DBNAME_SIZE        64
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index d5e826fbaa..24388bf50c 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -294,9 +294,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
 
   st = taosGetTimestampUs();
 
-  TAOS_RES* pSql = taos_query(con, command);
-  atomic_store_ptr(&result, pSql);  // set the global TAOS_RES pointer
-
+  TAOS_RES* pSql = taos_query_h(con, command, &result);
   if (taos_errno(pSql)) {
     taos_error(pSql);
     return;

From 83a1d5e54e005ded2b4bc7b610fd06bbe4a50dbd Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 23:12:17 +0800
Subject: [PATCH 17/30] [td-225] refactor code.

---
 src/client/src/tscSystem.c | 2 +-
 src/client/src/tscUtil.c   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 85c8a57058..4c5dbb079f 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -141,7 +141,7 @@ void taos_init_imp(void) {
   int64_t refreshTime = 10; // 10 seconds by default
   if (tscMetaCache == NULL) {
     tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeSqlObjInCache, "sqlObj");
+    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
   }
 
   tscDebug("client is initialized successfully");
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 33c51c5571..0235f037bd 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -389,7 +389,7 @@ static void tscFreeSubobj(SSqlObj* pSql) {
  *
  * @param pSql
  */
-void tscFreeSqlObjInCache(void *pSql) {
+void tscFreeRegisteredSqlObj(void *pSql) {
   assert(pSql != NULL);
 
   SSqlObj** p = (SSqlObj**)pSql;

From db951b38f303c412b4b08ad6c3fab0a41830c209 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 14 Oct 2020 23:47:36 +0800
Subject: [PATCH 18/30] [td-1637].

---
 src/client/src/tscSql.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index e7369e002b..2108b7fd06 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -763,6 +763,8 @@ void taos_stop_query(TAOS_RES *res) {
         rpcCancelRequest(pSql->pRpcCtx);
         pSql->pRpcCtx = NULL;
       }
+
+      tscQueueAsyncRes(pSql);
     }
   }
 

From 89a76a0d66c7bf51416484cacbca68098d959c35 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Thu, 15 Oct 2020 09:47:37 +0800
Subject: [PATCH 19/30] fix td-1706: leak of connection id

---
 src/rpc/src/rpcMain.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index ad247ad25b..63231bb1fa 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -1051,6 +1051,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     if (code != 0) { // parsing error
       if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
+        if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) {
+          rpcCloseConn(pConn);
+        }
         tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 

From 3f52d74a8ed71350578a7293fc065cbf02bd52f6 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 15 Oct 2020 03:23:47 +0000
Subject: [PATCH 20/30] TD-1696

---
 src/inc/taoserror.h       |  1 +
 src/inc/twal.h            |  1 +
 src/vnode/src/vnodeMain.c | 16 +++++++++++-----
 src/wal/src/walMain.c     | 12 ++++++++----
 4 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index 2600bd2c55..08621a81c6 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -193,6 +193,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "No write p
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "Missing data file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "Out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "Unexpected generic error in vnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE,     0, 0x050A, "Invalid version file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")
 
diff --git a/src/inc/twal.h b/src/inc/twal.h
index cf570aefdc..92204abd7d 100644
--- a/src/inc/twal.h
+++ b/src/inc/twal.h
@@ -51,6 +51,7 @@ int     walWrite(twalh, SWalHead *);
 void    walFsync(twalh);
 int     walRestore(twalh, void *pVnode, FWalWrite writeFp);
 int     walGetWalFile(twalh, char *name, uint32_t *index);
+int64_t walGetVersion(twalh);
 
 extern int wDebugFlag;
 
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index d89c383d6a..0c1f9d103b 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -239,8 +239,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   code = vnodeReadVersion(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
-    vnodeCleanUp(pVnode);
-    return code;
+    vError("vgId:%d, failed to read version, generate it from data file", pVnode->vgId);
+    // Allow vnode start even when read version fails, set version as walVersion or zero
+    // vnodeCleanUp(pVnode);
+    // return code;
   }
 
   pVnode->fversion = pVnode->version;
@@ -292,6 +294,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   }
 
   walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
+  if (pVnode->version == 0) {
+    pVnode->version = walGetVersion(pVnode->wal);
+  }
 
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
@@ -947,6 +952,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
+  fflush(fp);
   fclose(fp);
 
   vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion);
@@ -960,7 +966,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
   cJSON  *root = NULL;
   int     maxLen = 100;
 
-  terrno = TSDB_CODE_VND_APP_ERROR;
+  terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
   sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
   FILE *fp = fopen(versionFile, "r");
   if (!fp) {
@@ -974,7 +980,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
   }
 
   content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
+  int len = fread(content, 1, maxLen, fp);
   if (len <= 0) {
     vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
     goto PARSE_OVER;
@@ -999,6 +1005,6 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
 PARSE_OVER:
   taosTFree(content);
   cJSON_Delete(root);
-  if(fp) fclose(fp);
+  if (fp) fclose(fp);
   return terrno;
 }
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
index 7e0f987213..d7fd1b84c9 100644
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@@ -240,10 +240,7 @@ int walWrite(void *handle, SWalHead *pHead) {
 
   // no wal
   if (pWal->level == TAOS_WAL_NOLOG) return 0;
-  if (pHead->version <= pWal->version) {
-    wError("wal:%s, failed to write ver:%" PRIu64 ", last ver:%" PRIu64, pWal->name, pHead->version, pWal->version);
-    return 0;
-  }
+  if (pHead->version <= pWal->version) return 0;
 
   pHead->signature = walSignature;
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
@@ -560,3 +557,10 @@ static void walProcessFsyncTimer(void *param, void *tmrId) {
     pWal->timer = NULL;
   }
 }
+
+int64_t walGetVersion(twalh param) {
+  SWal *pWal = param;
+  if (pWal == 0) return 0;
+
+  return pWal->version;
+}
\ No newline at end of file

From 331b838ffa6593901ebb72039147b76e8461c7bc Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Thu, 15 Oct 2020 11:38:17 +0800
Subject: [PATCH 21/30] [TD-1676] balance jenkins

---
 Jenkinsfile       | 25 +++++++++++++++++++++++++
 tests/test-all.sh |  2 +-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 55fa27cf28..c262aa50f7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -29,6 +29,31 @@ pipeline {
 
     stage('Parallel test stage') {
       parallel {
+        stage('pytest') {
+          agent{label 'master'}
+          steps {
+            sh '''
+            date
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests
+            #./test-all.sh smoke
+            ./test-all.sh pytest
+            date'''
+          }
+        }
         stage('test_b1') {
           agent{label '184'}
           steps {
diff --git a/tests/test-all.sh b/tests/test-all.sh
index 983ef5ffec..e45dd15fed 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -121,7 +121,7 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
-  elif [ "$1" == "b1" ]; then
+  elif [ "$1" == "pytest" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
   elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then

From efec15d2670f0ed17caa9fcfae93fa0dd708b5a2 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 15 Oct 2020 13:59:08 +0800
Subject: [PATCH 22/30] compiled error in wn32

---
 cmake/env.inc                | 2 +-
 deps/zlib-1.2.11/inc/zconf.h | 2 +-
 deps/zlib-1.2.11/src/zconf.h | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/env.inc b/cmake/env.inc
index 67b934119d..18a6fea51d 100755
--- a/cmake/env.inc
+++ b/cmake/env.inc
@@ -45,7 +45,7 @@ IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
 ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
   MESSAGE(STATUS "Build Release Version")
 ELSE ()
-  IF (TD_WINDOWS_64)
+  IF (TD_WINDOWS)
     SET(CMAKE_BUILD_TYPE "Release")
     MESSAGE(STATUS "Build Release Version in Windows as default")
   ELSE ()
diff --git a/deps/zlib-1.2.11/inc/zconf.h b/deps/zlib-1.2.11/inc/zconf.h
index 9b83d3a898..b056fc5cf4 100644
--- a/deps/zlib-1.2.11/inc/zconf.h
+++ b/deps/zlib-1.2.11/inc/zconf.h
@@ -472,7 +472,7 @@ typedef uint64_t  z_crc_t;
 #endif
 #ifndef Z_SOLO
 #if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else
diff --git a/deps/zlib-1.2.11/src/zconf.h b/deps/zlib-1.2.11/src/zconf.h
index 361c098281..937d2b70ca 100644
--- a/deps/zlib-1.2.11/src/zconf.h
+++ b/deps/zlib-1.2.11/src/zconf.h
@@ -472,7 +472,7 @@ typedef uLong FAR uLongf;
 #endif
 #ifndef Z_SOLO
 #  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else

From b1cc403da165864faf50c93e77098641e20ef77c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 15 Oct 2020 18:23:28 +0800
Subject: [PATCH 23/30] TD-1725

---
 src/balance/src/balance.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
index fc1853f3d8..3ae65674c8 100644
--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@@ -126,6 +126,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
 
   balanceAccquireDnodeList();
 
+  mDebug("db:%s, alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
+         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     for (; dnode < tsBalanceDnodeListSize; ++dnode) {
       SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
@@ -135,17 +137,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
         pVnodeGid->pDnode = pDnode;
         dnode++;
         vnodes++;
+        mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i);
         break;
+      } else {
+        mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
+               mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
+               pDnode->alternativeRole);
       }
     }
   }
 
   if (vnodes != pVgroup->numOfVnodes) {
-    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
-           pVgroup->numOfVnodes, vnodes);
     balanceReleaseDnodeList();
     balanceUnLock();
 
+    mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
+
+    void *     pIter = NULL;
+    SDnodeObj *pDnode = NULL;
+    while (1) {
+      pIter = mnodeGetNextDnode(pIter, &pDnode);
+      if (pDnode == NULL) break;
+      mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
+             pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
+      mnodeDecDnodeRef(pDnode);
+    }
+    sdbFreeIter(pIter);
+
     if (mnodeGetOnlineDnodesNum() == 0) {
       return TSDB_CODE_MND_NOT_READY;
     } else {

From bca09e93e30157e595af2083779e455a3e0bcbf3 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Fri, 16 Oct 2020 05:18:59 +0000
Subject: [PATCH 24/30] TD-1705

---
 tests/pytest/fulltest.sh                      |  34 ++---
 .../pytest/functions/function_avg_restart.py  |  73 ++++++++++
 .../functions/function_bottom_restart.py      |  93 ++++++++++++
 .../functions/function_count_restart.py       |  79 ++++++++++
 .../pytest/functions/function_diff_restart.py |  99 +++++++++++++
 .../functions/function_first_restart.py       |  78 ++++++++++
 .../pytest/functions/function_last_restart.py |  78 ++++++++++
 .../functions/function_last_row_restart.py    |  78 ++++++++++
 .../function_leastsquares_restart.py          |  68 +++++++++
 .../pytest/functions/function_max_restart.py  |  76 ++++++++++
 .../pytest/functions/function_min_restart.py  |  73 ++++++++++
 .../functions/function_operations_restart.py  |  70 +++++++++
 .../functions/function_percentile_restart.py  | 136 ++++++++++++++++++
 .../functions/function_spread_restart.py      |  69 +++++++++
 .../functions/function_stddev_restart.py      |  75 ++++++++++
 .../pytest/functions/function_sum_restart.py  |  64 +++++++++
 .../pytest/functions/function_top_restart.py  |  93 ++++++++++++
 .../pytest/functions/function_twa_restart.py  | 130 +++++++++++++++++
 tests/pytest/test.py                          |  24 +++-
 19 files changed, 1471 insertions(+), 19 deletions(-)
 create mode 100644 tests/pytest/functions/function_avg_restart.py
 create mode 100644 tests/pytest/functions/function_bottom_restart.py
 create mode 100644 tests/pytest/functions/function_count_restart.py
 create mode 100644 tests/pytest/functions/function_diff_restart.py
 create mode 100644 tests/pytest/functions/function_first_restart.py
 create mode 100644 tests/pytest/functions/function_last_restart.py
 create mode 100644 tests/pytest/functions/function_last_row_restart.py
 create mode 100644 tests/pytest/functions/function_leastsquares_restart.py
 create mode 100644 tests/pytest/functions/function_max_restart.py
 create mode 100644 tests/pytest/functions/function_min_restart.py
 create mode 100644 tests/pytest/functions/function_operations_restart.py
 create mode 100644 tests/pytest/functions/function_percentile_restart.py
 create mode 100644 tests/pytest/functions/function_spread_restart.py
 create mode 100644 tests/pytest/functions/function_stddev_restart.py
 create mode 100644 tests/pytest/functions/function_sum_restart.py
 create mode 100644 tests/pytest/functions/function_top_restart.py
 create mode 100644 tests/pytest/functions/function_twa_restart.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index b679942054..fd973b8b76 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -172,23 +172,23 @@ python3 testNoCompress.py
 python3 testMinTablesPerVnode.py
 
 # functions
-python3 ./test.py -f functions/function_avg.py
-python3 ./test.py -f functions/function_bottom.py
-python3 ./test.py -f functions/function_count.py
-python3 ./test.py -f functions/function_diff.py
-python3 ./test.py -f functions/function_first.py
-python3 ./test.py -f functions/function_last.py
-python3 ./test.py -f functions/function_last_row.py
-python3 ./test.py -f functions/function_leastsquares.py
-python3 ./test.py -f functions/function_max.py
-python3 ./test.py -f functions/function_min.py
-python3 ./test.py -f functions/function_operations.py
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
 python3 ./test.py -f functions/function_percentile.py
-python3 ./test.py -f functions/function_spread.py
-python3 ./test.py -f functions/function_stddev.py
-python3 ./test.py -f functions/function_sum.py
-python3 ./test.py -f functions/function_top.py
-#python3 ./test.py -f functions/function_twa.py
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+#python3 ./test.py -f functions/function_twa.py -r 1
 python3 queryCount.py
 python3 ./test.py -f query/queryGroupbyWithInterval.py
 python3 client/twoClients.py
@@ -200,4 +200,4 @@ python3 test.py -f tools/taosdemo.py
 # subscribe
 python3 test.py -f subscribe/singlemeter.py
 #python3 test.py -f subscribe/stability.py
-python3 test.py -f subscribe/supertable.py
\ No newline at end of file
+python3 test.py -f subscribe/supertable.py
diff --git a/tests/pytest/functions/function_avg_restart.py b/tests/pytest/functions/function_avg_restart.py
new file mode 100644
index 0000000000..56b99cdf91
--- /dev/null
+++ b/tests/pytest/functions/function_avg_restart.py
@@ -0,0 +1,73 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        for i in range(self.rowNum):
+            #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+            #            % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # average verifacation 
+        tdSql.error("select avg(ts) from test")
+        tdSql.error("select avg(ts) from test1")
+        tdSql.error("select avg(col7) from test")
+        tdSql.error("select avg(col7) from test1")
+        tdSql.error("select avg(col8) from test")
+        tdSql.error("select avg(col8) from test1")
+        tdSql.error("select avg(col9) from test")
+        tdSql.error("select avg(col9) from test1")
+
+        tdSql.query("select avg(col1) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col2) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col3) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col4) from test")
+        tdSql.checkData(0, 0, np.average(intData))
+        tdSql.query("select avg(col5) from test")
+        tdSql.checkData(0, 0, np.average(floatData))              
+        tdSql.query("select avg(col6) from test")
+        tdSql.checkData(0, 0, np.average(floatData))              
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_bottom_restart.py b/tests/pytest/functions/function_bottom_restart.py
new file mode 100644
index 0000000000..74eb044645
--- /dev/null
+++ b/tests/pytest/functions/function_bottom_restart.py
@@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")    
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                            
+
+        # bottom verifacation 
+        tdSql.error("select bottom(ts, 10) from test")
+        tdSql.error("select bottom(col1, 0) from test")
+        tdSql.error("select bottom(col1, 101) from test")
+        tdSql.error("select bottom(col2, 0) from test")
+        tdSql.error("select bottom(col2, 101) from test")
+        tdSql.error("select bottom(col3, 0) from test")
+        tdSql.error("select bottom(col3, 101) from test")
+        tdSql.error("select bottom(col4, 0) from test")
+        tdSql.error("select bottom(col4, 101) from test")
+        tdSql.error("select bottom(col5, 0) from test")
+        tdSql.error("select bottom(col5, 101) from test")
+        tdSql.error("select bottom(col6, 0) from test")
+        tdSql.error("select bottom(col6, 101) from test")        
+        tdSql.error("select bottom(col7, 10) from test")        
+        tdSql.error("select bottom(col8, 10) from test")
+        tdSql.error("select bottom(col9, 10) from test")
+
+        tdSql.query("select bottom(col1, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col2, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col3, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col4, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select bottom(col5, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 0.1)
+        tdSql.checkData(1, 1, 1.1)
+
+        tdSql.query("select bottom(col6, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 0.1)
+        tdSql.checkData(1, 1, 1.1)
+                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_count_restart.py b/tests/pytest/functions/function_count_restart.py
new file mode 100644
index 0000000000..5eabb47d95
--- /dev/null
+++ b/tests/pytest/functions/function_count_restart.py
@@ -0,0 +1,79 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
+        
+        # Count verifacation
+        tdSql.query("select count(*) from test")        
+        tdSql.checkData(0, 0, 11)
+
+        tdSql.query("select count(ts) from test")        
+        tdSql.checkData(0, 0, 11)                        
+        tdSql.query("select count(col1) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col2) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col3) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col4) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col5) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col6) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col7) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col8) from test")        
+        tdSql.checkData(0, 0, 11)
+        tdSql.query("select count(col9) from test")        
+        tdSql.checkData(0, 0, 11)
+
+        #tdSql.execute("alter table test add column col10 int")
+        #tdSql.query("select count(col10) from test")        
+        #tdSql.checkRows(0)
+
+        ##tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)")
+        tdSql.query("select count(col10) from test")        
+        tdSql.checkData(0, 0, 1)        
+              
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_diff_restart.py b/tests/pytest/functions/function_diff_restart.py
new file mode 100644
index 0000000000..870ee03fc9
--- /dev/null
+++ b/tests/pytest/functions/function_diff_restart.py
@@ -0,0 +1,99 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #            col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        #tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1))
+        
+        # diff verifacation 
+        #tdSql.query("select diff(col1) from test1")
+        #tdSql.checkRows(0)
+        #
+        #tdSql.query("select diff(col2) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col3) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col4) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col5) from test1")
+        #tdSql.checkRows(0)
+
+        #tdSql.query("select diff(col6) from test1")
+        #tdSql.checkRows(0)
+
+        #for i in range(self.rowNum):
+        #    tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
+        #                % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                                
+        
+        tdSql.error("select diff(ts) from test")
+        tdSql.error("select diff(ts) from test1")
+        tdSql.error("select diff(col1) from test")
+        tdSql.error("select diff(col2) from test")
+        tdSql.error("select diff(col3) from test")
+        tdSql.error("select diff(col4) from test")
+        tdSql.error("select diff(col5) from test")
+        tdSql.error("select diff(col6) from test")
+        tdSql.error("select diff(col7) from test")     
+        tdSql.error("select diff(col7) from test1")               
+        tdSql.error("select diff(col8) from test") 
+        tdSql.error("select diff(col8) from test1")
+        tdSql.error("select diff(col9) from test")        
+        tdSql.error("select diff(col9) from test1")
+        
+        tdSql.query("select diff(col1) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col2) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col3) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col4) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col5) from test1")
+        tdSql.checkRows(10)
+
+        tdSql.query("select diff(col6) from test1")
+        tdSql.checkRows(10)
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_first_restart.py b/tests/pytest/functions/function_first_restart.py
new file mode 100644
index 0000000000..31904c3cde
--- /dev/null
+++ b/tests/pytest/functions/function_first_restart.py
@@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select first(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+
+        tdSql.query("select first(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdSql.query("select first(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 0.1)
+
+        tdSql.query("select first(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 0.1)
+
+        tdSql.query("select first(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, False)
+
+        tdSql.query("select first(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata1')
+
+        tdSql.query("select first(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据1')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_last_restart.py b/tests/pytest/functions/function_last_restart.py
new file mode 100644
index 0000000000..69a588d28d
--- /dev/null
+++ b/tests/pytest/functions/function_last_restart.py
@@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select last(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 10)
+
+        tdSql.query("select last(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+
+        tdSql.query("select last(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata10')
+
+        tdSql.query("select last(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据10')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_last_row_restart.py b/tests/pytest/functions/function_last_row_restart.py
new file mode 100644
index 0000000000..1043a8809d
--- /dev/null
+++ b/tests/pytest/functions/function_last_row_restart.py
@@ -0,0 +1,78 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        tdSql.query("select last_row(*) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 10)
+
+        tdSql.query("select last_row(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select last_row(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last_row(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select last_row(col7) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, True)
+
+        tdSql.query("select last_row(col8) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'taosdata10')
+
+        tdSql.query("select last_row(col9) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, '涛思数据10')
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_leastsquares_restart.py b/tests/pytest/functions/function_leastsquares_restart.py
new file mode 100644
index 0000000000..0221dc0aed
--- /dev/null
+++ b/tests/pytest/functions/function_leastsquares_restart.py
@@ -0,0 +1,68 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        # leastsquares verifacation 
+        tdSql.error("select leastsquares(ts, 1, 1) from test1")
+        tdSql.error("select leastsquares(col1, 1, 1) from test")
+        tdSql.error("select leastsquares(col2, 1, 1) from test")
+        tdSql.error("select leastsquares(col3, 1, 1) from test")
+        tdSql.error("select leastsquares(col4, 1, 1) from test")
+        tdSql.error("select leastsquares(col5, 1, 1) from test")
+        tdSql.error("select leastsquares(col6, 1, 1) from test")
+        tdSql.error("select leastsquares(col7, 1, 1) from test1")
+        tdSql.error("select leastsquares(col8, 1, 1) from test1")
+        tdSql.error("select leastsquares(col9, 1, 1) from test1")
+                
+        tdSql.query("select leastsquares(col1, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col2, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col3, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col4, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}')
+
+        tdSql.query("select leastsquares(col5, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}')
+
+        tdSql.query("select leastsquares(col6, 1, 1) from test1")
+        tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_max_restart.py b/tests/pytest/functions/function_max_restart.py
new file mode 100644
index 0000000000..8d18275617
--- /dev/null
+++ b/tests/pytest/functions/function_max_restart.py
@@ -0,0 +1,76 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db") 
+
+        intData = []        
+        floatData = []
+
+        #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #             col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
+        #tdSql.execute("create table test1 using test tags('beijing')")
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # max verifacation 
+        tdSql.error("select max(ts) from test")
+        tdSql.error("select max(ts) from test1")
+        tdSql.error("select max(col7) from test")
+        tdSql.error("select max(col7) from test1")
+        tdSql.error("select max(col8) from test")
+        tdSql.error("select max(col8) from test1")
+        tdSql.error("select max(col9) from test")
+        tdSql.error("select max(col9) from test1")
+        
+        tdSql.query("select max(col1) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col2) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col3) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+
+        tdSql.query("select max(col4) from test1")
+        tdSql.checkData(0, 0, np.max(intData))
+        
+        tdSql.query("select max(col5) from test1")
+        tdSql.checkData(0, 0, np.max(floatData))
+
+        tdSql.query("select max(col6) from test1")
+        tdSql.checkData(0, 0, np.max(floatData))
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_min_restart.py b/tests/pytest/functions/function_min_restart.py
new file mode 100644
index 0000000000..c8329bffa5
--- /dev/null
+++ b/tests/pytest/functions/function_min_restart.py
@@ -0,0 +1,73 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # min verifacation 
+        tdSql.error("select min(ts) from test")
+        tdSql.error("select min(ts) from test1")
+        tdSql.error("select min(col7) from test")
+        tdSql.error("select min(col7) from test1")
+        tdSql.error("select min(col8) from test")
+        tdSql.error("select min(col8) from test1")
+        tdSql.error("select min(col9) from test")
+        tdSql.error("select min(col9) from test1")
+        
+        tdSql.query("select min(col1) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col2) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col3) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+
+        tdSql.query("select min(col4) from test1")
+        tdSql.checkData(0, 0, np.min(intData))
+        
+        tdSql.query("select min(col5) from test1")
+        tdSql.checkData(0, 0, np.min(floatData))
+
+        tdSql.query("select min(col6) from test1")
+        tdSql.checkData(0, 0, np.min(floatData))
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_operations_restart.py b/tests/pytest/functions/function_operations_restart.py
new file mode 100644
index 0000000000..6e3990264c
--- /dev/null
+++ b/tests/pytest/functions/function_operations_restart.py
@@ -0,0 +1,70 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        # min verifacation 
+        tdSql.error("select ts + col1 from test")
+        tdSql.error("select ts + col1 from test1")
+        tdSql.error("select col1 + col7 from test")
+        tdSql.error("select col1 + col7 from test1")
+        tdSql.error("select col1 + col8 from test")
+        tdSql.error("select col1 + col8 from test1")
+        tdSql.error("select col1 + col9 from test")
+        tdSql.error("select col1 + col9 from test1")
+
+        tdSql.query("select col1 + col2 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0, 0, 2.0)        
+
+        tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0, 0, 3.2)
+
+        #tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11))
+        tdSql.query("select col1 + col2 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+        
+        tdSql.query("select col1 + col2 * col3 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+
+        tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1")
+        tdSql.checkRows(11)
+        tdSql.checkData(10, 0, None)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_percentile_restart.py b/tests/pytest/functions/function_percentile_restart.py
new file mode 100644
index 0000000000..ade10282de
--- /dev/null
+++ b/tests/pytest/functions/function_percentile_restart.py
@@ -0,0 +1,136 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # percentile verifacation 
+        tdSql.error("select percentile(ts 20) from test")
+        tdSql.error("select apercentile(ts 20) from test")
+        tdSql.error("select percentile(col7 20) from test")
+        tdSql.error("select apercentile(col7 20) from test")
+        tdSql.error("select percentile(col8 20) from test")        
+        tdSql.error("select apercentile(col8 20) from test")
+        tdSql.error("select percentile(col9 20) from test")
+        tdSql.error("select apercentile(col9 20) from test")        
+
+        tdSql.query("select percentile(col1, 0) from test")        
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col1, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col1, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col1, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col1, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100))
+        tdSql.query("select apercentile(col1, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))  
+
+        tdSql.query("select percentile(col2, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col2, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col2, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col2, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col2, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100)) 
+        tdSql.query("select apercentile(col2, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col3, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col3, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col3, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col3, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col3, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100))
+        tdSql.query("select apercentile(col3, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col4, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 0))
+        tdSql.query("select apercentile(col4, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col4, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 50))
+        tdSql.query("select apercentile(col4, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col4, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(intData, 100)) 
+        tdSql.query("select apercentile(col4, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+
+        tdSql.query("select percentile(col5, 0) from test")        
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 0))
+        tdSql.query("select apercentile(col5, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col5, 50) from test")
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 50))
+        tdSql.query("select apercentile(col5, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))        
+        tdSql.query("select percentile(col5, 100) from test")
+        print("query result: %s" % tdSql.getData(0, 0))
+        print("array result: %s" % np.percentile(floatData, 100)) 
+        tdSql.query("select apercentile(col5, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))     
+
+        tdSql.query("select percentile(col6, 0) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 0))
+        tdSql.query("select apercentile(col6, 0) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col6, 50) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 50))
+        tdSql.query("select apercentile(col6, 50) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))
+        tdSql.query("select percentile(col6, 100) from test")
+        tdSql.checkData(0, 0, np.percentile(floatData, 100)) 
+        tdSql.query("select apercentile(col6, 100) from test")
+        print("apercentile result: %s" % tdSql.getData(0, 0))       
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_spread_restart.py b/tests/pytest/functions/function_spread_restart.py
new file mode 100644
index 0000000000..ba71da0a18
--- /dev/null
+++ b/tests/pytest/functions/function_spread_restart.py
@@ -0,0 +1,69 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")        
+       
+        tdSql.error("select spread(col7) from test")     
+        tdSql.error("select spread(col7) from test1")               
+        tdSql.error("select spread(col8) from test") 
+        tdSql.error("select spread(col8) from test1")
+        tdSql.error("select spread(col9) from test")        
+        tdSql.error("select spread(col9) from test1")
+        
+        tdSql.query("select spread(col1) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col2) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col3) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col4) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select spread(col5) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+
+        tdSql.query("select spread(col6) from test1")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 9.1)
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_stddev_restart.py b/tests/pytest/functions/function_stddev_restart.py
new file mode 100644
index 0000000000..ec413b94b2
--- /dev/null
+++ b/tests/pytest/functions/function_stddev_restart.py
@@ -0,0 +1,75 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # stddev verifacation 
+        tdSql.error("select stddev(ts) from test1")
+        tdSql.error("select stddev(col1) from test")
+        tdSql.error("select stddev(col2) from test")
+        tdSql.error("select stddev(col3) from test")
+        tdSql.error("select stddev(col4) from test")
+        tdSql.error("select stddev(col5) from test")
+        tdSql.error("select stddev(col6) from test")
+        tdSql.error("select stddev(col7) from test1")
+        tdSql.error("select stddev(col8) from test1")
+        tdSql.error("select stddev(col9) from test1")
+
+        tdSql.query("select stddev(col1) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col2) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col3) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col4) from test1")
+        tdSql.checkData(0, 0, np.std(intData))
+
+        tdSql.query("select stddev(col5) from test1")
+        tdSql.checkData(0, 0, np.std(floatData))
+
+        tdSql.query("select stddev(col6) from test1")
+        tdSql.checkData(0, 0, np.std(floatData))
+            
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_sum_restart.py b/tests/pytest/functions/function_sum_restart.py
new file mode 100644
index 0000000000..7fd9ae7dd4
--- /dev/null
+++ b/tests/pytest/functions/function_sum_restart.py
@@ -0,0 +1,64 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # sum verifacation 
+        tdSql.error("select sum(ts) from test")
+        tdSql.error("select sum(col7) from test")
+        tdSql.error("select sum(col8) from test")
+        tdSql.error("select sum(col9) from test")
+
+        tdSql.query("select sum(col1) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col2) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col3) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col4) from test")
+        tdSql.checkData(0, 0, np.sum(intData))
+        tdSql.query("select sum(col5) from test")
+        tdSql.checkData(0, 0, np.sum(floatData)) 
+        tdSql.query("select sum(col6) from test")
+        tdSql.checkData(0, 0, np.sum(floatData))              
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_top_restart.py b/tests/pytest/functions/function_top_restart.py
new file mode 100644
index 0000000000..5ae96a2505
--- /dev/null
+++ b/tests/pytest/functions/function_top_restart.py
@@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # top verifacation 
+        tdSql.error("select top(ts, 10) from test")
+        tdSql.error("select top(col1, 0) from test")
+        tdSql.error("select top(col1, 101) from test")
+        tdSql.error("select top(col2, 0) from test")
+        tdSql.error("select top(col2, 101) from test")
+        tdSql.error("select top(col3, 0) from test")
+        tdSql.error("select top(col3, 101) from test")
+        tdSql.error("select top(col4, 0) from test")
+        tdSql.error("select top(col4, 101) from test")
+        tdSql.error("select top(col5, 0) from test")
+        tdSql.error("select top(col5, 101) from test")
+        tdSql.error("select top(col6, 0) from test")
+        tdSql.error("select top(col6, 101) from test")        
+        tdSql.error("select top(col7, 10) from test")        
+        tdSql.error("select top(col8, 10) from test")
+        tdSql.error("select top(col9, 10) from test")
+
+        tdSql.query("select top(col1, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col2, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col3, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col4, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 9)
+        tdSql.checkData(1, 1, 10)
+
+        tdSql.query("select top(col5, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 8.1)
+        tdSql.checkData(1, 1, 9.1)
+
+        tdSql.query("select top(col6, 2) from test")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 1, 8.1)
+        tdSql.checkData(1, 1, 9.1)
+                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/functions/function_twa_restart.py b/tests/pytest/functions/function_twa_restart.py
new file mode 100644
index 0000000000..2025e3c9dd
--- /dev/null
+++ b/tests/pytest/functions/function_twa_restart.py
@@ -0,0 +1,130 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.execute("use db")
+
+        intData = []        
+        floatData = []
+
+        for i in range(self.rowNum):
+            intData.append(i + 1)            
+            floatData.append(i + 0.1)                        
+
+        # twa verifacation 
+        tdSql.error("select twa(ts) from test")
+        tdSql.error("select twa(ts) from test1")
+
+        tdSql.error("select twa(col1) from test")
+        tdSql.error("select twa(col1) from test1")
+
+        tdSql.error("select twa(col2) from test")
+        tdSql.error("select twa(col2) from test1")
+
+        tdSql.error("select twa(col3) from test")
+        tdSql.error("select twa(col3) from test1")
+
+        tdSql.error("select twa(col4) from test")
+        tdSql.error("select twa(col4) from test1")
+
+        tdSql.error("select twa(col5) from test")
+        tdSql.error("select twa(col5) from test1")
+
+        tdSql.error("select twa(col6) from test")
+        tdSql.error("select twa(col6) from test1")        
+
+        tdSql.error("select twa(col7) from test")
+        tdSql.error("select twa(col7) from test1")
+
+        tdSql.error("select twa(col8) from test")
+        tdSql.error("select twa(col8) from test1")
+
+        tdSql.error("select twa(col9) from test")
+        tdSql.error("select twa(col9) from test1")
+
+        tdSql.error("select twa(col1) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col2) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col3) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col4) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts)
+            
+        tdSql.error("select twa(col5) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col6) from test where ts > %d" % self.ts)
+        tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts)
+
+        tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum))
+
+        tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum))
+        tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum))
+        
+        tdSql.query("select twa(col1) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col1) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col2) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col2) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col3) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+        tdSql.query("select twa(col3) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col4) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.query("select twa(col4) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col5) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum)) 
+        tdSql.query("select twa(col5) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+        tdSql.query("select twa(col6) from test where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))         
+        tdSql.query("select twa(col6) from test1 where ts > %d  and ts < %d" % (self.ts, self.ts + self.rowNum))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/test.py b/tests/pytest/test.py
index 678bd87336..a2c35444b5 100644
--- a/tests/pytest/test.py
+++ b/tests/pytest/test.py
@@ -16,6 +16,7 @@
 import sys
 import getopt
 import subprocess
+import time
 from distutils.log import warn as printf
 
 from util.log import *
@@ -33,7 +34,8 @@ if __name__ == "__main__":
     valgrind = 0
     logSql = True
     stop = 0
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scgh', [
+    restart = False
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [
         'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help'])
     for key, value in opts:
         if key in ['-h', '--help']:
@@ -46,8 +48,12 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-s stop All dnodes')
             tdLog.printNoPrefix('-c Test Cluster Flag')
             tdLog.printNoPrefix('-g valgrind Test Flag')
+            tdLog.printNoPrefix('-r taosd restart test')
             sys.exit(0)
 
+        if key in ['-r', '--restart']: 
+            restart = True
+
         if key in ['-f', '--file']:
             fileName = value
 
@@ -138,5 +144,19 @@ if __name__ == "__main__":
             tdCases.runAllLinux(conn)
         else:
             tdCases.runOneLinux(conn, fileName)
-
+    if restart:
+        if fileName == "all":
+            tdLog.info("not need to query ")
+        else:    
+            sp = fileName.rsplit(".", 1)
+            if len(sp) == 2 and sp[1] == "py":
+                tdDnodes.stopAll()
+                tdDnodes.start(1)
+                time.sleep(1)            
+                conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                tdLog.info("query test after taosd restart")
+                tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+            else:
+                tdLog.info("not need to query")
     conn.close()

From 0bb3794b2dd7c63659098295b56b6c391053f7ae Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 16 Oct 2020 14:29:25 +0800
Subject: [PATCH 25/30] TD-1725

---
 src/balance/src/balance.c   |  5 +++--
 src/mnode/src/mnodeDnode.c  |  3 ++-
 src/mnode/src/mnodeTable.c  | 12 ++++++-----
 src/mnode/src/mnodeVgroup.c | 40 +++++++++++++++++++------------------
 4 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
index 3ae65674c8..0fa4b3f346 100644
--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@@ -126,7 +126,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
 
   balanceAccquireDnodeList();
 
-  mDebug("db:%s, alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
+  mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
          mnodeGetDnodesNum(), tsBalanceDnodeListSize);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     for (; dnode < tsBalanceDnodeListSize; ++dnode) {
@@ -571,7 +571,8 @@ static void balanceCheckDnodeAccess() {
       if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
         pDnode->status = TAOS_DN_STATUS_OFFLINE;
         pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
-        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
+              pDnode->lastAccess);
         balanceSetVgroupOffline(pDnode);
       }
     }
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 33a869a353..1cd861e223 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -471,7 +471,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
              mnodeGetClusterId());
       return TSDB_CODE_MND_INVALID_CLUSTER_ID;
     } else {
-      mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
+      mTrace("dnode:%d, status received, access times %d openVnodes:%d:%d", pDnode->dnodeId, pDnode->lastAccess,
+             htons(pStatus->openVnodes), pDnode->openVnodes);
     }
   }
 
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index ca9d6cff96..1a1ff878a5 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -2409,14 +2409,16 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
     }
   } else {
     if (mnodeMsg->retry++ < 10) {
-      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
-             mnodeMsg->rpcMsg.handle);
+      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
+             " result:%s thandle:%p",
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->sid,
+             pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
 
       dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
     } else {
-      mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-             pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid,
+             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
 
       SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
       sdbDeleteRow(&oper);
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 69b9224d4f..28e4d17920 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -270,31 +270,34 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
   Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode 
 */
 void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
-  SVnodeLoad *pNextV = NULL;
-
   void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup;
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
-    pNextV = pVloads;
-    int32_t i;
-    for (i = 0; i < openVnodes; ++i) {
-      if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
-        break;
-      }
-      pNextV++;
-    }
+    for (int v = 0; v < pVgroup->numOfVnodes; ++v) {
+      if (pVgroup->vnodeGid[v].dnodeId == pDnode->dnodeId) {
+        // vgroup should have a vnode on this dnode
+        bool have = false;
+        for (int32_t i = 0; i < openVnodes; ++i) {
+          SVnodeLoad *pVload = pVloads + i;
+          if (pVgroup->vgId == pVload->vgId) {
+            have = true;
+            break;
+          }
+        }
 
-    if (i == openVnodes) {
-      if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
-        mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
-               vgroupStatus[pVgroup->status]);
-      } else {
-        mDebug("vgId:%d, not exist in dnode:%d and status is %s, send create msg", pVgroup->vgId, pDnode->dnodeId,
-               vgroupStatus[pVgroup->status]);
-        mnodeSendCreateVgroupMsg(pVgroup, NULL);
+        if (have) continue;
+
+        if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
+          mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
+                 vgroupStatus[pVgroup->status]);
+        } else {
+          mDebug("vgId:%d, not exist in dnode:%d and status is %s, send create msg", pVgroup->vgId, pDnode->dnodeId,
+                 vgroupStatus[pVgroup->status]);
+          mnodeSendCreateVgroupMsg(pVgroup, NULL);
+        }
       }
     }
 
@@ -302,7 +305,6 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
   }
 
   sdbFreeIter(pIter);
-  return;
 }
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {

From e47d694beffda7cf376039e1d5a83a0e90b86218 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Fri, 16 Oct 2020 16:33:45 +0800
Subject: [PATCH 26/30] <test> fix jenkins error

---
 Jenkinsfile | 24 +-----------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c262aa50f7..4410d81be6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -5,29 +5,7 @@ pipeline {
       WKC= '/var/lib/jenkins/workspace/TDinternal/community'
   }
   stages {
-    stage('build TDengine') {
-      agent{label 'master'}
-      steps {
-        sh '''
-        cd ${WKC}
-        git checkout develop
-        git pull
-        git submodule update
-        cd ${WK}
-        git checkout develop
-        git pull
-        export TZ=Asia/Harbin
-        date
-        rm -rf ${WK}/debug
-        mkdir debug
-        cd debug
-        #cmake .. > /dev/null
-        #make > /dev/null
-        '''
-      }
-    }
-
-    stage('Parallel test stage') {
+      stage('Parallel test stage') {
       parallel {
         stage('pytest') {
           agent{label 'master'}

From 2490995d6ca79b359f2e39808bbfc8dbab0628d5 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 16 Oct 2020 09:52:49 +0000
Subject: [PATCH 27/30] TD-1530 fix invalid read/write while execute
 tags_filter.sim

---
 src/inc/taosdef.h       | 4 ++--
 src/tsdb/src/tsdbRead.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 01a4ed32f1..aee60da201 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -194,9 +194,9 @@ static FORCE_INLINE bool isNull(const char *val, int32_t type) {
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
     case TSDB_DATA_TYPE_NCHAR:
-      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
+      return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:
-      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
+      return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
     default:
       return false;
   };
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index f0a2694b60..8ca71e4555 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -2284,7 +2284,7 @@ void filterPrepare(void* expr, void* param) {
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
   } else {
-    pInfo->q = calloc(1, pSchema->bytes);
+    pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE);   // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
     tVariantDump(pCond, pInfo->q, pSchema->type, true);
   }
 }

From 1ad7827fc19132b276d02e5bdc16ce990420d9a8 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 16 Oct 2020 18:47:02 +0800
Subject: [PATCH 28/30] TD-1530

---
 .../general/insert/query_block1_memory.sim    | 25 +++++++++++--------
 tests/script/general/parser/where.sim         | 10 +++++---
 tests/script/general/stream/table_n.sim       |  2 ++
 tests/script/unique/cluster/vgroup100.sim     |  9 ++++---
 4 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/tests/script/general/insert/query_block1_memory.sim b/tests/script/general/insert/query_block1_memory.sim
index 83509ad5b0..b4b3ca384b 100644
--- a/tests/script/general/insert/query_block1_memory.sim
+++ b/tests/script/general/insert/query_block1_memory.sim
@@ -30,9 +30,12 @@ print =============== step 1
 $x = $N
 $y = $N / 2
 while $x > $y
+  $y = $x * 60000
+  $ms = 1601481600000 - $y
+
   $ms = $x . m 
   $xt = - . $x 
-  sql insert into $tb values (now - $ms , -$x ) 
+  sql insert into $tb values ($ms , -$x ) 
   $x = $x - 1
 endw 
 
@@ -45,8 +48,10 @@ endi
 $x = $N / 2
 $y = $N
 while $x < $y
-  $ms = $x . m 
-  sql insert into $tb values (now + $ms , $x ) 
+  $y = $x * 60000
+  $ms = 1601481600000 + $y
+
+  sql insert into $tb values ($ms , $x ) 
   $x = $x + 1
 endw 
 sql select * from $tb
@@ -60,14 +65,14 @@ print =============== step 2
 $N1 = $N + 1
 $result1 = $N / 2
 $result2 = $N
-$step = $N1 . m
+$step = $N1 * 60000
 
-$start1 = now- . $step
-$start2 = now
-$start3 = now+ . $step
-$end1 = now- . $step
-$end2 = now
-$end3 = now+ . $step
+$start1 = 1601481600000 -  $step
+$start2 = 1601481600000
+$start3 = 1601481600000 + $step
+$end1 = 1601481600000 -  $step
+$end2 = 1601481600000
+$end3 = 1601481600000 +  $step
 
 sql select * from $tb where ts < $start1 and ts > $end1
 if $rows != 0 then 
diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim
index e609dda652..5cac3f4723 100644
--- a/tests/script/general/parser/where.sim
+++ b/tests/script/general/parser/where.sim
@@ -33,7 +33,8 @@ while $i < $tbNum
 
   $x = 0
   while $x < $rowNum
-    $ms = $x . m
+    $y = $x * 60000
+    $ms = 1600099200000 + $y
     $c = $x / 100
     $c = $c * 100
     $c = $x - $c
@@ -41,7 +42,7 @@ while $i < $tbNum
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values (now + $ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+    sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
     $x = $x + 1
   endw
 
@@ -299,7 +300,8 @@ while $i < 1
 
   $x = 0
   while $x < 10000
-    $ms = $x . m
+    $y = $x * 60000
+    $ms = 1601481600000 + $y
     $c = $x / 100
     $c = $c * 100
     $c = $x - $c
@@ -307,7 +309,7 @@ while $i < 1
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values (now + $ms , null , null , null , null , null , null , null , null , null )
+    sql insert into $tb values ($ms , null , null , null , null , null , null , null , null , null )
     $x = $x + 1
   endw
 
diff --git a/tests/script/general/stream/table_n.sim b/tests/script/general/stream/table_n.sim
index d1b4a87a9e..a336772a98 100644
--- a/tests/script/general/stream/table_n.sim
+++ b/tests/script/general/stream/table_n.sim
@@ -289,3 +289,5 @@ endi
 if $data09 != 20 then 
   return -1
 endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim
index 7879c5529e..68a5bad6b3 100644
--- a/tests/script/unique/cluster/vgroup100.sim
+++ b/tests/script/unique/cluster/vgroup100.sim
@@ -29,6 +29,9 @@ system sh/exec.sh -n dnode3 -s start
 
 sleep 3000
 
+$maxNum = 102
+$maxNum = 12
+
 $x = 0
 show2: 
 	$x = $x + 1
@@ -58,7 +61,7 @@ endi
 
 print ============================== step3
 $count = 2
-while $count < 102
+while $count < $maxNum
   $db = d . $count
   $tb = $db . .t
   $tb2 = $db . .t2
@@ -73,7 +76,7 @@ endw
 print ============================== step4
 
 $count = 2
-while $count < 102
+while $count < $maxNum
   $db = d . $count
   $tb = $db . .t
   sql select * from $tb
@@ -131,7 +134,7 @@ show8:
 	endi
 
 $count = 2
-while $count < 102
+while $count < $maxNum
   $db = d . $count
   $tb = $db . .t
   sql select * from $tb

From 51c09bb1edcb98c08acdb70cf6bbbec8e69619b8 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 16 Oct 2020 18:56:15 +0800
Subject: [PATCH 29/30] TD-1530 aligin double in arm32

---
 src/client/src/tscFunctionImpl.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 60e9596ec4..1b4f92d3fc 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2445,8 +2445,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
   // in the first round, get the min-max value of all involved data
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = pResInfo->interResultBuf;
-  pInfo->minval = DBL_MAX;
-  pInfo->maxval = -DBL_MAX;
+  SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
+  SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
   pInfo->numOfElems = 0;
 
   return true;
@@ -2461,12 +2461,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
   // the first stage, only acquire the min/max value
   if (pInfo->stage == 0) {
     if (pCtx->preAggVals.isSet) {
-      if (pInfo->minval > pCtx->preAggVals.statis.min) {
-        pInfo->minval = (double)pCtx->preAggVals.statis.min;
+      if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
+        SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
       }
 
-      if (pInfo->maxval < pCtx->preAggVals.statis.max) {
-        pInfo->maxval = (double)pCtx->preAggVals.statis.max;
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
+        SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
       }
 
       pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
@@ -2500,12 +2500,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
             break;
         }
 
-        if (v < pInfo->minval) {
-          pInfo->minval = v;
+        if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+          SET_DOUBLE_VAL(&pInfo->minval, v);
         }
 
-        if (v > pInfo->maxval) {
-          pInfo->maxval = v;
+        if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+          SET_DOUBLE_VAL(&pInfo->maxval, v);
         }
 
         pInfo->numOfElems += 1;
@@ -2564,12 +2564,12 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
         break;
     }
 
-    if (v < pInfo->minval) {
-      pInfo->minval = v;
+    if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+      SET_DOUBLE_VAL(&pInfo->minval, v);
     }
 
-    if (v > pInfo->maxval) {
-      pInfo->maxval = v;
+    if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+      SET_DOUBLE_VAL(&pInfo->maxval, v);
     }
 
     pInfo->numOfElems += 1;
@@ -2609,7 +2609,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
     }
 
     pInfo->stage += 1;
-    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
   } else {
     pResInfo->complete = true;
   }

From 9c9593d58c277c463f62cfbeebda1295207dde90 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 16 Oct 2020 13:59:00 +0000
Subject: [PATCH 30/30] scripts

---
 tests/script/general/insert/query_block1_memory.sim | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/script/general/insert/query_block1_memory.sim b/tests/script/general/insert/query_block1_memory.sim
index b4b3ca384b..bec9190f9b 100644
--- a/tests/script/general/insert/query_block1_memory.sim
+++ b/tests/script/general/insert/query_block1_memory.sim
@@ -30,10 +30,9 @@ print =============== step 1
 $x = $N
 $y = $N / 2
 while $x > $y
-  $y = $x * 60000
-  $ms = 1601481600000 - $y
+  $z = $x * 60000
+  $ms = 1601481600000 - $z
 
-  $ms = $x . m 
   $xt = - . $x 
   sql insert into $tb values ($ms , -$x ) 
   $x = $x - 1
@@ -48,8 +47,8 @@ endi
 $x = $N / 2
 $y = $N
 while $x < $y
-  $y = $x * 60000
-  $ms = 1601481600000 + $y
+  $z = $x * 60000
+  $ms = 1601481600000 + $z
 
   sql insert into $tb values ($ms , $x ) 
   $x = $x + 1