Merge branch 'develop' into hotfix/TD-2257

Jenkinsfile

@@ -14,10 +14,12 @@ pipeline {
             sh '''
             date
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -39,11 +41,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -65,11 +69,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -108,11 +114,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -167,7 +175,47 @@ pipeline {
           
          }
        }
-
+       stage('arm64_build'){
+                agent{label 'arm64'}
+                steps{
+                    sh '''
+                    cd ${WK}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    cd ${WKC}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    git submodule update                    
+                    cd ${WKC}/packaging
+                    ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
+                    
+                    '''
+                }
+            }
+            stage('arm32_build'){
+                agent{label 'arm32'}
+                steps{
+                    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                        sh '''
+                        cd ${WK}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        cd ${WKC}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        git submodule update
+                        cd ${WKC}/packaging
+                        ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
+                        
+                        '''
+                    }
+                    
+                }
+            }
       }
     }
 

documentation20/webdocs/markdowndocs/administrator-ch.md

@@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 - logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。
 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
+- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
 
 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。
 

documentation20/webdocs/markdowndocs/connector-ch.md

@@ -142,7 +142,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
   获取最近一次API调用失败的原因，返回值为错误代码。
 
 
-**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该连接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。
+**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
 
 
 ### 异步查询API

packaging/cfg/taos.cfg

@@ -236,7 +236,7 @@
 # httpDebugFlag         131
 
 # debug flag for monitor
-# monitorDebugFlag      131
+# monDebugFlag          131
 
 # debug flag for query
 # qDebugflag            131
@@ -260,4 +260,7 @@
 # maxBinaryDisplayWidth 30
 
 # enable/disable telemetry reporting
-# telemetryReporting    1
+# telemetryReporting    1
+
+# enable/disable stream (continuous query)
+# stream                1

packaging/tools/install.sh

@@ -172,6 +172,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos     || :
     ${csudo} rm -f ${bin_link_dir}/taosd    || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
@@ -182,6 +183,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                      || :
     [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd                   || :
     [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo          || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump          || :
     [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos          || :
     [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core    || :
     [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :

packaging/tools/install_client.sh

@@ -84,8 +84,9 @@ function install_main_path() {
 function install_bin() {
   # Remove links
   ${csudo} rm -f ${bin_link_dir}/taos         || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+      ${csudo} rm -f ${bin_link_dir}/taosdump || :
   fi
   ${csudo} rm -f ${bin_link_dir}/rmtaos       || :
   ${csudo} rm -f ${bin_link_dir}/set_core     || :
@@ -94,8 +95,9 @@ function install_bin() {
 
   #Make link
   [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                 || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
+      [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
   fi
   [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

packaging/tools/install_client_power.sh

@@ -84,8 +84,9 @@ function install_main_path() {
 function install_bin() {
   # Remove links
   ${csudo} rm -f ${bin_link_dir}/power         || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       ${csudo} rm -f ${bin_link_dir}/powerdemo || :
+      ${csudo} rm -f ${bin_link_dir}/powerdump || :
   fi
   ${csudo} rm -f ${bin_link_dir}/rmpower       || :
   ${csudo} rm -f ${bin_link_dir}/set_core      || :
@@ -94,8 +95,9 @@ function install_bin() {
 
   #Make link
   [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power                 || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
+      [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
   fi
   [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

packaging/tools/install_power.sh

@@ -172,6 +172,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/power     || :
     ${csudo} rm -f ${bin_link_dir}/powerd    || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump || :
     ${csudo} rm -f ${bin_link_dir}/rmpower   || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
@@ -182,6 +183,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power                        || :
     [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd                     || :
     [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo            || :
+    [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump            || :
     [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower  || :
     [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core         || :
     [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator      || :

packaging/tools/post.sh

@@ -92,6 +92,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos     || :
     ${csudo} rm -f ${bin_link_dir}/taosd    || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
     ${csudo} rm -f ${bin_link_dir}/set_core || :
 
@@ -101,6 +102,7 @@ function install_bin() {
     [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos             || :
     [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd          || :
     [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || :
+    [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || :
     [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || :
 }
 

packaging/tools/remove.sh

@@ -72,6 +72,7 @@ function clean_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos        || :
     ${csudo} rm -f ${bin_link_dir}/taosd       || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo    || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump    || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos      || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator || :
     ${csudo} rm -f ${bin_link_dir}/set_core    || :
@@ -222,4 +223,4 @@ elif  echo $osinfo | grep -qwi "centos" ; then
 fi
 
 echo -e "${GREEN}TDengine is removed successfully!${NC}"
-echo 
+echo 

packaging/tools/remove_client.sh

@@ -38,6 +38,7 @@ function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/taos      || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo  || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump  || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos    || :
     ${csudo} rm -f ${bin_link_dir}/set_core  || :
 }

packaging/tools/remove_client_power.sh

@@ -38,6 +38,7 @@ function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/power      || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo  || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump  || :
     ${csudo} rm -f ${bin_link_dir}/rmpower    || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
 }

packaging/tools/remove_power.sh

@@ -72,6 +72,7 @@ function clean_bin() {
     ${csudo} rm -f ${bin_link_dir}/power        || :
     ${csudo} rm -f ${bin_link_dir}/powerd       || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo    || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump    || :
     ${csudo} rm -f ${bin_link_dir}/rmpower      || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator  || :
     ${csudo} rm -f ${bin_link_dir}/set_core     || :
@@ -223,4 +224,4 @@ fi
 #fi
 
 echo -e "${GREEN}PowerDB is removed successfully!${NC}"
-echo 
+echo 

src/balance/inc/bnInt.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_INT_H
+#define TDENGINE_BALANCE_INT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeInt.h"
+#include "mnodeDef.h"
+#include "mnodeDnode.h"
+
+typedef struct {
+  int32_t     size;
+  int32_t     maxSize;
+  SDnodeObj **list;
+} SBnDnodes;
+
+typedef struct {
+  void *          timer;
+  bool            stop;
+  pthread_mutex_t mutex;
+  pthread_cond_t  cond;
+  pthread_t       thread;
+} SBnThread;
+
+typedef struct {
+  pthread_mutex_t mutex;
+} SBnMgmt;
+
+int32_t bnInit();
+void    bnCleanUp();
+bool    bnStart();
+void    bnCheckStatus();
+void    bnCheckModules();
+
+extern SBnDnodes tsBnDnodes;
+extern void *tsMnodeTmr;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/inc/bnScore.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_SCORE_H
+#define TDENGINE_BALANCE_SCORE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "bnInt.h"
+
+void  bnInitDnodes();
+void  bnCleanupDnodes();
+void  bnAccquireDnodes();
+void  bnReleaseDnodes();
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/inc/bnThread.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_THREAD_H
+#define TDENGINE_BALANCE_THREAD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "bnInt.h"
+
+int32_t bnInitThread();
+void    bnCleanupThread();
+void    bnNotify();
+void    bnStartTimer(int64_t mseconds);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/src/bnMain.c

@@ -15,17 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tutil.h"
-#include "tbalance.h"
 #include "tsync.h"
-#include "ttimer.h"
 #include "tglobal.h"
-#include "tdataformat.h"
 #include "dnode.h"
-#include "mnode.h"
+#include "bnInt.h"
-#include "mnodeDef.h"
+#include "bnScore.h"
-#include "mnodeInt.h"
+#include "bnThread.h"
-#include "mnodeDnode.h"
 #include "mnodeDb.h"
 #include "mnodeMnode.h"
 #include "mnodeSdb.h"
@@ -33,36 +28,18 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-/*
+static SBnMgmt tsBnMgmt;;
- * once sdb work as mater, then tsAccessSquence reset to zero
+static void  bnMonitorDnodeModule();
- * increase tsAccessSquence every balance interval
- */
-extern void *       tsMnodeTmr;
-static void *       tsBalanceTimer = NULL;
-static int32_t      tsBalanceDnodeListSize = 0;
-static SDnodeObj ** tsBalanceDnodeList = NULL;
-static int32_t      tsBalanceDnodeListMallocSize = 16;
-static pthread_mutex_t tsBalanceMutex;
 
-static void  balanceStartTimer(int64_t mseconds);
+static void bnLock() {
-static void  balanceInitDnodeList();
+  pthread_mutex_lock(&tsBnMgmt.mutex);
-static void  balanceCleanupDnodeList();
-static void  balanceAccquireDnodeList();
-static void  balanceReleaseDnodeList();
-static void  balanceMonitorDnodeModule();
-static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-
-static void balanceLock() {
-  pthread_mutex_lock(&tsBalanceMutex);
 }
 
-static void balanceUnLock() {
+static void bnUnLock() {
-  pthread_mutex_unlock(&tsBalanceMutex);
+  pthread_mutex_unlock(&tsBnMgmt.mutex);
 }
 
-static bool balanceCheckFree(SDnodeObj *pDnode) {
+static bool bnCheckFree(SDnodeObj *pDnode) {
   if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
     return false;
@@ -86,7 +63,7 @@ static bool balanceCheckFree(SDnodeObj *pDnode) {
   return true;
 }
 
-static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
 
   SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
@@ -111,27 +88,26 @@ static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mnodeUpdateVgroup(pVgroup);
 }
 
-static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
   // SVnodeGid tmp = *pVnodeGid1;
   // *pVnodeGid1 = *pVnodeGid2;
   // *pVnodeGid2 = tmp;
 }
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t bnAllocVnodes(SVgObj *pVgroup) {
   static int32_t randIndex = 0;
   int32_t dnode = 0;
   int32_t vnodes = 0;
 
-  balanceLock();
+  bnLock();
-
+  bnAccquireDnodes();
-  balanceAccquireDnodeList();
 
   mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
-         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
+         mnodeGetDnodesNum(), tsBnDnodes.size);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
+    for (; dnode < tsBnDnodes.size; ++dnode) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
+      SDnodeObj *pDnode = tsBnDnodes.list[dnode];
-      if (balanceCheckFree(pDnode)) {
+      if (bnCheckFree(pDnode)) {
         SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
         pVnodeGid->dnodeId = pDnode->dnodeId;
         pVnodeGid->pDnode = pDnode;
@@ -148,8 +124,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   }
 
   if (vnodes != pVgroup->numOfVnodes) {
-    balanceReleaseDnodeList();
+    bnReleaseDnodes();
-    balanceUnLock();
+    bnUnLock();
 
     mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
 
@@ -162,7 +138,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
              pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
       mnodeDecDnodeRef(pDnode);
     }
-    sdbFreeIter(pIter);
 
     if (mnodeGetOnlineDnodesNum() == 0) {
       return TSDB_CODE_MND_NOT_READY;
@@ -180,33 +155,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes == 1) {
   } else if (pVgroup->numOfVnodes == 2) {
     if (randIndex++ % 2 == 0) {
-      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
     }
   } else {
     int32_t randVal = randIndex++ % 6;
     if (randVal == 1) {  // 1, 0, 2
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
     } else if (randVal == 2) {  // 1, 2, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else if (randVal == 3) {  // 2, 1, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
     } else if (randVal == 4) {  // 2, 0, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     }
     if (randVal == 5) {  // 0, 2, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else {
     }  // 0, 1, 2
   }
 
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-  balanceUnLock();
+  bnUnLock();
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
   if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
     return false;
   }
@@ -233,7 +208,7 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
  * desc: remove one vnode from vgroup
  * all vnodes in vgroup should in ready state, except the balancing one
  **/
-static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
+static int32_t bnRemoveVnode(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes <= 1) return -1;
 
   SVnodeGid *pRmVnode = NULL;
@@ -275,17 +250,17 @@ static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
     pSelVnode = pRmVnode;
   }
 
-  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+  if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
     mDebug("vgId:%d, is not ready", pVgroup->vgId);
     return -1;
   } else {
     mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
-    balanceDiscardVnode(pVgroup, pSelVnode);
+    bnDiscardVnode(pVgroup, pSelVnode);
     return TSDB_CODE_SUCCESS;
   }
 }
 
-static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pGid = &pVgroup->vnodeGid[i];
     if (pGid->dnodeId == 0) break;
@@ -300,13 +275,13 @@ static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
 /**
  * desc: add vnode to vgroup, find a new one if dest dnode is null
  **/
-static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
   if (pDestDnode == NULL) {
-    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+      SDnodeObj *pDnode = tsBnDnodes.list[i];
       if (pDnode == pSrcDnode) continue;
-      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
-      if (!balanceCheckFree(pDnode)) continue;
+      if (!bnCheckFree(pDnode)) continue;
       
       pDestDnode = pDnode;
       mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
@@ -334,25 +309,25 @@ static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceMonitorBalance() {
+static bool bnMonitorBalance() {
-  if (tsBalanceDnodeListSize < 2) return false;
+  if (tsBnDnodes.size < 2) return false;
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
+  for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[src];
+    SDnodeObj *pDnode = tsBnDnodes.list[src];
-    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
            pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
            pDnode->openVnodes);
   }
 
-  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
   if (scoresDiff < 0.01) {
-    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
     return false;
   }
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
+  for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) {
-    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
+    SDnodeObj *pSrcDnode = tsBnDnodes.list[src];
-    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+    float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1);
     if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
       continue;
     }
@@ -363,29 +338,27 @@ static bool balanceMonitorBalance() {
       pIter = mnodeGetNextVgroup(pIter, &pVgroup);
       if (pVgroup == NULL) break;
 
-      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+      if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
         for (int32_t dest = 0; dest < src; dest++) {
-          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
+          SDnodeObj *pDestDnode = tsBnDnodes.list[dest];
-          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+          if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
 
-          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          float destScore = bnTryCalcDnodeScore(pDestDnode, 1);
           if (srcScore + 0.0001 < destScore) continue;
-          if (!balanceCheckFree(pDestDnode)) continue;
+          if (!bnCheckFree(pDestDnode)) continue;
           
           mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
                  pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
                  srcScore, pDestDnode->score, destScore);
-          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
           mnodeDecVgroupRef(pVgroup);
-          sdbFreeIter(pIter);
+          mnodeCancelGetNextVgroup(pIter);
           return true;
         }
       }
 
       mnodeDecVgroupRef(pVgroup);
     }
-
-    sdbFreeIter(pIter);
   }
 
   return false;
@@ -395,7 +368,7 @@ static bool balanceMonitorBalance() {
 // 1. reset balanceAccessSquence to zero
 // 2. reset state of dnodes to offline
 // 3. reset lastAccess of dnodes to zero
-void balanceReset() {
+void bnReset() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   while (1) {
@@ -413,12 +386,10 @@ void balanceReset() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   tsAccessSquence = 0;
 }
 
-static int32_t balanceMonitorVgroups() {
+static int32_t bnMonitorVgroups() {
   void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
   bool    hasUpdatingVgroup = false;
@@ -434,25 +405,24 @@ static int32_t balanceMonitorVgroups() {
     if (vgReplica > dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceRemoveVnode(pVgroup);
+      code = bnRemoveVnode(pVgroup);
     } else if (vgReplica < dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceAddVnode(pVgroup, NULL, NULL);
+      code = bnAddVnode(pVgroup, NULL, NULL);
     }
 
     mnodeDecVgroupRef(pVgroup);
     if (code == TSDB_CODE_SUCCESS) {
+      mnodeCancelGetNextVgroup(pIter);
       break;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return hasUpdatingVgroup;
 }
 
-static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) {
   mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
 
   void *  pIter = NULL;
@@ -462,14 +432,15 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
-    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup);
     mnodeDecVgroupRef(pVgroup);
 
-    if (hasThisDnode) break;
+    if (hasThisDnode) {
+      mnodeCancelGetNextVgroup(pIter);
+      break;
+    }
   }
 
-  sdbFreeIter(pIter);
-
   if (!hasThisDnode) {
     mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
     mnodeDropDnode(pDnode, NULL);
@@ -479,7 +450,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
   return false;
 }
 
-static bool balanceMontiorDropping() {
+static bool bnMontiorDropping() {
   void *pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
@@ -499,50 +470,46 @@ static bool balanceMontiorDropping() {
       pDnode->status = TAOS_DN_STATUS_DROPPING;
       mnodeUpdateDnode(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return true;
     }
 
     if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
-      bool ret = balanceMonitorDnodeDropping(pDnode);
+      bool ret = bnMonitorDnodeDropping(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return ret;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return false;
 }
 
-static bool balanceStart() {
+bool bnStart() {
   if (!sdbIsMaster()) return false;
 
-  balanceLock();
+  bnLock();
+  bnAccquireDnodes();
 
-  balanceAccquireDnodeList();
+  bnMonitorDnodeModule();
 
-  balanceMonitorDnodeModule();
+  bool updateSoon = bnMontiorDropping();
-
-  bool updateSoon = balanceMontiorDropping();
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorVgroups();
+    updateSoon = bnMonitorVgroups();
   }
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorBalance();
+    updateSoon = bnMonitorBalance();
   }
  
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-
+  bnUnLock();
-  balanceUnLock();
 
   return updateSoon;
 }
 
-static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+static void bnSetVgroupOffline(SDnodeObj* pDnode) {
   void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup;
@@ -556,11 +523,9 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
     }
     mnodeDecVgroupRef(pVgroup);
   }
-
-  sdbFreeIter(pIter);
 }
 
-static void balanceCheckDnodeAccess() {
+void bnCheckStatus() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
@@ -573,85 +538,39 @@ static void balanceCheckDnodeAccess() {
         pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
         mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
               pDnode->lastAccess, pDnode->status);
-        balanceSetVgroupOffline(pDnode);
+        bnSetVgroupOffline(pDnode);
       }
     }
     mnodeDecDnodeRef(pDnode);
   }
-
-  sdbFreeIter(pIter);
 }
 
-static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
+void bnCheckModules() {
-  if (!sdbIsMaster()) return;
-
-  tsBalanceTimer = NULL;
-  tsAccessSquence ++;
-
-  balanceCheckDnodeAccess();  
-  bool updateSoon = false;
-
-  if (handle == NULL) {
-    if (tsAccessSquence % tsBalanceInterval == 0) {
-      mDebug("balance function is scheduled by timer");
-      updateSoon = balanceStart();
-    }
-  } else {
-    int64_t mseconds = (int64_t)handle;
-    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
-    updateSoon = balanceStart();
-  }
-
-  if (updateSoon) {
-    balanceStartTimer(1000);
-  } else {
-    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
-  }
-}
-
-static void balanceStartTimer(int64_t mseconds) {
-  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
-}
-
-void balanceSyncNotify() {
   if (sdbIsMaster()) {
-    balanceLock();
+    bnLock();
-    balanceAccquireDnodeList();
+    bnAccquireDnodes();
-    balanceMonitorDnodeModule();
+    bnMonitorDnodeModule();
-    balanceReleaseDnodeList();
+    bnReleaseDnodes();
-    balanceUnLock();
+    bnUnLock();
   }
 }
 
-void balanceAsyncNotify() {
+int32_t bnInit() {
-  balanceStartTimer(500); 
+  pthread_mutex_init(&tsBnMgmt.mutex, NULL);
-}
+  bnInitDnodes();
-
+  bnInitThread();
-int32_t balanceInit() {
+  bnReset();
-  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
-  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
-  
-  pthread_mutex_init(&tsBalanceMutex, NULL);
-  balanceInitDnodeList();
-  balanceStartTimer(2000);
-  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
-
-  balanceReset();
   
   return 0;
 }
 
-void balanceCleanUp() {
+void bnCleanUp() {
-  if (tsBalanceTimer != NULL) {
+  bnCleanupThread();
-    taosTmrStopA(&tsBalanceTimer);
+  bnCleanupDnodes();
-    pthread_mutex_destroy(&tsBalanceMutex);
+  pthread_mutex_destroy(&tsBnMgmt.mutex);
-    tsBalanceTimer = NULL;
-    mDebug("stop balance timer");
-  }
-  balanceCleanupDnodeList();
 }
 
-int32_t balanceDropDnode(SDnodeObj *pDnode) {
+int32_t bnDropDnode(SDnodeObj *pDnode) {
   int32_t    totalFreeVnodes = 0;
   void *     pIter = NULL;
   SDnodeObj *pTempDnode = NULL;
@@ -660,15 +579,13 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
     pIter = mnodeGetNextDnode(pIter, &pTempDnode);
     if (pTempDnode == NULL) break;
 
-    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+    if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) {
       totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
     }
 
     mnodeDecDnodeRef(pTempDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (pDnode->openVnodes > totalFreeVnodes) {
     mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;
@@ -677,296 +594,17 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
   pDnode->status = TAOS_DN_STATUS_DROPPING;
   mnodeUpdateDnode(pDnode);
   
-  balanceStartTimer(1100);
+  bnStartTimer(1100);
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
+static void bnMonitorDnodeModule() {
-  if (pDnode->cpuAvgUsage < 80)
-    return 0;
-  else if (pDnode->cpuAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
-  if (pDnode->memoryAvgUsage < 80)
-    return 0;
-  else if (pDnode->memoryAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
-  if (pDnode->diskAvgUsage < 80)
-    return 0;
-  else if (pDnode->diskAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
-  if (pDnode->bandwidthUsage < 30)
-    return 0;
-  else if (pDnode->bandwidthUsage < 80)
-    return 10;
-  else
-    return 50;
-}
-
-static float balanceCalcModuleScore(SDnodeObj *pDnode) {
-  if (pDnode->numOfCores <= 0) return 0;
-  if (pDnode->isMgmt) {
-     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
-  }
-  return 0;
-}
-
-static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
-  if (pDnode->numOfCores <= 0) return 0;
-  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
-}
-
-/**
- * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
- * 1. get the score config
- * 2. if the value is out of range, use border data
- * 3. otherwise use interpolation method
- **/
-void balanceCalcDnodeScore(SDnodeObj *pDnode) {
-  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
-                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
-}
-
-float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                        balanceCalcBandwidthScore(pDnode);
-  float moduleScore = balanceCalcModuleScore(pDnode);
-  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
-
-  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
-  return score;
-}
-
-static void balanceInitDnodeList() {
-  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
-}
-
-static void balanceCleanupDnodeList() {
-  if (tsBalanceDnodeList != NULL) {
-    free(tsBalanceDnodeList);
-    tsBalanceDnodeList = NULL;
-  }
-}
-
-static void balanceCheckDnodeListSize(int32_t dnodesNum) {
-  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
-    tsBalanceDnodeListMallocSize = dnodesNum * 2;
-    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
-  }
-}
-
-void balanceAccquireDnodeList() {
-  int32_t dnodesNum = mnodeGetDnodesNum();
-  balanceCheckDnodeListSize(dnodesNum);
-
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  int32_t    dnodeIndex = 0;
-
-  while (1) {  
-    if (dnodeIndex >= dnodesNum) break;
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-      mnodeDecDnodeRef(pDnode);
-      continue;
-    }
-
-    balanceCalcDnodeScore(pDnode);
-    
-    int32_t orderIndex = dnodeIndex;
-    for (; orderIndex > 0; --orderIndex) {
-      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
-        break;
-      }
-      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
-    }
-    tsBalanceDnodeList[orderIndex] = pDnode;
-    dnodeIndex++;
-  }
-
-  sdbFreeIter(pIter);
-
-  tsBalanceDnodeListSize = dnodeIndex;
-}
-
-void balanceReleaseDnodeList() {
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
-    if (pDnode != NULL) {
-      mnodeDecDnodeRef(pDnode);
-    }
-  }
-}
-
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mnodeGetUserFromConn(pConn);
-  if (pUser == NULL) return 0;
-
-  if (strcmp(pUser->pAcct->user, "root") != 0)  {
-    mnodeDecUserRef(pUser);
-    return TSDB_CODE_MND_NO_RIGHTS;
-  }
-
-  int32_t cols = 0;
-  SSchema *pSchema = pMeta->schema;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "system scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "custom scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "module scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "vnode scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "total scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "open vnodes");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "cpu cores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "balance state");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htons(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->numOfRows = mnodeGetDnodesNum();
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pIter = NULL;
-
-  mnodeDecUserRef(pUser);
-
-  return 0;
-}
-
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
-  int32_t    numOfRows = 0;
-  SDnodeObj *pDnode = NULL;
-  char *     pWrite;
-  int32_t    cols = 0;
-  
-  while (numOfRows < rows) {
-    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                      balanceCalcBandwidthScore(pDnode);
-    float moduleScore = balanceCalcModuleScore(pDnode);
-    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
-
-    cols = 0;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->dnodeId;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = systemScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = pDnode->customScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)moduleScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)vnodeScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->openVnodes;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->numOfCores;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
-    cols++;
-
-    numOfRows++;
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  pShow->numOfReads += numOfRows;
-  return numOfRows;
-}
-
-static void balanceMonitorDnodeModule() {
   int32_t numOfMnodes = mnodeGetMnodesNum();
   if (numOfMnodes >= tsNumOfMnodes) return;
 
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
     if (pDnode == NULL) break;
 
     if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@@ -990,7 +628,7 @@ static void balanceMonitorDnodeModule() {
   }
 }
 
-int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
   if (!sdbIsMaster()) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
@@ -1014,29 +652,29 @@ int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
   }
 
-  balanceLock();
+  bnLock();
-  balanceAccquireDnodeList();
+  bnAccquireDnodes();
 
   int32_t code = TSDB_CODE_SUCCESS;
-  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+  if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, pSrcDnode->dnodeId);
     code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
-  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+  } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, dnodeId);
     code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
-  } else if (!balanceCheckFree(pDestDnode)) {
+  } else if (!bnCheckFree(pDestDnode)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
            dnodeId);
     code = TSDB_CODE_MND_DNODE_NOT_FREE;
   } else {
-    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
     mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
   }
 
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-  balanceUnLock();
+  bnUnLock();
 
   mnodeDecVgroupRef(pVgroup);
   mnodeDecDnodeRef(pDestDnode);

src/balance/src/bnScore.c

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "bnScore.h"
+
+SBnDnodes tsBnDnodes;
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static int32_t bnCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcBandScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float bnCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+    return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+static void bnCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                  bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) +
+                  pDnode->customScore;
+}
+
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                        bnCalcBandScore(pDnode);
+  float moduleScore = bnCalcModuleScore(pDnode);
+  float vnodeScore = bnCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+void bnInitDnodes() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
+
+  memset(&tsBnDnodes, 0, sizeof(SBnDnodes));
+  tsBnDnodes.maxSize = 16;
+  tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *));
+}
+
+void bnCleanupDnodes() {
+  if (tsBnDnodes.list != NULL) {
+    free(tsBnDnodes.list);
+    tsBnDnodes.list = NULL;
+  }
+}
+
+static void bnCheckDnodesSize(int32_t dnodesNum) {
+  if (tsBnDnodes.maxSize <= dnodesNum) {
+    tsBnDnodes.maxSize = dnodesNum * 2;
+    tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *));
+  }
+}
+
+void bnAccquireDnodes() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  bnCheckDnodesSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {
+    if (dnodeIndex >= dnodesNum) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
+
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    bnCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) {
+        break;
+      }
+      tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1];
+    }
+    tsBnDnodes.list[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  tsBnDnodes.size = dnodeIndex;
+}
+
+void bnReleaseDnodes() {
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0) {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode);
+    float moduleScore = bnCalcModuleScore(pDnode);
+    float vnodeScore = bnCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}

src/balance/src/bnThread.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "mnodeSdb.h"
+#include "bnThread.h"
+
+static SBnThread tsBnThread;
+
+static void *bnThreadFunc(void *arg) {
+  while (1) {
+    pthread_mutex_lock(&tsBnThread.mutex);
+    if (tsBnThread.stop) {
+      pthread_mutex_unlock(&tsBnThread.mutex);
+      break;
+    }
+
+    pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
+    bool updateSoon = bnStart();
+    bnStartTimer(updateSoon ? 1000 : -1);
+    pthread_mutex_unlock(&(tsBnThread.mutex));
+  }
+
+  mDebug("balance thread is stopped");
+  return NULL;
+}
+
+int32_t bnInitThread() {
+  memset(&tsBnThread, 0, sizeof(SBnThread));
+  tsBnThread.stop = false;
+  pthread_mutex_init(&tsBnThread.mutex, NULL);
+  pthread_cond_init(&tsBnThread.cond, NULL);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    mError("failed to create balance thread since %s", strerror(errno));
+    return -1;
+  }
+
+  bnStartTimer(2000);
+  mDebug("balance thread is created");
+  return 0;
+}
+
+void bnCleanupThread() {
+  mDebug("balance thread will be cleanup");
+
+  if (tsBnThread.timer != NULL) {
+    taosTmrStopA(&tsBnThread.timer);
+    tsBnThread.timer = NULL;
+    mDebug("stop balance timer");
+  }
+
+  pthread_mutex_lock(&tsBnThread.mutex);
+  tsBnThread.stop = true;
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+  pthread_join(tsBnThread.thread, NULL);
+
+  pthread_cond_destroy(&tsBnThread.cond);
+  pthread_mutex_destroy(&tsBnThread.mutex);
+}
+
+static void bnPostSignal() {
+  if (tsBnThread.stop) return;
+
+  pthread_mutex_lock(&tsBnThread.mutex);
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+}
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+
+static void bnProcessTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+  if (tsBnThread.stop) return;
+
+  tsBnThread.timer = NULL;
+  tsAccessSquence++;
+
+  bnCheckStatus();
+  bnStartTimer(-1);
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      bnPostSignal();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    bnPostSignal();
+  }
+}
+
+void bnStartTimer(int64_t mseconds) {
+  if (tsBnThread.stop) return;
+
+  bool updateSoon = (mseconds != -1);
+  if (updateSoon) {
+    taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
+  } else {
+    taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
+  }
+}
+
+void bnNotify() {
+  bnStartTimer(500); 
+}

src/client/inc/tscLocalMerge.h

@@ -56,7 +56,6 @@ typedef struct SLocalReducer {
   tFilePage *            pTempBuffer;
   struct SQLFunctionCtx *pCtx;
   int32_t                rowSize;      // size of each intermediate result.
-  int32_t                finalRowSize; // final result row size
   int32_t                status;       // denote it is in reduce process, in reduce process, it
   bool                   hasPrevRow;   // cannot be released
   bool                   hasUnprocessedRow;

src/client/inc/tsclient.h

@@ -246,11 +246,14 @@ typedef struct SQueryInfo {
   int16_t          fillType;      // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
-  struct STSBuf *  tsBuf;
+  struct STSBuf   *tsBuf;
   int64_t *        fillVal;       // default value for fill
   char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;   // limit for current sub clause
+
   int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
+  int64_t          tableLimit;    // table limit in case of super table projection query + global order + limit
+
   int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
   int16_t          resColumnId;   // result column id
 } SQueryInfo;
@@ -333,7 +336,7 @@ typedef struct STscObj {
   char               superAuth : 1;
   uint32_t           connId;
   uint64_t           rid;      // ref ID returned by taosAddRef
-  struct SSqlObj *   pHb;
+  int64_t            hbrid;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
   SRpcCorEpSet       *tscCorMgmtEpSet;
@@ -374,7 +377,7 @@ typedef struct SSqlObj {
   struct SSqlObj **pSubs;
 
   struct SSqlObj  *prev, *next;
-  struct SSqlObj **self;
+  int64_t          self;
 } SSqlObj;
 
 typedef struct SSqlStream {
@@ -508,7 +511,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
 }
 
 extern SCacheObj*    tscMetaCache;
-extern SCacheObj*    tscObjCache;
+extern int           tscObjRef;
 extern void *    tscTmr;
 extern void *    tscQhandle;
 extern int       tscKeepConn[];

src/client/src/tscFunctionImpl.c

@@ -64,13 +64,13 @@
     }                                                                            \
   } while (0);
 
-#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
+#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                   \
-do {\
+  do {                                                          \
-for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) {                  \
+    for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \
-      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];                     \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];    \
-      aAggs[TSDB_FUNC_TAG].xFunction(__ctx);                                     \
+      aAggs[TSDB_FUNC_TAG].xFunction(__ctx);                    \
-    }     \
+    }                                                           \
-} while(0);
+  } while (0);
 
 void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {}
 void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {}
@@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   }
   
   SET_VAL(pCtx, 1, 1);
-  
+  *((int64_t *)pCtx->aOutputBuf) += pCtx->size;
-  *((int64_t *)pCtx->aOutputBuf) += 1;
   
   // do not need it actually
   SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
@@ -3624,94 +3623,158 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
     return false;
   }
   
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);  //->aOutputBuf + pCtx->outputBytes;
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  STwaInfo *   pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
-  
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-  pInfo->lastKey = INT64_MIN;
+  pInfo->lastKey  = INT64_MIN;
-  pInfo->type = pCtx->inputType;
+  pInfo->win      = TSWINDOW_INITIALIZER;
-  
   return true;
 }
 
-static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, int32_t i, STwaInfo *pInfo) {
+static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
-  switch (pCtx->inputType) {
+  int32_t notNullElems = 0;
-    case TSDB_DATA_TYPE_INT:
+  TSKEY *primaryKey = pCtx->ptsList;
-      pInfo->iLastValue = GET_INT32_VAL(data + pCtx->inputBytes * i);
+  
-      break;
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-    case TSDB_DATA_TYPE_TINYINT:
+
-      pInfo->iLastValue = GET_INT8_VAL(data + pCtx->inputBytes * i);
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-      break;
+
-    case TSDB_DATA_TYPE_SMALLINT:
+  int32_t i = index;
-      pInfo->iLastValue = GET_INT16_VAL(data + pCtx->inputBytes * i);
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
-      break;
+
-    case TSDB_DATA_TYPE_BIGINT:
+  if (pCtx->start.key != INT64_MIN) {
-      pInfo->iLastValue = GET_INT64_VAL(data + pCtx->inputBytes * i);
+    assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
-      break;
+               (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
-    case TSDB_DATA_TYPE_FLOAT:
+
-      pInfo->dLastValue = GET_FLOAT_VAL(data + pCtx->inputBytes * i);
+    assert(pInfo->lastKey == INT64_MIN);
-      break;
+
-    case TSDB_DATA_TYPE_DOUBLE:
+    pInfo->lastKey = primaryKey[tsIndex + i];
-      pInfo->dLastValue = GET_DOUBLE_VAL(data + pCtx->inputBytes * i);
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
-      break;
+
-    default:
+    pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
-      assert(0);
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += step;
+  } else if (pInfo->lastKey == INT64_MIN) {
+    pInfo->lastKey = primaryKey[tsIndex + i];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pInfo->lastKey;
+    notNullElems++;
+    i += step;
   }
+
+  // calculate the value of
+  switch(pCtx->inputType) {
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = (double) val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    default: assert(0);
+  }
+
+  // the last interpolated time window value
+  if (pCtx->end.key != INT64_MIN) {
+    pInfo->dOutput  += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
+    pInfo->lastValue = pCtx->end.val;
+    pInfo->lastKey   = pCtx->end.key;
+  }
+
+  pInfo->win.ekey  = pInfo->lastKey;
+  return notNullElems;
 }
 
 static void twa_function(SQLFunctionCtx *pCtx) {
   void * data = GET_INPUT_CHAR(pCtx);
-  TSKEY *primaryKey = pCtx->ptsList;
+
-  
-  int32_t notNullElems = 0;
-  
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   STwaInfo *   pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   
-  int32_t i = 0;
-  
   // skip null value
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1);
   while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
-    i++;
+    i += step;
   }
-  
+
-  if (i >= pCtx->size) {
+  int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size);
-    return;
-  }
-  
-  if (pInfo->lastKey == INT64_MIN) {
-    pInfo->lastKey = pCtx->nStartQueryTimestamp;
-    setTWALastVal(pCtx, data, i, pInfo);
-    
-    pInfo->hasResult = DATA_SET_FLAG;
-  }
-  
-  notNullElems++;
-  
-  if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
-    pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
-  } else {
-    pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
-  }
-  
-  pInfo->lastKey = primaryKey[i];
-  setTWALastVal(pCtx, data, i, pInfo);
-  
-  for (++i; i < pCtx->size; i++) {
-    if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
-      continue;
-    }
-    
-    notNullElems++;
-    if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
-      pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
-    } else {
-      pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
-    }
-    
-    pInfo->lastKey = primaryKey[i];
-    setTWALastVal(pCtx, data, i, pInfo);
-  }
-  
   SET_VAL(pCtx, notNullElems, 1);
   
   if (notNullElems > 0) {
@@ -3721,8 +3784,6 @@ static void twa_function(SQLFunctionCtx *pCtx) {
   if (pCtx->stableQuery) {
     memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo));
   }
-  
-  //  pCtx->numOfIteratedElems += notNullElems;
 }
 
 static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
@@ -3730,34 +3791,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
-  SET_VAL(pCtx, 1, 1);
+  int32_t notNullElems = 0;
-  
   TSKEY *primaryKey = pCtx->ptsList;
-  
+
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
   STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-  
+  int32_t i = pCtx->startOffset;
-  if (pInfo->lastKey == INT64_MIN) {
+  int32_t size = pCtx->size;
-    pInfo->lastKey = pCtx->nStartQueryTimestamp;
+
-    setTWALastVal(pCtx, pData, 0, pInfo);
+  if (pCtx->start.key != INT64_MIN) {
-    
+    assert(pInfo->lastKey == INT64_MIN);
+
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
+
     pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += 1;
+  } else if (pInfo->lastKey == INT64_MIN) {
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pInfo->lastKey;
+    notNullElems++;
+    i += 1;
   }
-  
+
-  if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+  // calculate the value of
-    pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey);
+  switch(pCtx->inputType) {
-  } else {
+    case TSDB_DATA_TYPE_TINYINT: {
-    pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey);
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = (double) val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    default: assert(0);
   }
-  
+
-  // record the last key/value
+  // the last interpolated time window value
-  pInfo->lastKey = primaryKey[index];
+  if (pCtx->end.key != INT64_MIN) {
-  setTWALastVal(pCtx, pData, 0, pInfo);
+    pInfo->dOutput  += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
-  
+    pInfo->lastValue = pCtx->end.val;
-  //  pCtx->numOfIteratedElems += 1;
+    pInfo->lastKey   = pCtx->end.key;
-  pResInfo->hasResult = DATA_SET_FLAG;
+  }
-  
+
+  pInfo->win.ekey  = pInfo->lastKey;
+
+  SET_VAL(pCtx, notNullElems, 1);
+
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+
   if (pCtx->stableQuery) {
     memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
   }
@@ -3778,16 +3941,10 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
     }
     
     numOfNotNull++;
-    if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
+    pBuf->dOutput += pInput->dOutput;
-      pBuf->iOutput += pInput->iOutput;
+
-    } else {
+    pBuf->win = pInput->win;
-      pBuf->dOutput += pInput->dOutput;
-    }
-    
-    pBuf->SKey = pInput->SKey;
-    pBuf->EKey = pInput->EKey;
     pBuf->lastKey = pInput->lastKey;
-    pBuf->iLastValue = pInput->iLastValue;
   }
   
   SET_VAL(pCtx, numOfNotNull, 1);
@@ -3814,21 +3971,17 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   
   STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
-  assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
+  assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
   
   if (pInfo->hasResult != DATA_SET_FLAG) {
     setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
     return;
   }
   
-  if (pInfo->SKey == pInfo->EKey) {
+  if (pInfo->win.ekey == pInfo->win.skey) {
-    *(double *)pCtx->aOutputBuf = 0;
+    *(double *)pCtx->aOutputBuf = pInfo->lastValue;
-  } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) {
-    pInfo->iOutput += pInfo->iLastValue * (pInfo->EKey - pInfo->lastKey);
-    *(double *)pCtx->aOutputBuf = pInfo->iOutput / (double)(pInfo->EKey - pInfo->SKey);
   } else {
-    pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey);
+    *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey);
-    *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey);
   }
   
   GET_RES_INFO(pCtx)->numOfRes = 1;

src/client/src/tscLocal.c

@@ -825,8 +825,11 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) {
 static int32_t tscProcessServStatus(SSqlObj *pSql) {
   STscObj* pObj = pSql->pTscObj;
 
-  if (pObj->pHb != NULL) {
+  SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-    if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+  if (pHb != NULL) {
+    int32_t code = pHb->res.code;
+    taosReleaseRef(tscObjRef, pObj->hbrid);
+    if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
       pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
       return pSql->res.code;
     }

src/client/src/tscLocalMerge.c

@@ -198,6 +198,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   if (numOfFlush == 0 || numOfBuffer == 0) {
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer);
+    pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty
     tscDebug("%p retrieved no data", pSql);
     return;
   }
@@ -330,22 +331,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16;
   pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage));
 
-  pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList);
   pReducer->resColModel = finalmodel;
   pReducer->resColModel->capacity = pReducer->nResultBufSize;
-
   pReducer->finalModel = pFFModel;
 
-  assert(pReducer->finalRowSize > 0);
+  if (finalmodel->rowSize > 0) {
-  if (pReducer->finalRowSize > 0) {
+    pReducer->resColModel->capacity /= finalmodel->rowSize;
-    pReducer->resColModel->capacity /= pReducer->finalRowSize;
   }
-  assert(pReducer->finalRowSize <= pReducer->rowSize);
 
+  assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pReducer->rowSize);
   pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
 
   if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
-      /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
+      pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
     tfree(pReducer->pTempBuffer);
     tfree(pReducer->discardData);
     tfree(pReducer->pResultBuf);
@@ -723,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   // final result depends on the fields number
   memset(pSchema, 0, sizeof(SSchema) * size);
+
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
 
-    SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    SSchema p1 = {0};
+    if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) {
+      p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    } else {
+      p1 = tGetTableNameColumnSchema();
+    }
 
     int32_t inter = 0;
     int16_t type = -1;
@@ -745,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
         functionId = TSDB_FUNC_LAST;
       }
 
-      getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      assert(ret == TSDB_CODE_SUCCESS);
     }
 
     pSchema[i].type = (uint8_t)type;
@@ -920,7 +925,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
     savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo);
   }
 
-  memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalRowSize));
+  memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize));
 
   pRes->numOfClauseTotal += pRes->numOfRows;
   pBeforeFillData->num = 0;
@@ -1256,7 +1261,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
   tColModelCompact(pModel, pResBuf, pModel->capacity);
 
   if (tscIsSecondStageQuery(pQueryInfo)) {
-    pLocalReducer->finalRowSize = doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize);
+    doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize);
   }
 
 #ifdef _DEBUG_VIEW
@@ -1627,7 +1632,8 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
 }
 
 int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
-  char* pbuf = calloc(1, pOutput->num * rowSize);
+  int32_t maxRowSize = MAX(rowSize, finalRowSize);
+  char* pbuf = calloc(1, pOutput->num * maxRowSize);
 
   size_t size = tscNumOfFields(pQueryInfo);
   SArithmeticSupport arithSup = {0};
@@ -1660,7 +1666,6 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
     offset += pSup->field.bytes;
   }
 
-  assert(finalRowSize <= rowSize);
   memcpy(pOutput->data, pbuf, pOutput->num * offset);
 
   tfree(pbuf);

src/client/src/tscProfile.c

@@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) {
     tscSlowQueryConnInitialized = true;
     tscSaveSlowQueryFp(sql, NULL);
   }
+  taos_free_result(result);
 }
 
 void tscAddIntoSqlList(SSqlObj *pSql) {
@@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) {
   } else {
     tscDebug("success to save slow query, code:%d", code);
   }
+  taos_free_result(result);
 }
 
 void tscSaveSlowQueryFp(void *handle, void *tmrId) {

src/client/src/tscSQLParser.c

@@ -4248,7 +4248,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     tExprTreeDestroy(&p, NULL);
     
     taosArrayDestroy(colList);
-    if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+    if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table");
     }
   }
@@ -4256,6 +4256,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
   pCondExpr->pTagCond = NULL;
   return ret;
 }
+
 int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) {
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
@@ -5102,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
   const int tokenDebugFlagEnd = 20;
   const SDNodeDynConfOption cfgOptions[] = {
       {"resetLog", 8},    {"resetQueryCache", 15},  {"balance", 7},     {"monitor", 7},
-      {"debugFlag", 9},   {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10},
+      {"debugFlag", 9},   {"monDebugFlag", 12},     {"vDebugFlag", 10}, {"mDebugFlag", 10},
       {"cDebugFlag", 10}, {"httpDebugFlag", 13},    {"qDebugflag", 10}, {"sdbDebugFlag", 12},
       {"uDebugFlag", 10}, {"tsdbDebugFlag", 13},    {"sDebugflag", 10}, {"rpcDebugFlag", 12},
       {"dDebugFlag", 10}, {"mqttDebugFlag", 13},    {"wDebugFlag", 10}, {"tmrDebugFlag", 12},
@@ -5306,15 +5307,18 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
 
     // keep original limitation value in globalLimit
     pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
-    pQueryInfo->prjOffset = pQueryInfo->limit.offset;
+    pQueryInfo->prjOffset   = pQueryInfo->limit.offset;
+    pQueryInfo->tableLimit  = -1;
 
     if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
       /*
-       * the limitation/offset value should be removed during retrieve data from virtual node,
+       * the offset value should be removed during retrieve data from virtual node, since the
-       * since the global order are done in client side, so the limitation should also
+       * global order are done in client side, so the offset is applied at the client side
-       * be done at the client side.
+       * However, note that the maximum allowed number of result for each table should be less
+       * than or equal to the value of limit.
        */
       if (pQueryInfo->limit.limit > 0) {
+        pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset;
         pQueryInfo->limit.limit = -1;
       }
 
@@ -6648,7 +6652,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
       
       return TSDB_CODE_SUCCESS;
     } else {
-      return TSDB_CODE_TSC_INVALID_SQL;
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
     }
     
   } else {

src/client/src/tscServer.c

@@ -165,10 +165,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
       if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
     }
   } else {
-    tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
+    tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code));
   }
 
-  if (pObj->pHb != NULL) {
+  if (pObj->hbrid != 0) {
     int32_t waitingDuring = tsShellActivityTimer * 500;
     tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
 
@@ -183,20 +183,12 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
   STscObj *pObj = taosAcquireRef(tscRefId, rid);
   if (pObj == NULL) return; 
 
-  SSqlObj* pHB = pObj->pHb;
+  SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid);
-
+  assert(pHB->self == pObj->hbrid);
-  void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
-  if (p == NULL) {
-    tscWarn("%p HB object has been released already", pHB);
-    taosReleaseRef(tscRefId, pObj->rid);
-    return;
-  }
-
-  assert(*pHB->self == pHB);
 
   pHB->retry = 0;
   int32_t code = tscProcessSql(pHB);
-  taosCacheRelease(tscObjCache, (void**) &p, false);
+  taosReleaseRef(tscObjRef, pObj->hbrid);
 
   if (code != TSDB_CODE_SUCCESS) {
     tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
@@ -226,7 +218,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .msgType = pSql->cmd.msgType,
       .pCont   = pMsg,
       .contLen = pSql->cmd.payloadLen,
-      .ahandle = pSql,
+      .ahandle = (void*)pSql->self,
       .handle  = NULL,
       .code    = 0
   };
@@ -237,26 +229,24 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle;
-  void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(TSDB_CACHE_PTR_TYPE));
+  SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle);
-  if (p == NULL) {
+  if (pSql == NULL) {
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
-
+  assert(pSql->self == handle);
-  SSqlObj* pSql = *p;
-  assert(pSql != NULL);
 
   STscObj *pObj = pSql->pTscObj;
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
 
-  assert(*pSql->self == pSql);
   pSql->rpcRid = -1;
 
   if (pObj->signature != pObj) {
     tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
 
-    taosCacheRelease(tscObjCache, (void**) &p, true);
+    taosRemoveRef(tscObjRef, pSql->self);
+    taosReleaseRef(tscObjRef, pSql->self);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
@@ -266,10 +256,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
         pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
 
-    void** p1 = p;
+    taosRemoveRef(tscObjRef, pSql->self);
-    taosCacheRelease(tscObjCache, (void**) &p1, false);
+    taosReleaseRef(tscObjRef, pSql->self);
-
-    taosCacheRelease(tscObjCache, (void**) &p, true);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
@@ -312,7 +300,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
 
       // if there is an error occurring, proceed to the following error handling procedure.
       if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
-        taosCacheRelease(tscObjCache, (void**) &p, false);
+        taosReleaseRef(tscObjRef, pSql->self);
         rpcFreeCont(rpcMsg->pCont);
         return;
       }
@@ -380,11 +368,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
   }
 
-  void** p1 = p;
+  taosReleaseRef(tscObjRef, pSql->self);
-  taosCacheRelease(tscObjCache, (void**) &p1, false);
 
   if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
-    taosCacheRelease(tscObjCache, (void **)&p, true);
+    taosRemoveRef(tscObjRef, pSql->self);
     tscDebug("%p sqlObj is automatically freed", pSql);
   }
 
@@ -687,6 +674,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
   pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->queryType      = htonl(pQueryInfo->type);
+  pQueryMsg->tableLimit     = htobe64(pQueryInfo->tableLimit);
   
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number
@@ -2010,7 +1998,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
 
 // TODO multithread problem
 static void createHBObj(STscObj* pObj) {
-  if (pObj->pHb != NULL) {
+  if (pObj->hbrid != 0) {
     return;
   }
 
@@ -2042,7 +2030,7 @@ static void createHBObj(STscObj* pObj) {
   registerSqlObj(pSql);
   tscDebug("%p HB is allocated, pObj:%p", pSql, pObj);
 
-  pObj->pHb = pSql;
+  pObj->hbrid = pSql->self;
 }
 
 int tscProcessConnectRsp(SSqlObj *pSql) {

src/client/src/tscSql.c

@@ -292,8 +292,8 @@ void taos_close(TAOS *taos) {
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
 
-  SSqlObj* pHb = pObj->pHb;
+  SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-  if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
+  if (pHb != NULL) {
     if (pHb->rpcRid > 0) {  // wait for rsp from dnode
       rpcCancelRequest(pHb->rpcRid);
       pHb->rpcRid = -1;
@@ -301,6 +301,7 @@ void taos_close(TAOS *taos) {
 
     tscDebug("%p HB is freed", pHb);
     taos_free_result(pHb);
+    taosReleaseRef(tscObjRef, pHb->self);
   }
 
   int32_t ref = T_REF_DEC(pObj);
@@ -622,8 +623,7 @@ void taos_free_result(TAOS_RES *res) {
   bool freeNow = tscKillQueryInDnode(pSql);
   if (freeNow) {
     tscDebug("%p free sqlObj in cache", pSql);
-    SSqlObj** p = pSql->self;
+    taosReleaseRef(tscObjRef, pSql->self);
-    taosCacheRelease(tscObjCache, (void**) &p, true);
   }
 }
 
@@ -716,13 +716,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
       continue;
     }
 
-    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
+    SSqlObj* pSubObj = pSub;
-    if (p == NULL) {
-      continue;
-    }
-
-    SSqlObj* pSubObj = (SSqlObj*) (*p);
-    assert(pSubObj->self == (SSqlObj**) p);
 
     pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
     if (pSubObj->rpcRid > 0) {
@@ -731,7 +725,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
     }
 
     tscQueueAsyncRes(pSubObj);
-    taosCacheRelease(tscObjCache, (void**) &p, false);
+    taosReleaseRef(tscObjRef, pSubObj->self);
   }
 
   tscDebug("%p super table query cancelled", pSql);

src/client/src/tscSub.c

@@ -157,7 +157,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
 
   registerSqlObj(pSql);
 
-  code = tsParseSql(pSql, false);
+  code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     tsem_wait(&pSub->sem);
     code = pSql->res.code;
@@ -168,7 +168,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
     goto fail;
   }
 
-  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+  if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
     line = __LINE__;
     code = TSDB_CODE_TSC_INVALID_SQL;
     goto fail;
@@ -179,10 +179,10 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
 fail:
   tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code));
   if (pSql != NULL) {
-    if (pSql->self != NULL) {
+    if (pSql->self != 0) {
-      taos_free_result(pSql);
+      taosReleaseRef(tscObjRef, pSql->self);
     } else {
-      tscFreeSqlObj(pSql);
+    tscFreeSqlObj(pSql);
     }
 
     pSql = NULL;
@@ -401,9 +401,11 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
     tscLoadSubscriptionProgress(pSub);
   }
 
-  if (!tscUpdateSubscription(pObj, pSub)) {
+  if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) {
-    taos_unsubscribe(pSub, 1);
+    if (!tscUpdateSubscription(pObj, pSub)) {
-    return NULL;
+      taos_unsubscribe(pSub, 1);
+      return NULL;
+    }
   }
 
   pSub->interval = interval;
@@ -417,10 +419,80 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
   return pSub;
 }
 
+SSqlObj* recreateSqlObj(SSub* pSub) {
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  if (pSql == NULL) {
+    return NULL;
+  }
+
+  pSql->signature = pSql;
+  pSql->pTscObj = pSub->taos;
+
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pSql->param = pSub;
+  pSql->maxRetry = TSDB_MAX_REPLICA;
+  pSql->fp = asyncCallback;
+  pSql->fetchFp = asyncCallback;
+  pSql->sqlstr = strdup(pSub->pSql->sqlstr);
+  if (pSql->sqlstr == NULL) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pRes->qhandle = 0;
+  pRes->numOfRows = 1;
+
+  int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  registerSqlObj(pSql);
+
+  code = tsParseSql(pSql, true);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    tsem_wait(&pSub->sem);
+    code = pSql->res.code;
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  return pSql;
+}
+
 TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   SSub *pSub = (SSub *)tsub;
   if (pSub == NULL) return NULL;
 
+  if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
+    SSqlObj* pSql = recreateSqlObj(pSub);
+    if (pSql == NULL) {
+      return NULL;
+    }
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
+    pSub->pSql = pSql;
+    pSql->pSubscription = pSub;
+  }
+
   tscSaveSubscriptionProgress(pSub);
 
   SSqlObj *pSql = pSub->pSql;
@@ -512,10 +584,13 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   if (pSub->pSql != NULL) {
-    taos_free_result(pSub->pSql);
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
   }
 
-  tscFreeSqlObj(pSub->pSql);
   taosArrayDestroy(pSub->progress);
   tsem_destroy(&pSub->sem);
   memset(pSub, 0, sizeof(*pSub));

src/client/src/tscSubquery.c

@@ -2198,6 +2198,9 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
   STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
   int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
 
+  // free the data block created from insert sql string
+  pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
+
   if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
     tscQueueAsyncRes(pSql);
     return code;  // here the pSql may have been released already.

src/client/src/tscSystem.c

@@ -15,7 +15,7 @@
 
 #include "os.h"
 #include "taosmsg.h"
-#include "tcache.h"
+#include "tref.h"
 #include "trpc.h"
 #include "tsystem.h"
 #include "ttimer.h"
@@ -31,7 +31,7 @@
 
 // global, not configurable
 SCacheObj*  tscMetaCache;
-SCacheObj*  tscObjCache;
+int     tscObjRef = -1;
 void *  tscTmr;
 void *  tscQhandle;
 void *  tscCheckDiskUsageTmr;
@@ -139,7 +139,7 @@ void taos_init_imp(void) {
   int64_t refreshTime = 10; // 10 seconds by default
   if (tscMetaCache == NULL) {
     tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
+    tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
   }
 
   tscRefId = taosOpenRef(200, tscCloseTscObj);
@@ -162,9 +162,9 @@ void taos_cleanup(void) {
     taosCacheCleanup(m);
   }
 
-  m = tscObjCache;
+  int refId = atomic_exchange_32(&tscObjRef, -1);
-  if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+  if (refId != -1) {
-    taosCacheCleanup(m);
+    taosCloseRef(refId);
   }
 
   m = tscQhandle;

src/client/src/tscUtil.c

@@ -447,20 +447,18 @@ static void tscFreeSubobj(SSqlObj* pSql) {
 void tscFreeRegisteredSqlObj(void *pSql) {
   assert(pSql != NULL);
 
-  SSqlObj** p = (SSqlObj**)pSql;
+  SSqlObj* p = *(SSqlObj**)pSql;
-  STscObj* pTscObj = (*p)->pTscObj;
+  STscObj* pTscObj = p->pTscObj;
 
-  assert((*p)->self != 0 && (*p)->self == (p));
+  assert(p->self != 0);
-
+  tscFreeSqlObj(p);
-  SSqlObj* ptr = *p;
-  tscFreeSqlObj(*p);
 
   int32_t ref = T_REF_DEC(pTscObj);
   assert(ref >= 0);
 
-  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref);
+  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref);
   if (ref == 0) {
-    tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj);
+    tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj);
     taosRemoveRef(tscRefId, pTscObj->rid);
   }
 }
@@ -840,7 +838,6 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
 
     // the length does not include the SSubmitBlk structure
     pBlocks->dataLen = htonl(finalLen);
-
     dataBuf->numOfTables += 1;
   }
 
@@ -1565,19 +1562,6 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) {
   }
 }
 
-void tscSetFreeHeatBeat(STscObj* pObj) {
-  if (pObj == NULL || pObj->signature != pObj || pObj->pHb == NULL) {
-    return;
-  }
-
-  SSqlObj* pHeatBeat = pObj->pHb;
-  assert(pHeatBeat == pHeatBeat->signature);
-
-  // to denote the heart-beat timer close connection and free all allocated resources
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHeatBeat->cmd, 0);
-  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-}
-
 /*
  * the following four kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
@@ -1596,7 +1580,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
   }
   
   STscObj* pTscObj = pSql->pTscObj;
-  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
+  if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) {
     return false;
   }
 
@@ -1888,13 +1872,10 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
 }
 
 void registerSqlObj(SSqlObj* pSql) {
-  int32_t DEFAULT_LIFE_TIME = 2 * 600 * 1000;  // 1200 sec
-
   int32_t ref = T_REF_INC(pSql->pTscObj);
   tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref);
 
-  TSDB_CACHE_PTR_TYPE p = (TSDB_CACHE_PTR_TYPE) pSql;
+  pSql->self = taosAddRef(tscObjRef, pSql);
-  pSql->self = taosCachePut(tscObjCache, &p, sizeof(TSDB_CACHE_PTR_TYPE), &p, sizeof(TSDB_CACHE_PTR_TYPE), DEFAULT_LIFE_TIME);
 }
 
 SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {

src/common/inc/tglobal.h

@@ -125,6 +125,9 @@ extern char    tsMonitorDbName[];
 extern char    tsInternalPass[];
 extern int32_t tsMonitorInterval;
 
+// stream
+extern int32_t tsEnableStream;
+
 // internal
 extern int32_t tsPrintAuth;
 extern int32_t tscEmbedded;
@@ -176,7 +179,7 @@ extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
 extern int32_t mqttDebugFlag;
-extern int32_t monitorDebugFlag;
+extern int32_t monDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t odbcDebugFlag;

src/common/src/tglobal.c

@@ -161,6 +161,9 @@ char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+// stream
+int32_t tsEnableStream = 1;
+
 // internal
 int32_t tsPrintAuth = 0;
 int32_t tscEmbedded = 0;
@@ -200,13 +203,13 @@ int32_t tsNumOfLogLines = 10000000;
 int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
 int32_t dDebugFlag = 135;
-int32_t vDebugFlag = 131;
+int32_t vDebugFlag = 135;
 int32_t cDebugFlag = 131;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
 int32_t mqttDebugFlag = 131;
-int32_t monitorDebugFlag = 131;
+int32_t monDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;
@@ -216,9 +219,9 @@ int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 135;
 
-int32_t (*monitorStartSystemFp)() = NULL;
+int32_t (*monStartSystemFp)() = NULL;
-void (*monitorStopSystemFp)() = NULL;
+void (*monStopSystemFp)() = NULL;
-void (*monitorExecuteSQLFp)(char *sql) = NULL;
+void (*monExecuteSQLFp)(char *sql) = NULL;
 
 char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
 
@@ -235,7 +238,7 @@ void taosSetAllDebugFlag() {
     odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
     mqttDebugFlag = debugFlag;
-    monitorDebugFlag = debugFlag;
+    monDebugFlag = debugFlag;
     qDebugFlag = debugFlag;    
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -276,15 +279,15 @@ bool taosCfgDynamicOptions(char *msg) {
 
     if (strncasecmp(cfg->option, "monitor", olen) == 0) {
       if (1 == vint) {
-        if (monitorStartSystemFp) {
+        if (monStartSystemFp) {
-          (*monitorStartSystemFp)();
+          (*monStartSystemFp)();
           uInfo("monitor is enabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
         }
       } else {
-        if (monitorStopSystemFp) {
+        if (monStopSystemFp) {
-          (*monitorStopSystemFp)();
+          (*monStopSystemFp)();
           uInfo("monitor is disabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
@@ -307,8 +310,8 @@ bool taosCfgDynamicOptions(char *msg) {
   }
 
   if (strncasecmp(option, "resetQueryCache", 15) == 0) {
-    if (monitorExecuteSQLFp) {
+    if (monExecuteSQLFp) {
-      (*monitorExecuteSQLFp)("resetQueryCache");
+      (*monExecuteSQLFp)("resetQueryCache");
       uInfo("resetquerycache is executed");
     } else {
       uError("resetquerycache can't be executed, for monitor not started");
@@ -1015,6 +1018,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "stream";
+  cfg.ptr = &tsEnableStream;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "httpEnableRecordSql";
   cfg.ptr = &tsHttpEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1227,8 +1240,8 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "monitorDebugFlag";
+  cfg.option = "monDebugFlag";
-  cfg.ptr = &monitorDebugFlag;
+  cfg.ptr = &monDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;

src/connector/go

@@ -1 +1 @@
-Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
+Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766

src/connector/grafanaplugin

@@ -1 +1 @@
-Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944
+Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c

src/cq/src/cqMain.c

@@ -40,15 +40,14 @@
 
 typedef struct {
   int32_t  vgId;
+  int32_t  master;
+  int32_t  num;      // number of continuous streams
   char     user[TSDB_USER_LEN];
   char     pass[TSDB_PASSWORD_LEN];
   char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
-  void    *ahandle;
-  int32_t  num;      // number of continuous streams
   struct SCqObj *pHead;
   void    *dbConn;
-  int32_t  master;
   void    *tmrCtrl;
   pthread_mutex_t mutex;
 } SCqContext;
@@ -70,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
 static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
 
 void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = calloc(sizeof(SCqContext), 1);
   if (pContext == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -90,7 +92,6 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   tstrncpy(pContext->db, db, sizeof(pContext->db));
   pContext->vgId = pCfg->vgId;
   pContext->cqWrite = pCfg->cqWrite;
-  pContext->ahandle = ahandle;
   tscEmbedded = 1;
 
   pthread_mutex_init(&pContext->mutex, NULL);
@@ -101,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
 }
 
 void cqClose(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (handle == NULL) return;
 
@@ -131,6 +135,9 @@ void cqClose(void *handle) {
 }
 
 void cqStart(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (pContext->dbConn || pContext->master) return;
 
@@ -149,6 +156,9 @@ void cqStart(void *handle) {
 }
 
 void cqStop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   cInfo("vgId:%d, stop all CQs", pContext->vgId);
   if (pContext->dbConn == NULL || pContext->master == 0) return;
@@ -176,6 +186,9 @@ void cqStop(void *handle) {
 }
 
 void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = handle;
 
   SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -205,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
 }
 
 void cqDrop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqObj *pObj = handle;
   SCqContext *pContext = pObj->pContext;
 
@@ -241,8 +257,12 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
   SCqObj* pObj = (SCqObj*)param;
   SCqContext* pContext = pObj->pContext;
   SSqlObj* pSql = (SSqlObj*)result;
-  pContext->dbConn = pSql->pTscObj;
+  if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) {
+    taos_close(pSql->pTscObj);
+  }
+  pthread_mutex_lock(&pContext->mutex);
   cqCreateStream(pContext, pObj);
+  pthread_mutex_unlock(&pContext->mutex);
 }
 
 static void cqProcessCreateTimer(void *param, void *tmrId) {
@@ -253,7 +273,9 @@ static void cqProcessCreateTimer(void *param, void *tmrId) {
     cDebug("vgId:%d, try connect to TDengine", pContext->vgId);
     taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL);
   } else {
+    pthread_mutex_lock(&pContext->mutex);
     cqCreateStream(pContext, pObj);
+    pthread_mutex_unlock(&pContext->mutex);
   }
 }
 
@@ -267,12 +289,14 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
   }
   pObj->tmrId = 0;
 
-  pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
+  if (pObj->pStream == NULL) {
-  if (pObj->pStream) {
+    pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
-    pContext->num++;
+    if (pObj->pStream) {
-    cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
+      pContext->num++;
-  } else {
+      cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
-    cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
+    } else {
+      cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
+    }
   }
 }
 
@@ -334,7 +358,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   pHead->version = 0;
 
   // write into vnode write queue
-  pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
+  pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL);
   free(buffer);
 }
 

src/cq/test/cqtest.c

@@ -24,7 +24,7 @@
 int64_t  ver = 0;
 void    *pCq = NULL;
 
-int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { 
+int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) { 
   return 0;
 }
 

src/dnode/src/dnodeMain.c

@@ -19,6 +19,7 @@
 #include "tutil.h"
 #include "tconfig.h"
 #include "tglobal.h"
+#include "tfile.h"
 #include "twal.h"
 #include "trpc.h"
 #include "dnode.h"
@@ -55,6 +56,7 @@ typedef struct {
 } SDnodeComponent;
 
 static const SDnodeComponent tsDnodeComponents[] = {
+  {"tfile",     tfInit,              tfCleanup},
   {"rpc",       rpcInit,             rpcCleanup},
   {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
   {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},

src/dnode/src/dnodeMgmt.c

@@ -24,7 +24,7 @@
 #include "tqueue.h"
 #include "tsync.h"
 #include "ttimer.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "dnode.h"
 #include "vnode.h"
@@ -444,12 +444,12 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
   SCreateMnodeMsg *pCfg = pMsg->pCont;
   pCfg->dnodeId = htonl(pCfg->dnodeId);
   if (pCfg->dnodeId != dnodeGetDnodeId()) {
-    dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
+    dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
     return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
   }
 
   if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
-    dError("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
+    dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
     return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
   }
 

src/dnode/src/dnodeModule.c

@@ -78,10 +78,10 @@ static void dnodeAllocModules() {
 
   tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
-  tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
+  tsModule[TSDB_MOD_MONITOR].initFp    = monInitSystem;
-  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem;
+  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem;
-  tsModule[TSDB_MOD_MONITOR].startFp   = monitorStartSystem;
+  tsModule[TSDB_MOD_MONITOR].startFp   = monStartSystem;
-  tsModule[TSDB_MOD_MONITOR].stopFp    = monitorStopSystem;
+  tsModule[TSDB_MOD_MONITOR].stopFp    = monStopSystem;
   if (tsEnableMonitorModule) {
     dnodeSetModuleStatus(TSDB_MOD_MONITOR);
   }

src/dnode/src/dnodePeer.c

@@ -182,6 +182,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcEpSet epSet = {0};
   dnodeGetEpSetForPeer(&epSet);
+
+  assert(tsClientRpc != 0);
   rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
 }
 

src/inc/monitor.h

@@ -47,13 +47,13 @@ typedef struct {
   int8_t  accessState;
 } SAcctMonitorObj;
 
-int32_t monitorInitSystem();
+int32_t monInitSystem();
-int32_t monitorStartSystem();
+int32_t monStartSystem();
-void    monitorStopSystem();
+void    monStopSystem();
-void    monitorCleanUpSystem();
+void    monCleanupSystem();
-void    monitorSaveAcctLog(SAcctMonitorObj *pMonObj);
+void    monSaveAcctLog(SAcctMonitorObj *pMonObj);
-void    monitorSaveLog(int32_t level, const char *const format, ...);
+void    monSaveLog(int32_t level, const char *const format, ...);
-void    monitorExecuteSQL(char *sql);
+void    monExecuteSQL(char *sql);
 
 #ifdef __cplusplus
 }

src/inc/taosdef.h

@@ -257,7 +257,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_SQL_SHOW_LEN     512
-#define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 8mb
+#define TSDB_MAX_ALLOWED_SQL_LEN  (1*1024*1024U)          // sql length should be less than 1mb
 
 #define TSDB_APPNAME_LEN          TSDB_UNI_LEN
 

src/inc/taoserror.h

@@ -184,6 +184,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES,       0, 0x0385, "Too many d
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database not available")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         0, 0x0387, "Database unsynced")
 
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS,   0, 0x0390, "Invalid database option: days out of range")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days")
+
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            0, 0x0401, "Dnode out of memory")
@@ -367,6 +370,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG,   0, 0x11A4, "tag value
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL,           0, 0x11A5, "value not find")
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE,           0, 0x11A6, "value type should be boolean, number or string")
 
+// odbc
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM,                     0, 0x2100, "out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM,       0, 0x2101, "convertion not a valid literal input")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF,              0, 0x2102, "convertion undefined")
@@ -390,7 +394,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ,        0, 0x2113, "src bad se
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE,     0, 0x2114, "src incomplete")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL,        0, 0x2115, "src general")
 
-
 #ifdef TAOS_ERROR_C
 };
 #endif

src/inc/taosmsg.h

@@ -476,19 +476,21 @@ typedef struct {
   int16_t     numOfGroupCols;   // num of group by columns
   int16_t     orderByIdx;
   int16_t     orderType;        // used in group by xx order by xxx
+  int64_t     tableLimit;       // limit the number of rows for each table, used in order by + limit in stable projection query.
+  int16_t     prjOrder;         // global order in super table projection query.
   int64_t     limit;
   int64_t     offset;
   uint32_t    queryType;        // denote another query process
   int16_t     numOfOutput;      // final output columns numbers
   int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
-  int16_t     fillType;      // interpolate type
+  int16_t     fillType;         // interpolate type
-  uint64_t    fillVal;       // default value array list
+  uint64_t    fillVal;          // default value array list
   int32_t     secondStageOutput;
-  int32_t     tsOffset;       // offset value in current msg body, NOTE: ts list is compressed
+  int32_t     tsOffset;         // offset value in current msg body, NOTE: ts list is compressed
-  int32_t     tsLen;          // total length of ts comp block
+  int32_t     tsLen;            // total length of ts comp block
-  int32_t     tsNumOfBlocks;  // ts comp block numbers
+  int32_t     tsNumOfBlocks;    // ts comp block numbers
-  int32_t     tsOrder;        // ts comp block order
+  int32_t     tsOrder;          // ts comp block order
-  int32_t     numOfTags;      // number of tags columns involved
+  int32_t     numOfTags;        // number of tags columns involved
   SColumnInfo colList[];
 } SQueryTableMsg;
 

src/inc/tbn.h

@@ -23,14 +23,14 @@ extern "C" {
 struct SVgObj;
 struct SDnodeObj;
 
-int32_t balanceInit();
+int32_t bnInit();
-void    balanceCleanUp();
+void    bnCleanUp();
-void    balanceAsyncNotify();
+void    bnNotify();
-void    balanceSyncNotify();
+void    bnCheckModules();
-void    balanceReset();
+void    bnReset();
-int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
+int32_t bnAllocVnodes(struct SVgObj *pVgroup);
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
+int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
-int32_t balanceDropDnode(struct SDnodeObj *pDnode);
+int32_t bnDropDnode(struct SDnodeObj *pDnode);
 
 #ifdef __cplusplus
 }

src/inc/tcq.h

@@ -21,7 +21,7 @@ extern "C" {
 
 #include "tdataformat.h"
 
-typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
+typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
 
 typedef struct {
   int32_t  vgId;

src/inc/tsync.h

@@ -85,6 +85,9 @@ typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 // when data file is synced successfully, notity app
 typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
 
+// get file version
+typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
+
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
@@ -97,6 +100,7 @@ typedef struct {
   FNotifyRole       notifyRole;
   FNotifyFlowCtrl   notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
+  FGetVersion       getVersion;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/inc/twal.h

@@ -51,9 +51,8 @@ typedef struct {
 typedef void *  twalh;  // WAL HANDLE
 typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
 
-int32_t walInit();
+int32_t  walInit();
-void    walCleanUp();
+void     walCleanUp();
-
 twalh    walOpen(char *path, SWalCfg *pCfg);
 int32_t  walAlter(twalh pWal, SWalCfg *pCfg);
 void     walStop(twalh);

src/kit/shell/src/shellDarwin.c

@@ -407,7 +407,11 @@ void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid
 
 void clearScreen(int ecmd_pos, int cursor_pos) {
   struct winsize w;
-  ioctl(0, TIOCGWINSZ, &w);
+  if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
+    //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
+    w.ws_col = 120;
+    w.ws_row = 30;
+  }
 
   int cursor_x = cursor_pos / w.ws_col;
   int cursor_y = cursor_pos % w.ws_col;
@@ -425,8 +429,9 @@ void clearScreen(int ecmd_pos, int cursor_pos) {
 void showOnScreen(Command *cmd) {
   struct winsize w;
   if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
-    fprintf(stderr, "No stream device\n");
+    //fprintf(stderr, "No stream device\n");
-    exit(EXIT_FAILURE);
+    w.ws_col = 120;
+    w.ws_row = 30;
   }
 
   wchar_t wc;

src/kit/shell/src/shellLinux.c

@@ -413,7 +413,11 @@ void get_history_path(char *history) { snprintf(history, TSDB_FILENAME_LEN, "%s/
 
 void clearScreen(int ecmd_pos, int cursor_pos) {
   struct winsize w;
-  ioctl(0, TIOCGWINSZ, &w);
+  if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
+    //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
+    w.ws_col = 120;
+    w.ws_row = 30;
+  }
 
   int cursor_x = cursor_pos / w.ws_col;
   int cursor_y = cursor_pos % w.ws_col;
@@ -431,8 +435,9 @@ void clearScreen(int ecmd_pos, int cursor_pos) {
 void showOnScreen(Command *cmd) {
   struct winsize w;
   if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
-    fprintf(stderr, "No stream device\n");
+    //fprintf(stderr, "No stream device\n");
-    exit(EXIT_FAILURE);
+    w.ws_col = 120;
+    w.ws_row = 30;
   }
 
   wchar_t wc;

src/mnode/inc/mnodeAcct.h

@@ -27,6 +27,7 @@ void    mnodeCleanupAccts();
 void    mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo);
 void *  mnodeGetAcct(char *acctName);
 void *  mnodeGetNextAcct(void *pIter, SAcctObj **pAcct);
+void    mnodeCancelGetNextAcct(void *pIter);
 void    mnodeIncAcctRef(SAcctObj *pAcct);
 void    mnodeDecAcctRef(SAcctObj *pAcct);
 void    mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);

src/mnode/inc/mnodeDb.h

@@ -34,6 +34,7 @@ int64_t mnodeGetDbNum();
 SDbObj *mnodeGetDb(char *db);
 SDbObj *mnodeGetDbByTableId(char *db);
 void *  mnodeGetNextDb(void *pIter, SDbObj **pDb);
+void    mnodeCancelGetNextDb(void *pIter);
 void    mnodeIncDbRef(SDbObj *pDb);
 void    mnodeDecDbRef(SDbObj *pDb);
 bool    mnodeCheckIsMonitorDB(char *db, char *monitordb);

src/mnode/inc/mnodeDnode.h

@@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum();
 int32_t mnodeGetOnlinDnodesCpuCoreNum();
 int32_t mnodeGetOnlineDnodesNum();
 void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
+void    mnodeCancelGetNextDnode(void *pIter);
 void    mnodeIncDnodeRef(SDnodeObj *pDnode);
 void    mnodeDecDnodeRef(SDnodeObj *pDnode);
 void *  mnodeGetDnode(int32_t dnodeId);

src/mnode/inc/mnodeInt.h

@@ -41,9 +41,9 @@ extern int32_t sdbDebugFlag;
 #define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
 #define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
 
-#define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
+#define mLError(...) { monSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
-#define mLWarn(...)  { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)  }
+#define mLWarn(...)  { monSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)  }
-#define mLInfo(...)  { monitorSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
+#define mLInfo(...)  { monSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeMnode.h

@@ -38,6 +38,7 @@ void    mnodeDropMnodeLocal(int32_t dnodeId);
 void *  mnodeGetMnode(int32_t mnodeId);
 int32_t mnodeGetMnodesNum();
 void *  mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
+void    mnodeCancelGetNextMnode(void *pIter);
 void    mnodeIncMnodeRef(struct SMnodeObj *pMnode);
 void    mnodeDecMnodeRef(struct SMnodeObj *pMnode);
 

src/mnode/inc/mnodeSdb.h

@@ -79,10 +79,13 @@ typedef struct {
   int32_t (*fpRestored)();
 } SSdbTableDesc;
 
+int32_t sdbInitRef();
+void    sdbCleanUpRef();
 int32_t sdbInit();
 void    sdbCleanUp();
-void *  sdbOpenTable(SSdbTableDesc *desc);
+int64_t sdbOpenTable(SSdbTableDesc *desc);
-void    sdbCloseTable(void *handle);
+void    sdbCloseTable(int64_t rid);
+void*   sdbGetTableByRid(int64_t rid);
 bool    sdbIsMaster();
 bool    sdbIsServing();
 void    sdbUpdateMnodeRoles();
@@ -92,9 +95,9 @@ int32_t sdbDeleteRow(SSdbRow *pRow);
 int32_t sdbUpdateRow(SSdbRow *pRow);
 int32_t sdbInsertRowToQueue(SSdbRow *pRow);
 
-void    *sdbGetRow(void *pTable, void *key);
+void *   sdbGetRow(void *pTable, void *key);
-void    *sdbFetchRow(void *pTable, void *pIter, void **ppRow);
+void *   sdbFetchRow(void *pTable, void *pIter, void **ppRow);
-void     sdbFreeIter(void *pIter);
+void     sdbFreeIter(void *pTable, void *pIter);
 void     sdbIncRef(void *pTable, void *pRow);
 void     sdbDecRef(void *pTable, void *pRow);
 int64_t  sdbGetNumOfRows(void *pTable);

src/mnode/inc/mnodeShow.h

@@ -26,8 +26,10 @@ void    mnodeCleanUpShow();
 
 typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+typedef void    (*SShowFreeIterFp)(void *pIter);
 void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp);
 void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
+void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp);
 void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
 
 #ifdef __cplusplus

src/mnode/inc/mnodeTable.h

@@ -31,6 +31,8 @@ void    mnodeIncTableRef(void *pTable);
 void    mnodeDecTableRef(void *pTable);
 void *  mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
 void *  mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
+void    mnodeCancelGetNextChildTable(void *pIter);
+void    mnodeCancelGetNextSuperTable(void *pIter);
 void    mnodeDropAllChildTables(SDbObj *pDropDb);
 void    mnodeDropAllSuperTables(SDbObj *pDropDb);
 void    mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);

src/mnode/inc/mnodeUser.h

@@ -25,6 +25,7 @@ int32_t   mnodeInitUsers();
 void      mnodeCleanupUsers();
 SUserObj *mnodeGetUser(char *name);
 void *    mnodeGetNextUser(void *pIter, SUserObj **pUser);
+void      mnodeCancelGetNextUser(void *pIter);
 void      mnodeIncUserRef(SUserObj *pUser);
 void      mnodeDecUserRef(SUserObj *pUser);
 SUserObj *mnodeGetUserFromConn(void *pConn);

src/mnode/inc/mnodeVgroup.h

@@ -34,6 +34,7 @@ void    mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 //void  mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
 
 void *  mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
+void    mnodeCancelGetNextVgroup(void *pIter);
 void    mnodeUpdateVgroup(SVgObj *pVgroup);
 void    mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
 void    mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);

src/mnode/src/mnodeAcct.c

@@ -26,6 +26,7 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
+int64_t tsAcctRid = -1;
 void *  tsAcctSdb = NULL;
 static int32_t tsAcctUpdateSize;
 static int32_t mnodeCreateRootAcct();
@@ -114,7 +115,8 @@ int32_t mnodeInitAccts() {
     .fpRestored   = mnodeAcctActionRestored
   };
 
-  tsAcctSdb = sdbOpenTable(&desc);
+  tsAcctRid = sdbOpenTable(&desc);
+  tsAcctSdb = sdbGetTableByRid(tsAcctRid);
   if (tsAcctSdb == NULL) {
     mError("table:%s, failed to create hash", desc.name);
     return -1;
@@ -126,7 +128,7 @@ int32_t mnodeInitAccts() {
 
 void mnodeCleanupAccts() {
   acctCleanUp();
-  sdbCloseTable(tsAcctSdb);
+  sdbCloseTable(tsAcctRid);
   tsAcctSdb = NULL;
 }
 
@@ -144,7 +146,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
     pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries;
     mnodeDecAcctRef(pAcct);
   }
-  sdbFreeIter(pIter);
 
   SVgObj *pVgroup = NULL;
   pIter = NULL;
@@ -158,7 +159,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
     pAcctInfo->totalPoints += pVgroup->pointsWritten;
     mnodeDecVgroupRef(pVgroup);
   }
-  sdbFreeIter(pIter);
 }
 
 void *mnodeGetAcct(char *name) {
@@ -169,6 +169,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) {
   return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); 
 }
 
+void mnodeCancelGetNextAcct(void *pIter) {
+  sdbFreeIter(tsAcctSdb, pIter);
+}
+
 void mnodeIncAcctRef(SAcctObj *pAcct) {
   sdbIncRef(tsAcctSdb, pAcct);
 }

src/mnode/src/mnodeCluster.c

@@ -24,6 +24,7 @@
 #include "mnodeShow.h"
 #include "tglobal.h"
 
+int64_t        tsClusterRid = -1;
 static void *  tsClusterSdb = NULL;
 static int32_t tsClusterUpdateSize;
 static char    tsClusterId[TSDB_CLUSTER_ID_LEN];
@@ -31,6 +32,7 @@ static int32_t mnodeCreateCluster();
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void    mnodeCancelGetNextCluster(void *pIter);
 
 static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) {
   tfree(pRow->pObj);
@@ -100,21 +102,23 @@ int32_t mnodeInitCluster() {
     .fpRestored   = mnodeClusterActionRestored
   };
 
-  tsClusterSdb = sdbOpenTable(&desc);
+  tsClusterRid = sdbOpenTable(&desc);
+  tsClusterSdb = sdbGetTableByRid(tsClusterRid);
   if (tsClusterSdb == NULL) {
-    mError("table:%s, failed to create hash", desc.name);
+    mError("table:%s, rid:%" PRId64 ", failed to create hash", desc.name, tsClusterRid);
     return -1;
   }
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster);
 
   mDebug("table:%s, hash is created", desc.name);
   return TSDB_CODE_SUCCESS;
 }
 
 void mnodeCleanupCluster() {
-  sdbCloseTable(tsClusterSdb);
+  sdbCloseTable(tsClusterRid);
   tsClusterSdb = NULL;
 }
 
@@ -122,6 +126,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) {
   return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); 
 }
 
+void mnodeCancelGetNextCluster(void *pIter) {
+  sdbFreeIter(tsClusterSdb, pIter);
+}
+
 void mnodeIncClusterRef(SClusterObj *pCluster) {
   sdbIncRef(tsClusterSdb, pCluster);
 }
@@ -167,7 +175,7 @@ void mnodeUpdateClusterId() {
   }
 
   mnodeDecClusterRef(pCluster);
-  sdbFreeIter(pIter);
+  mnodeCancelGetNextCluster(pIter);
 }
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {

src/mnode/src/mnodeDb.c

@@ -20,7 +20,7 @@
 #include "tgrant.h"
 #include "tglobal.h"
 #include "tname.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tdataformat.h"
 #include "mnode.h"
 #include "mnodeDef.h"
@@ -38,6 +38,7 @@
 #include "mnodeVgroup.h"
 
 #define VG_LIST_SIZE 8
+int64_t        tsDbRid = -1;
 static void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
@@ -160,7 +161,8 @@ int32_t mnodeInitDbs() {
     .fpRestored   = mnodeDbActionRestored
   };
 
-  tsDbSdb = sdbOpenTable(&desc);
+  tsDbRid = sdbOpenTable(&desc);
+  tsDbSdb = sdbGetTableByRid(tsDbRid);
   if (tsDbSdb == NULL) {
     mError("failed to init db data");
     return -1;
@@ -171,6 +173,7 @@ int32_t mnodeInitDbs() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb);
   
   mDebug("table:dbs table is created");
   return 0;
@@ -180,6 +183,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) {
   return sdbFetchRow(tsDbSdb, pIter, (void **)pDb);
 }
 
+void mnodeCancelGetNextDb(void *pIter) {
+  sdbFreeIter(tsDbSdb, pIter);
+}
+
 SDbObj *mnodeGetDb(char *db) {
   return (SDbObj *)sdbGetRow(tsDbSdb, db);
 }
@@ -229,30 +236,28 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
   if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
     mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE,
            TSDB_MAX_DAYS_PER_FILE);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
+    return TSDB_CODE_MND_INVALID_DB_OPTION_DAYS;
   }
 
   if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) {
     mError("invalid db option daysToKeep:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
+    return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
   }
 
   if (pCfg->daysToKeep < pCfg->daysPerFile) {
     mError("invalid db option daysToKeep:%d should larger than daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
+    return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
   }
 
-#if 0
   if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) {
-    mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep);
+    mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep2, TSDB_MIN_KEEP, pCfg->daysToKeep);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
+    return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
   }
 
   if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) {
     mError("invalid db option daysToKeep1:%d valid range: [%d, %d]", pCfg->daysToKeep1, TSDB_MIN_KEEP, pCfg->daysToKeep2);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
+    return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
   }
-#endif
 
   if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
     mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock,
@@ -491,7 +496,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) {
 }
 
 void mnodeCleanupDbs() {
-  sdbCloseTable(tsDbSdb);
+  sdbCloseTable(tsDbRid);
   tsDbSdb = NULL;
 }
 
@@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
   SDbObj *pDb = pMsg->pDb;
 
   void *pIter = NULL;
-  while (1) {
+  SVgObj *pVgroup = NULL;
-    SVgObj *pVgroup = NULL;
+    while (1) {
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
     if (pVgroup->pDb == pDb) {
@@ -995,12 +1000,11 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
     }
     mnodeDecVgroupRef(pVgroup);
   }
-  sdbFreeIter(pIter);
 
   mDebug("db:%s, all vgroups is altered", pDb->name);
   mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
 
-  balanceAsyncNotify();
+  bnNotify();
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1146,7 +1150,5 @@ void  mnodeDropAllDbs(SAcctObj *pAcct)  {
     mnodeDecDbRef(pDb);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs);
 }

src/mnode/src/mnodeDnode.c

@@ -16,12 +16,12 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tgrant.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "tconfig.h"
 #include "tutil.h"
 #include "tsocket.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tsync.h"
 #include "tdataformat.h"
 #include "mnode.h"
@@ -39,6 +39,7 @@
 #include "mnodeCluster.h"
 
 int32_t tsAccessSquence = 0;
+int64_t        tsDnodeRid = -1;
 static void *  tsDnodeSdb = NULL;
 static int32_t tsDnodeUpdateSize = 0;
 extern void *  tsMnodeSdb;
@@ -114,7 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
   mnodeDropAllDnodeVgroups(pDnode);
 #endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
-  balanceAsyncNotify();
+  bnNotify();
   mnodeUpdateDnodeEps();
 
   mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
@@ -187,7 +188,8 @@ int32_t mnodeInitDnodes() {
     .fpRestored   = mnodeDnodeActionRestored
   };
 
-  tsDnodeSdb = sdbOpenTable(&desc);
+  tsDnodeRid = sdbOpenTable(&desc);
+  tsDnodeSdb = sdbGetTableByRid(tsDnodeRid);
   if (tsDnodeSdb == NULL) {
     mError("failed to init dnodes data");
     return -1;
@@ -206,13 +208,14 @@ int32_t mnodeInitDnodes() {
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode);
  
   mDebug("table:dnodes table is created");
   return 0;
 }
 
 void mnodeCleanupDnodes() {
-  sdbCloseTable(tsDnodeSdb);
+  sdbCloseTable(tsDnodeRid);
   pthread_mutex_destroy(&tsDnodeEpsMutex);
   free(tsDnodeEps);
   tsDnodeEps = NULL;
@@ -223,6 +226,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) {
   return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); 
 }
 
+void mnodeCancelGetNextDnode(void *pIter) {
+  sdbFreeIter(tsDnodeSdb, pIter);
+}
+
 int32_t mnodeGetDnodesNum() {
   return sdbGetNumOfRows(tsDnodeSdb);
 }
@@ -241,8 +248,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (cpuCores < 2) cpuCores = 2;
   return cpuCores;
 }
@@ -259,8 +264,6 @@ int32_t mnodeGetOnlineDnodesNum() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   return onlineDnodes;
 }
 
@@ -276,13 +279,12 @@ void *mnodeGetDnodeByEp(char *ep) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return pDnode;
     }
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
 
   return NULL;
 }
@@ -345,7 +347,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
       return TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION;
     }
 
-    int32_t code = balanceAlterDnode(pDnode, vnodeId, dnodeId);
+    int32_t code = bnAlterDnode(pDnode, vnodeId, dnodeId);
     mnodeDecDnodeRef(pDnode);
     return code;
   } else {
@@ -464,7 +466,10 @@ static void mnodeUpdateDnodeEps() {
   while (1) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
-    if (dnodesNum >= totalDnodes) break;
+    if (dnodesNum >= totalDnodes) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
 
     SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
     dnodesNum++;
@@ -474,7 +479,6 @@ static void mnodeUpdateDnodeEps() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
   pthread_mutex_unlock(&tsDnodeEpsMutex);
 }
 
@@ -587,8 +591,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
     mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
     pDnode->status = TAOS_DN_STATUS_READY;
     pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
-    balanceSyncNotify();
+    bnCheckModules();
-    balanceAsyncNotify();
+    bnNotify();
   }
 
   if (openVnodes != pDnode->openVnodes) {
@@ -704,7 +708,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 #ifndef _SYNC
   int32_t code = mnodeDropDnode(pDnode, pMsg);
 #else
-  int32_t code = balanceDropDnode(pDnode);
+  int32_t code = bnDropDnode(pDnode);
 #endif  
   mnodeDecDnodeRef(pDnode);
   return code;
@@ -1100,7 +1104,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
     pDnode = mnodeGetDnodeByEp(pShow->payload);
   } else {
     void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode);
-    sdbFreeIter(pIter);
+    mnodeCancelGetNextDnode(pIter);
   }
 
   if (pDnode != NULL) {
@@ -1148,7 +1152,6 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
 
       mnodeDecVgroupRef(pVgroup);
     }
-    sdbFreeIter(pIter);
   } else {
     numOfRows = 0;
   }
@@ -1179,12 +1182,12 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
 
 #ifndef _SYNC
 
-int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
+int32_t bnInit() { return TSDB_CODE_SUCCESS; }
-void    balanceCleanUp() {}
+void    bnCleanUp() {}
-void    balanceAsyncNotify() {}
+void    bnNotify() {}
-void    balanceSyncNotify() {}
+void    bnCheckModules() {}
-void    balanceReset() {}
+void    bnReset() {}
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
+int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
 
 char* syncRole[] = {
   "offline",
@@ -1194,7 +1197,7 @@ char* syncRole[] = {
   "master"
 };
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t bnAllocVnodes(SVgObj *pVgroup) {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
@@ -1217,8 +1220,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (pSelDnode == NULL) {
     mError("failed to alloc vnode to vgroup");
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;

src/mnode/src/mnodeMain.c

@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "ttimer.h"
 #include "tglobal.h"
@@ -47,6 +47,7 @@ void *tsMnodeTmr = NULL;
 static bool tsMgmtIsRunning = false;
 
 static const SMnodeComponent tsMnodeComponents[] = {
+  {"sdbref",  sdbInitRef,       sdbCleanUpRef},
   {"profile", mnodeInitProfile, mnodeCleanupProfile},
   {"cluster", mnodeInitCluster, mnodeCleanupCluster},
   {"accts",   mnodeInitAccts,   mnodeCleanupAccts},
@@ -57,7 +58,7 @@ static const SMnodeComponent tsMnodeComponents[] = {
   {"tables",  mnodeInitTables,  mnodeCleanupTables},  
   {"mnodes",  mnodeInitMnodes,  mnodeCleanupMnodes},
   {"sdb",     sdbInit,          sdbCleanUp},
-  {"balance", balanceInit,      balanceCleanUp},
+  {"balance", bnInit,           bnCleanUp},
   {"grant",   grantInit,        grantCleanUp},
   {"show",    mnodeInitShow,    mnodeCleanUpShow}
 };

src/mnode/src/mnodeMnode.c

@@ -19,7 +19,7 @@
 #include "tglobal.h"
 #include "trpc.h"
 #include "tsync.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tutil.h"
 #include "tsocket.h"
 #include "tdataformat.h"
@@ -34,6 +34,7 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
+int64_t              tsMnodeRid = -1;
 static void *        tsMnodeSdb = NULL;
 static int32_t       tsMnodeUpdateSize = 0;
 static SRpcEpSet     tsMnodeEpSetForShell;
@@ -123,7 +124,7 @@ static int32_t mnodeMnodeActionRestored() {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
       mnodeDecMnodeRef(pMnode);
     }
-    sdbFreeIter(pIter);
+    mnodeCancelGetNextMnode(pIter);
   }
 
   mnodeUpdateMnodeEpSet();
@@ -153,7 +154,8 @@ int32_t mnodeInitMnodes() {
     .fpRestored   = mnodeMnodeActionRestored
   };
 
-  tsMnodeSdb = sdbOpenTable(&desc);
+  tsMnodeRid = sdbOpenTable(&desc);
+  tsMnodeSdb = sdbGetTableByRid(tsMnodeRid);
   if (tsMnodeSdb == NULL) {
     mError("failed to init mnodes data");
     return -1;
@@ -161,13 +163,14 @@ int32_t mnodeInitMnodes() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode);
 
   mDebug("table:mnodes table is created");
   return TSDB_CODE_SUCCESS;
 }
 
 void mnodeCleanupMnodes() {
-  sdbCloseTable(tsMnodeSdb);
+  sdbCloseTable(tsMnodeRid);
   tsMnodeSdb = NULL;
   mnodeMnodeDestroyLock();
 }
@@ -192,6 +195,10 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) {
   return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); 
 }
 
+void mnodeCancelGetNextMnode(void *pIter) {
+  sdbFreeIter(tsMnodeSdb, pIter);
+}
+
 void mnodeUpdateMnodeEpSet() {
   mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum());
 
@@ -239,8 +246,6 @@ void mnodeUpdateMnodeEpSet() {
   tsMnodeEpSetForShell.numOfEps = index;
   tsMnodeEpSetForPeer.numOfEps = index;
 
-  sdbFreeIter(pIter);
-
   mnodeMnodeUnLock();
 }
 
@@ -248,12 +253,30 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) {
   mnodeMnodeRdLock();
   *epSet = tsMnodeEpSetForPeer;
   mnodeMnodeUnLock();
+
+  for (int32_t i = 0; i < epSet->numOfEps; ++i) {
+    if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
+      epSet->inUse = (i + 1) % epSet->numOfEps;
+      mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+    } else {
+      mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+    }
+  }
 }
 
 void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
   mnodeMnodeRdLock();
   *epSet = tsMnodeEpSetForShell;
   mnodeMnodeUnLock();
+
+  for (int32_t i = 0; i < epSet->numOfEps; ++i) {
+    if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
+      epSet->inUse = (i + 1) % epSet->numOfEps;
+      mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+    } else {
+      mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+    }
+  }
 }
 
 char* mnodeGetMnodeMasterEp() {

src/mnode/src/mnodePeer.c

@@ -20,7 +20,7 @@
 #include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "mnode.h"
 #include "dnode.h"
@@ -58,16 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("msg:%p, ahandle:%p type:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg,
+    mDebug("msg:%p, ahandle:%p type:%s in mpeer queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
-           pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
+           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
-    for (int32_t i = 0; i < epSet->numOfEps; ++i) {
-      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
-        epSet->inUse = (i + 1) % epSet->numOfEps;
-        mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
-      } else {
-        mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
-      }
-    }
 
     return TSDB_CODE_RPC_REDIRECT;
   }

src/mnode/src/mnodeProfile.c

@@ -34,7 +34,6 @@
 #define QUERY_ID_SIZE   20
 #define QUERY_STREAM_SAVE_SIZE 20
 
-extern void *tsMnodeTmr;
 static SCacheObj *tsMnodeConnCache = NULL;
 static int32_t tsConnIndex = 0;
 
@@ -42,6 +41,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void    mnodeCancelGetNextConn(void *pIter);
 static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static void    mnodeFreeConn(void *data);
@@ -52,10 +52,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg);
 int32_t mnodeInitProfile() {
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn);
 
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
@@ -137,21 +140,15 @@ static void mnodeFreeConn(void *data) {
   mDebug("connId:%d, is destroyed", pConn->connId);
 }
 
-static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
+static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) {
   *pConn = NULL;
 
-  if (pIter == NULL) {
+  pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter);
-    pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable);
+  if (pIter == NULL) return NULL;
-  }
 
-  if (!taosHashIterNext(pIter)) {
+  SCacheDataNode **pNode = pIter;
-    taosHashDestroyIter(pIter);
-    return NULL;
-  }
-
-  SCacheDataNode **pNode = taosHashIterGet(pIter);
   if (pNode == NULL || *pNode == NULL) {
-    taosHashDestroyIter(pIter);
+    taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
     return NULL;
   }
 
@@ -159,6 +156,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
   return pIter;
 }
 
+static void mnodeCancelGetNextConn(void *pIter) {
+  taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
+}
+
 static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;

src/mnode/src/mnodeRead.c

@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "ttimer.h"
 #include "tglobal.h"
@@ -51,21 +51,12 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
     SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
     SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
     mnodeGetMnodeEpSetForShell(epSet);
-
-    mDebug("msg:%p, app:%p type:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
-           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
-    for (int32_t i = 0; i < epSet->numOfEps; ++i) {
-      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
-        epSet->inUse = (i + 1) % epSet->numOfEps;
-        mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
-      } else {
-        mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
-      }
-    }
-
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
+    mDebug("msg:%p, app:%p type:%s in mread queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
+           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
+
     return TSDB_CODE_RPC_REDIRECT;
   }
 

src/mnode/src/mnodeSdb.c

@@ -18,7 +18,8 @@
 #include "taoserror.h"
 #include "hash.h"
 #include "tutil.h"
-#include "tbalance.h"
+#include "tref.h"
+#include "tbn.h"
 #include "tqueue.h"
 #include "twal.h"
 #include "tsync.h"
@@ -98,6 +99,7 @@ typedef struct {
   SSdbWorker *worker;
 } SSdbWorkerPool;
 
+int32_t tsSdbRid;
 extern void *     tsMnodeTmr;
 static void *     tsSdbTmr;
 static SSdbMgmt   tsSdbMgmt = {0};
@@ -118,6 +120,7 @@ static void    sdbFreeQueue();
 static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow);
 static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow);
 static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow);
+static void    sdbCloseTableObj(void *handle);
 
 int32_t sdbGetId(void *pTable) {
   return ((SSdbTable *)pTable)->autoIndex;
@@ -241,13 +244,23 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]);
 
   if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) {
-    balanceReset();
+    bnReset();
   }
   tsSdbMgmt.role = role;
 
   sdbUpdateMnodeRoles();
 }
 
+static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
+
+static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
+
+static int32_t sdbGetSyncVersion(int32_t vgId, uint64_t *fver, uint64_t *vver) {
+  *fver = 0;
+  *vver = 0;
+  return 0;
+}
+
 // failed to forward, need revert insert
 static void sdbHandleFailedConfirm(SSdbRow *pRow) {
   SWalHead *pHead = pRow->pHead;
@@ -325,7 +338,6 @@ void sdbUpdateSync(void *pMnodes) {
       mnodeDecDnodeRef(pDnode);
       mnodeDecMnodeRef(pMnode);
     }
-    sdbFreeIter(pIter);
     syncCfg.replica = index;
     mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
   } else {
@@ -370,11 +382,14 @@ void sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.getWalInfo = sdbGetWalInfo;
   syncInfo.getFileInfo = sdbGetFileInfo;
+  syncInfo.getWalInfo = sdbGetWalInfo;
   syncInfo.writeToCache = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
   syncInfo.notifyRole = sdbNotifyRole;
+  syncInfo.notifyFileSynced = sdbNotifyFileSynced;
+  syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
+  syncInfo.getVersion = sdbGetSyncVersion;
   tsSdbMgmt.cfg = syncCfg;
 
   if (tsSdbMgmt.sync) {
@@ -386,6 +401,17 @@ void sdbUpdateSync(void *pMnodes) {
   sdbUpdateMnodeRoles();
 }
 
+int32_t sdbInitRef() {
+  tsSdbRid = taosOpenRef(10, sdbCloseTableObj);
+  if (tsSdbRid <= 0) {
+    sdbError("failed to init sdb ref");
+    return -1;
+  }
+  return 0;
+}
+
+void sdbCleanUpRef() { taosCloseRef(tsSdbRid); }
+
 int32_t sdbInit() {
   pthread_mutex_init(&tsSdbMgmt.mutex, NULL);
 
@@ -424,7 +450,7 @@ void sdbCleanUp() {
     walClose(tsSdbMgmt.wal);
     tsSdbMgmt.wal = NULL;
   }
-  
+
   pthread_mutex_destroy(&tsSdbMgmt.mutex);
 }
 
@@ -507,7 +533,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) {
     atomic_add_fetch_32(&pTable->autoIndex, 1);
   }
 
-  sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name,
+  sdbTrace("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name,
            sdbGetRowStr(pTable, pRow->pObj), pRow->rowSize, pTable->numOfRows, pRow->pMsg);
 
   int32_t code = (*pTable->fpInsert)(pRow);
@@ -543,7 +569,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
 
   atomic_sub_fetch_32(&pTable->numOfRows, 1);
 
-  sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
+  sdbTrace("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
            sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
 
   sdbDecRef(pTable, pRow->pObj);
@@ -552,7 +578,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
 }
 
 static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) {
-  sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
+  sdbTrace("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
            sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
 
   (*pTable->fpUpdate)(pRow);
@@ -650,7 +676,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *
     return syncCode;
   }
 
-  sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->name,
+  sdbTrace("vgId:1, sdb:%s, record from %s is disposed, action:%s key:%s hver:%" PRIu64, pTable->name, qtypeStr[qtype],
            actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version);
 
   // even it is WAL/FWD, it shall be called to update version in sync
@@ -775,24 +801,17 @@ int32_t sdbUpdateRow(SSdbRow *pRow) {
   }
 }
 
-void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) {
+void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) {
   SSdbTable *pTable = tparam;
   *ppRow = NULL;
   if (pTable == NULL) return NULL;
 
-  SHashMutableIterator *pIter = pNode;
+  pIter = taosHashIterate(pTable->iHandle, pIter);
-  if (pIter == NULL) {
+  if (pIter == NULL) return NULL;
-    pIter = taosHashCreateIter(pTable->iHandle);
-  }
 
-  if (!taosHashIterNext(pIter)) {
+  void **ppMetaRow = pIter;
-    taosHashDestroyIter(pIter);
-    return NULL;
-  }
-
-  void **ppMetaRow = taosHashIterGet(pIter);
   if (ppMetaRow == NULL) {
-    taosHashDestroyIter(pIter);
+    taosHashCancelIterate(pTable->iHandle, pIter);
     return NULL;
   }
 
@@ -802,16 +821,17 @@ void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) {
   return pIter;
 }
 
-void sdbFreeIter(void *pIter) {
+void sdbFreeIter(void *tparam, void *pIter) {
-  if (pIter != NULL) {
+  SSdbTable *pTable = tparam;
-    taosHashDestroyIter(pIter);
+  if (pTable == NULL || pIter == NULL) return;
-  }
+
+  taosHashCancelIterate(pTable->iHandle, pIter);
 }
 
-void *sdbOpenTable(SSdbTableDesc *pDesc) {
+int64_t sdbOpenTable(SSdbTableDesc *pDesc) {
   SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
   
-  if (pTable == NULL) return NULL;
+  if (pTable == NULL) return -1;
 
   pthread_mutex_init(&pTable->mutex, NULL);
   tstrncpy(pTable->name, pDesc->name, SDB_TABLE_LEN);
@@ -836,19 +856,31 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
 
   tsSdbMgmt.numOfTables++;
   tsSdbMgmt.tableList[pTable->id] = pTable;
-  return pTable;
+
+  return taosAddRef(tsSdbRid, pTable);
 }
 
-void sdbCloseTable(void *handle) {
+void sdbCloseTable(int64_t rid) {
+  taosRemoveRef(tsSdbRid, rid);
+}
+
+void *sdbGetTableByRid(int64_t rid) {
+  void *handle = taosAcquireRef(tsSdbRid, rid);
+  taosReleaseRef(tsSdbRid, rid);
+  return handle;
+}
+
+static void sdbCloseTableObj(void *handle) {
   SSdbTable *pTable = (SSdbTable *)handle;
   if (pTable == NULL) return;
   
   tsSdbMgmt.numOfTables--;
   tsSdbMgmt.tableList[pTable->id] = NULL;
 
-  SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle);
+  void *pIter = taosHashIterate(pTable->iHandle, NULL);
-  while (taosHashIterNext(pIter)) {
+  while (pIter) {
-    void **ppRow = taosHashIterGet(pIter);
+    void **ppRow = pIter;
+    pIter = taosHashIterate(pTable->iHandle, pIter);
     if (ppRow == NULL) continue;
 
     SSdbRow row = {
@@ -859,8 +891,9 @@ void sdbCloseTable(void *handle) {
     (*pTable->fpDestroy)(&row);
   }
 
-  taosHashDestroyIter(pIter);
+  taosHashCancelIterate(pTable->iHandle, pIter);
   taosHashCleanup(pTable->iHandle);
+  pTable->iHandle = NULL;
   pthread_mutex_destroy(&pTable->mutex);
 
   sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->name, tsSdbMgmt.numOfTables);

src/mnode/src/mnodeShow.c

@@ -52,11 +52,11 @@ static bool  mnodeCheckShowFinished(SShowObj *pShow);
 static void *mnodePutShowObj(SShowObj *pShow);
 static void  mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove);
 
-extern void *tsMnodeTmr;
 static void *tsMnodeShowCache = NULL;
 static int32_t tsShowObjIndex = 0;
 static SShowMetaFp     tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
+static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0};
 
 int32_t mnodeInitShow() {
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg);
@@ -85,6 +85,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) {
   tsMnodeShowRetrieveFp[msgType] = fp;
 }
 
+void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) {
+  tsMnodeShowFreeIterFp[msgType] = fp;
+}
+
 static char *mnodeGetShowType(int32_t showType) {
   switch (showType) {
     case TSDB_MGMT_TABLE_ACCT:    return "show accounts";
@@ -412,7 +416,9 @@ static void* mnodePutShowObj(SShowObj *pShow) {
 
 static void mnodeFreeShowObj(void *data) {
   SShowObj *pShow = *(SShowObj **)data;
-  sdbFreeIter(pShow->pIter);
+  if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) {
+    (*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter);
+  }
 
   mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
   tfree(pShow);

src/mnode/src/mnodeTable.c

@@ -49,7 +49,9 @@
 #define CREATE_CTABLE_RETRY_TIMES 10
 #define CREATE_CTABLE_RETRY_SEC   14
 
+int64_t        tsCTableRid = -1;
 static void *  tsChildTableSdb;
+int64_t        tsSTableRid = -1;
 static void *  tsSuperTableSdb;
 static int32_t tsChildTableUpdateSize;
 static int32_t tsSuperTableUpdateSize;
@@ -342,8 +344,7 @@ static int32_t mnodeChildTableActionRestored() {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
+  mnodeCancelGetNextChildTable(pIter);
-
   return 0;
 }
 
@@ -351,7 +352,7 @@ static int32_t mnodeInitChildTables() {
   SCTableObj tObj;
   tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
-  SSdbTableDesc tableDesc = {
+  SSdbTableDesc desc = {
     .id           = SDB_TABLE_CTABLE,
     .name         = "ctables",
     .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
@@ -367,7 +368,8 @@ static int32_t mnodeInitChildTables() {
     .fpRestored   = mnodeChildTableActionRestored
   };
 
-  tsChildTableSdb = sdbOpenTable(&tableDesc);
+  tsCTableRid = sdbOpenTable(&desc);
+  tsChildTableSdb = sdbGetTableByRid(tsCTableRid);
   if (tsChildTableSdb == NULL) {
     mError("failed to init child table data");
     return -1;
@@ -378,7 +380,7 @@ static int32_t mnodeInitChildTables() {
 }
 
 static void mnodeCleanupChildTables() {
-  sdbCloseTable(tsChildTableSdb);
+  sdbCloseTable(tsCTableRid);
   tsChildTableSdb = NULL;
 }
 
@@ -544,7 +546,7 @@ static int32_t mnodeInitSuperTables() {
   SSTableObj tObj;
   tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
-  SSdbTableDesc tableDesc = {
+  SSdbTableDesc desc = {
     .id           = SDB_TABLE_STABLE,
     .name         = "stables",
     .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
@@ -560,7 +562,8 @@ static int32_t mnodeInitSuperTables() {
     .fpRestored   = mnodeSuperTableActionRestored
   };
 
-  tsSuperTableSdb = sdbOpenTable(&tableDesc);
+  tsSTableRid = sdbOpenTable(&desc);
+  tsSuperTableSdb = sdbGetTableByRid(tsSTableRid);
   if (tsSuperTableSdb == NULL) {
     mError("failed to init stables data");
     return -1;
@@ -571,7 +574,7 @@ static int32_t mnodeInitSuperTables() {
 }
 
 static void mnodeCleanupSuperTables() {
-  sdbCloseTable(tsSuperTableSdb);
+  sdbCloseTable(tsSTableRid);
   tsSuperTableSdb = NULL;
 }
 
@@ -602,10 +605,13 @@ int32_t mnodeInitTables() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -626,14 +632,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) {
     pIter = mnodeGetNextSuperTable(pIter, &pStable);
     if (pStable == NULL) break;
     if (pStable->uid == uid) {
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextSuperTable(pIter);
       return pStable;
     }
     mnodeDecTableRef(pStable);
   }
 
-  sdbFreeIter(pIter);
-
   return NULL;
 }
 
@@ -655,10 +659,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) {
   return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
 }
 
+void mnodeCancelGetNextChildTable(void *pIter) {
+  sdbFreeIter(tsChildTableSdb, pIter);
+}
+
 void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) {
   return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
 }
 
+void mnodeCancelGetNextSuperTable(void *pIter) {
+  sdbFreeIter(tsSuperTableSdb, pIter);
+}
+
 void mnodeIncTableRef(void *p1) {
   STableObj *pTable = (STableObj *)p1;
   if (pTable->type == TSDB_SUPER_TABLE) {
@@ -914,10 +926,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
 
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
    if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
-    SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
+    int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
-    while (taosHashIterNext(pIter)) {
+    while (pVgId) {
-      int32_t *pVgId = taosHashIterGet(pIter);
       SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
+      pVgId = taosHashIterate(pStable->vgHash, pVgId);
       if (pVgroup == NULL) break;
 
       SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
@@ -933,7 +945,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
       dnodeSendMsgToDnode(&epSet, &rpcMsg);
       mnodeDecVgroupRef(pVgroup);
     }
-    taosHashDestroyIter(pIter);
+
+    taosHashCancelIterate(pStable->vgHash, pVgId);
 
     mnodeDropAllChildTablesInStable(pStable);
   } 
@@ -1430,8 +1443,6 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -1523,11 +1534,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
     } else {
       SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
 
-      SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
+      int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL);
-      int32_t               vgSize = 0;
+      int32_t  vgSize = 0;
-      while (taosHashIterNext(pIter)) {
+      while (pVgId) {
-        int32_t *pVgId = taosHashIterGet(pIter);
+        SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
-        SVgObj * pVgroup = mnodeGetVgroup(*pVgId);
+        pVgId = taosHashIterate(pTable->vgHash, pVgId);
         if (pVgroup == NULL) continue;
 
         pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
@@ -1547,7 +1558,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
         mnodeDecVgroupRef(pVgroup);
       }
 
-      taosHashDestroyIter(pIter);
+      taosHashCancelIterate(pTable->vgHash, pVgId);
       mnodeDecTableRef(pTable);
 
       pVgroupMsg->numOfVgroups = htonl(vgSize);
@@ -2230,8 +2241,6 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
 }
 
@@ -2263,8 +2272,6 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -2293,8 +2300,6 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
 }
 

src/mnode/src/mnodeUser.c

@@ -33,6 +33,7 @@
 #include "mnodeWrite.h"
 #include "mnodePeer.h"
 
+int64_t        tsUserRid = -1;
 static void *  tsUserSdb = NULL;
 static int32_t tsUserUpdateSize = 0;
 static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -123,7 +124,6 @@ static void mnodePrintUserAuth() {
   }
 
   fflush(fp);
-  sdbFreeIter(pIter);
   fclose(fp);
 }
 
@@ -166,7 +166,8 @@ int32_t mnodeInitUsers() {
     .fpRestored   = mnodeUserActionRestored
   };
 
-  tsUserSdb = sdbOpenTable(&desc);
+  tsUserRid = sdbOpenTable(&desc);
+  tsUserSdb = sdbGetTableByRid(tsUserRid);
   if (tsUserSdb == NULL) {
     mError("table:%s, failed to create hash", desc.name);
     return -1;
@@ -177,6 +178,8 @@ int32_t mnodeInitUsers() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser);
+
   mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg);
    
   mDebug("table:%s, hash is created", desc.name);
@@ -184,7 +187,7 @@ int32_t mnodeInitUsers() {
 }
 
 void mnodeCleanupUsers() {
-  sdbCloseTable(tsUserSdb);
+  sdbCloseTable(tsUserRid);
   tsUserSdb = NULL;
 }
 
@@ -196,6 +199,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) {
   return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); 
 }
 
+void mnodeCancelGetNextUser(void *pIter) {
+ sdbFreeIter(tsUserSdb, pIter);
+}
+
 void mnodeIncUserRef(SUserObj *pUser) { 
   return sdbIncRef(tsUserSdb, pUser); 
 }
@@ -574,8 +581,6 @@ void mnodeDropAllUsers(SAcctObj *pAcct)  {
     mnodeDecUserRef(pUser);
   }
 
-  sdbFreeIter(pIter);
-
   mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
 }
 

src/mnode/src/mnodeVgroup.c

@@ -20,7 +20,7 @@
 #include "tsocket.h"
 #include "tidpool.h"
 #include "tsync.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "tdataformat.h"
 #include "dnode.h"
@@ -51,6 +51,7 @@ char* vgroupStatus[] = {
   "updating"
 };
 
+int64_t        tsVgroupRid = -1;
 static void   *tsVgroupSdb = NULL;
 static int32_t tsVgUpdateSize = 0;
 
@@ -222,7 +223,8 @@ int32_t mnodeInitVgroups() {
     .fpRestored   = mnodeVgroupActionRestored,
   };
 
-  tsVgroupSdb = sdbOpenTable(&desc);
+  tsVgroupRid = sdbOpenTable(&desc);
+  tsVgroupSdb = sdbGetTableByRid(tsVgroupRid);
   if (tsVgroupSdb == NULL) {
     mError("failed to init vgroups data");
     return -1;
@@ -230,6 +232,7 @@ int32_t mnodeInitVgroups() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp);
@@ -304,7 +307,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
+  mnodeCancelGetNextVgroup(pIter);
 }
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
@@ -491,6 +494,10 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
   return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
+void mnodeCancelGetNextVgroup(void *pIter) {
+  sdbFreeIter(tsVgroupSdb, pIter);
+}
+
 static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) {
   SVgObj *pVgroup = pMsg->pVgroup;
   SDbObj *pDb = pMsg->pDb;
@@ -556,7 +563,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
   pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
-  int32_t code = balanceAllocVnodes(pVgroup);
+  int32_t code = bnAllocVnodes(pVgroup);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, no enough dnode to alloc %d vnodes to vgroup, reason:%s", pDb->name, pVgroup->numOfVnodes,
            tstrerror(code));
@@ -605,7 +612,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
 }
 
 void mnodeCleanupVgroups() {
-  sdbCloseTable(tsVgroupSdb);
+  sdbCloseTable(tsVgroupRid);
   tsVgroupSdb = NULL;
 }
 
@@ -1095,8 +1102,6 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups);
 }
 
@@ -1118,8 +1123,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name);
 }
 #endif
@@ -1147,8 +1150,6 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) {
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
 }
 
@@ -1170,7 +1171,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
     numOfVgroups++;
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
 }

src/mnode/src/mnodeWrite.c

@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "tglobal.h"
 #include "trpc.h"
@@ -54,18 +54,8 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("msg:%p, app:%p type:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
+    mDebug("msg:%p, app:%p type:%s in write queue, is redirected, numOfEps:%d inUse:%d", pMsg,
-           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
+           pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
-    for (int32_t i = 0; i < epSet->numOfEps; ++i) {
-      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
-        epSet->inUse = (i + 1) % epSet->numOfEps;
-        mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d, set inUse to %d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
-               htons(epSet->port[i]), epSet->inUse);
-      } else {
-        mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
-               htons(epSet->port[i]));
-      }
-    }
 
     return TSDB_CODE_RPC_REDIRECT;
   }

src/os/inc/osFile.h

@@ -20,17 +20,6 @@
 extern "C" {
 #endif
 
-#define tread(fd, buf, count) read(fd, buf, count)
-#define twrite(fd, buf, count) write(fd, buf, count)
-#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
-#define tclose(fd)    \
-  {                       \
-    if (FD_VALID(fd)) {    \
-      close(fd);           \
-      fd = FD_INITIALIZER; \
-    }                     \
-  }
-
 int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
@@ -39,7 +28,13 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
 #define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
-#define taosClose(x) tclose(x)
+#define taosClose(fd)      \
+  {                        \
+    if (FD_VALID(fd)) {    \
+      close(fd);           \
+      fd = FD_INITIALIZER; \
+    }                      \
+  }
 
 // TAOS_OS_FUNC_FILE_SENDIFLE
 int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);

src/os/src/detail/osFile.c

@@ -116,7 +116,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
 }
 
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
-  return (int64_t)tlseek(fd, (long)offset, whence);
+  return (int64_t)lseek(fd, (long)offset, whence);
 }
 
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE

src/plugins/monitor/src/monMain.c

@@ -27,12 +27,12 @@
 #include "monitor.h"
 #include "taoserror.h"
 
-#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
+#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
+#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define mnWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
+#define monWarn(...)  { if (monDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define mnInfo(...)  { if (monitorDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
+#define monInfo(...)  { if (monDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
-#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
 
 #define SQL_LENGTH     1030
 #define LOG_LEN_STR    100
@@ -48,12 +48,12 @@ typedef enum {
   MON_CMD_CREATE_TB_ACCT_ROOT,
   MON_CMD_CREATE_TB_SLOWQUERY,
   MON_CMD_MAX
-} EMonitorCommand;
+} EMonCmd;
 
 typedef enum {
   MON_STATE_NOT_INIT,
   MON_STATE_INITED
-} EMonitorState;
+} EMonState;
 
 typedef struct {
   pthread_t thread;
@@ -64,17 +64,17 @@ typedef struct {
   int8_t    start;   // enable/disable by mnode
   int8_t    quiting; // taosd is quiting 
   char      sql[SQL_LENGTH + 1];
-} SMonitorConn;
+} SMonConn;
 
-static SMonitorConn tsMonitor = {0};
+static SMonConn tsMonitor = {0};
-static void  monitorSaveSystemInfo();
+static void  monSaveSystemInfo();
-static void *monitorThreadFunc(void *param);
+static void *monThreadFunc(void *param);
-static void  monitorBuildMonitorSql(char *sql, int32_t cmd);
+static void  monBuildMonitorSql(char *sql, int32_t cmd);
-extern int32_t (*monitorStartSystemFp)();
+extern int32_t (*monStartSystemFp)();
-extern void    (*monitorStopSystemFp)();
+extern void    (*monStopSystemFp)();
-extern void    (*monitorExecuteSQLFp)(char *sql);
+extern void    (*monExecuteSQLFp)(char *sql);
 
-int32_t monitorInitSystem() {
+int32_t monInitSystem() {
   if (tsMonitor.ep[0] == 0) {
     strcpy(tsMonitor.ep, tsLocalEp);
   }
@@ -90,29 +90,29 @@ int32_t monitorInitSystem() {
   pthread_attr_init(&thAttr);
   pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-  if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
+  if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) {
-    mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+    monError("failed to create thread to for monitor module, reason:%s", strerror(errno));
     return -1;
   }
 
   pthread_attr_destroy(&thAttr);
-  mnDebug("monitor thread is launched");
+  monDebug("monitor thread is launched");
 
-  monitorStartSystemFp = monitorStartSystem;
+  monStartSystemFp = monStartSystem;
-  monitorStopSystemFp = monitorStopSystem;
+  monStopSystemFp = monStopSystem;
   return 0;
 }
 
-int32_t monitorStartSystem() {
+int32_t monStartSystem() {
   taos_init();
   tsMonitor.start = 1;
-  monitorExecuteSQLFp = monitorExecuteSQL;
+  monExecuteSQLFp = monExecuteSQL;
-  mnInfo("monitor module start");
+  monInfo("monitor module start");
   return 0;
 }
 
-static void *monitorThreadFunc(void *param) {
+static void *monThreadFunc(void *param) {
-  mnDebug("starting to initialize monitor module ...");
+  monDebug("starting to initialize monitor module ...");
 
   while (1) {
     static int32_t accessTimes = 0;
@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
 
     if (tsMonitor.quiting) {
       tsMonitor.state = MON_STATE_NOT_INIT;
-      mnInfo("monitor thread will quit, for taosd is quiting");
+      monInfo("monitor thread will quit, for taosd is quiting");
       break;
     } else {
       taosGetDisk();
@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
     }
     
     if (dnodeGetDnodeId() <= 0) {
-      mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
+      monDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
       continue;
     }
 
@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
       tsMonitor.state = MON_STATE_NOT_INIT;
       tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
       if (tsMonitor.conn == NULL) {
-        mnError("failed to connect to database, reason:%s", tstrerror(terrno));
+        monError("failed to connect to database, reason:%s", tstrerror(terrno));
         continue;
       } else {
-        mnDebug("connect to database success");
+        monDebug("connect to database success");
       }
     }
 
@@ -151,16 +151,16 @@ static void *monitorThreadFunc(void *param) {
       int code = 0;
 
       for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) {
-        monitorBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
+        monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
         void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
         code = taos_errno(res);
         taos_free_result(res);
 
         if (code != 0) {
-          mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
+          monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
           break;
         } else {
-          mnDebug("successfully to exec sql:%s", tsMonitor.sql);
+          monDebug("successfully to exec sql:%s", tsMonitor.sql);
         }
       }
 
@@ -171,16 +171,16 @@ static void *monitorThreadFunc(void *param) {
 
     if (tsMonitor.state == MON_STATE_INITED) {
       if (accessTimes % tsMonitorInterval == 0) {
-        monitorSaveSystemInfo();
+        monSaveSystemInfo();
       }
     }
   }
 
-  mnInfo("monitor thread is stopped");
+  monInfo("monitor thread is stopped");
   return NULL;
 }
 
-static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
+static void monBuildMonitorSql(char *sql, int32_t cmd) {
   memset(sql, 0, SQL_LENGTH);
 
   if (cmd == MON_CMD_CREATE_DB) {
@@ -236,47 +236,47 @@ static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
   sql[SQL_LENGTH] = 0;
 }
 
-void monitorStopSystem() {
+void monStopSystem() {
   tsMonitor.start = 0;
   tsMonitor.state = MON_STATE_NOT_INIT;
-  monitorExecuteSQLFp = NULL;
+  monExecuteSQLFp = NULL;
-  mnInfo("monitor module stopped");
+  monInfo("monitor module stopped");
 }
 
-void monitorCleanUpSystem() {
+void monCleanupSystem() {
   tsMonitor.quiting = 1;
-  monitorStopSystem();
+  monStopSystem();
   pthread_join(tsMonitor.thread, NULL);
   if (tsMonitor.conn != NULL) {
     taos_close(tsMonitor.conn);
     tsMonitor.conn = NULL;
   }
-  mnInfo("monitor module is cleaned up");
+  monInfo("monitor module is cleaned up");
 }
 
 // unit is MB
-static int32_t monitorBuildMemorySql(char *sql) {
+static int32_t monBuildMemorySql(char *sql) {
   float sysMemoryUsedMB = 0;
   bool  suc = taosGetSysMemory(&sysMemoryUsedMB);
   if (!suc) {
-    mnDebug("failed to get sys memory info");
+    monDebug("failed to get sys memory info");
   }
 
   float procMemoryUsedMB = 0;
   suc = taosGetProcMemory(&procMemoryUsedMB);
   if (!suc) {
-    mnDebug("failed to get proc memory info");
+    monDebug("failed to get proc memory info");
   }
 
   return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
 }
 
 // unit is %
-static int32_t monitorBuildCpuSql(char *sql) {
+static int32_t monBuildCpuSql(char *sql) {
   float sysCpuUsage = 0, procCpuUsage = 0;
   bool  suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
   if (!suc) {
-    mnDebug("failed to get cpu usage");
+    monDebug("failed to get cpu usage");
   }
 
   if (sysCpuUsage <= procCpuUsage) {
@@ -287,72 +287,72 @@ static int32_t monitorBuildCpuSql(char *sql) {
 }
 
 // unit is GB
-static int32_t monitorBuildDiskSql(char *sql) {
+static int32_t monBuildDiskSql(char *sql) {
   return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB);
 }
 
 // unit is Kb
-static int32_t monitorBuildBandSql(char *sql) {
+static int32_t monBuildBandSql(char *sql) {
   float bandSpeedKb = 0;
   bool  suc = taosGetBandSpeed(&bandSpeedKb);
   if (!suc) {
-    mnDebug("failed to get bandwidth speed");
+    monDebug("failed to get bandwidth speed");
   }
 
   return sprintf(sql, ", %f", bandSpeedKb);
 }
 
-static int32_t monitorBuildReqSql(char *sql) {
+static int32_t monBuildReqSql(char *sql) {
   SStatisInfo info = dnodeGetStatisInfo();
   return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
 }
 
-static int32_t monitorBuildIoSql(char *sql) {
+static int32_t monBuildIoSql(char *sql) {
   float readKB = 0, writeKB = 0;
   bool  suc = taosGetProcIO(&readKB, &writeKB);
   if (!suc) {
-    mnDebug("failed to get io info");
+    monDebug("failed to get io info");
   }
 
   return sprintf(sql, ", %f, %f", readKB, writeKB);
 }
 
-static void monitorSaveSystemInfo() {
+static void monSaveSystemInfo() {
   int64_t ts = taosGetTimestampUs();
   char *  sql = tsMonitor.sql;
   int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts);
 
-  pos += monitorBuildCpuSql(sql + pos);
+  pos += monBuildCpuSql(sql + pos);
-  pos += monitorBuildMemorySql(sql + pos);
+  pos += monBuildMemorySql(sql + pos);
-  pos += monitorBuildDiskSql(sql + pos);
+  pos += monBuildDiskSql(sql + pos);
-  pos += monitorBuildBandSql(sql + pos);
+  pos += monBuildBandSql(sql + pos);
-  pos += monitorBuildIoSql(sql + pos);
+  pos += monBuildIoSql(sql + pos);
-  pos += monitorBuildReqSql(sql + pos);
+  pos += monBuildReqSql(sql + pos);
 
   void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
   int   code = taos_errno(res);
   taos_free_result(res);
 
   if (code != 0) {
-    mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
+    monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
   } else {
-    mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
+    monDebug("successfully to save system info, sql:%s", tsMonitor.sql);
   }
 }
 
-static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
+static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
   int32_t c = taos_errno(result);
   if (c != TSDB_CODE_SUCCESS) {
-    mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
+    monError("save %s failed, reason:%s", (char *)param, tstrerror(c));
   } else {
     int32_t rows = taos_affected_rows(result);
-    mnDebug("save %s succ, rows:%d", (char *)param, rows);
+    monDebug("save %s succ, rows:%d", (char *)param, rows);
   }
 
   taos_free_result(result);
 }
 
-void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
+void monSaveAcctLog(SAcctMonitorObj *pMon) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
   char sql[1024] = {0};
@@ -382,11 +382,11 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
           pMon->totalConns, pMon->maxConns,
           pMon->accessState);
 
-  mnDebug("save account info, sql:%s", sql);
+  monDebug("save account info, sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info");
 }
 
-void monitorSaveLog(int32_t level, const char *const format, ...) {
+void monSaveLog(int32_t level, const char *const format, ...) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
   va_list argpointer;
@@ -403,13 +403,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
   len += sprintf(sql + len, "', '%s')", tsLocalEp);
   sql[len++] = 0;
 
-  mnDebug("save log, sql: %s", sql);
+  monDebug("save log, sql: %s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log");
 }
 
-void monitorExecuteSQL(char *sql) {
+void monExecuteSQL(char *sql) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
-  mnDebug("execute sql:%s", sql);
+  monDebug("execute sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql");
 }

src/query/inc/qExecutor.h

@@ -63,9 +63,11 @@ typedef struct SSqlGroupbyExpr {
 
 typedef struct SResultRow {
   int32_t       pageId;      // pageId & rowId is the position of current result in disk-based output buffer
-  int32_t       rowId:15;
+  int32_t       rowId:29;    // row index in buffer page
-  bool          closed:1;    // this result status: closed or opened
+  bool          startInterp; // the time window start timestamp has done the interpolation already.
-  uint16_t      numOfRows;   // number of rows of current time window
+  bool          endInterp;   // the time window end timestamp has done the interpolation already.
+  bool          closed;      // this result status: closed or opened
+  uint32_t      numOfRows;   // number of rows of current time window
   SResultRowCellInfo*  pCellInfo;  // For each result column, there is a resultInfo
   union {STimeWindow win; char* key;};  // start key of current time window
 } SResultRow;
@@ -81,16 +83,15 @@ typedef struct SResultRec {
   int32_t threshold;  // result size threshold in rows.
 } SResultRec;
 
-typedef struct SWindowResInfo {
+typedef struct SResultRowInfo {
-  SResultRow**   pResult;    // result list
+  SResultRow** pResult;    // result list
-  int16_t        type:8;     // data type for hash key
+  int16_t      type:8;     // data type for hash key
-  int32_t        size:24;    // number of result set
+  int32_t      size:24;    // number of result set
-  int32_t        threshold;  // threshold to halt query and return the generated results.
+  int32_t      capacity;   // max capacity
-  int32_t        capacity;   // max capacity
+  int32_t      curIndex;   // current start active index
-  int32_t        curIndex;   // current start active index
+  int64_t      startTime;  // start time of the first time window for sliding query
-  int64_t        startTime;  // start time of the first time window for sliding query
+  int64_t      prevSKey;   // previous (not completed) sliding window start key
-  int64_t        prevSKey;   // previous (not completed) sliding window start key
+} SResultRowInfo;
-} SWindowResInfo;
 
 typedef struct SColumnFilterElem {
   int16_t           bytes;  // column length
@@ -113,7 +114,7 @@ typedef struct STableQueryInfo {
   STimeWindow win;
   STSCursor   cur;
   void*       pTable;         // for retrieve the page id list
-  SWindowResInfo windowResInfo;
+  SResultRowInfo windowResInfo;
 } STableQueryInfo;
 
 typedef struct SQueryCostInfo {
@@ -177,7 +178,7 @@ typedef struct SQueryRuntimeEnv {
   uint16_t*            offset;
   uint16_t             scanFlag;         // denotes reversed scan of data or not
   SFillInfo*           pFillInfo;
-  SWindowResInfo       windowResInfo;
+  SResultRowInfo       windowResInfo;
   STSBuf*              pTSBuf;
   STSCursor            cur;
   SQueryCostInfo       summary;
@@ -187,6 +188,8 @@ typedef struct SQueryRuntimeEnv {
   bool                 topBotQuery;      // false
   bool                 groupbyNormalCol; // denote if this is a groupby normal column query
   bool                 hasTagResults;    // if there are tag values in final result or not
+  bool                 timeWindowInterpo;// if the time window start/end required interpolation
+  bool                 queryWindowIdentical; // all query time windows are identical for all tables in one group
   int32_t              interBufSize;     // intermediate buffer sizse
   int32_t              prevGroupId;      // previous executed group id
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
@@ -195,6 +198,8 @@ typedef struct SQueryRuntimeEnv {
   SResultRowPool*      pool;             // window result object pool
 
   int32_t*             rowCellInfoOffset;// offset value for each row result cell info
+  char**               prevRow;
+  char**               nextRow;
 } SQueryRuntimeEnv;
 
 enum {
@@ -212,7 +217,8 @@ typedef struct SQInfo {
   STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
   STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
   SQueryRuntimeEnv runtimeEnv;
-  SArray*          arrTableIdInfo;
+//  SArray*          arrTableIdInfo;
+  SHashObj*        arrTableIdInfo;
   int32_t          groupIndex;
 
   /*

src/query/inc/qUtil.h

@@ -30,19 +30,19 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ
 void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type);
 SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
 
-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);
+int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type);
 
-void    cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
+void    cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo);
-void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
+void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo);
 void    clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
 
 void    clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
-int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
+int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo);
-void    closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
+void    closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot);
-void    closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
+void    closeAllTimeWindow(SResultRowInfo* pWindowResInfo);
-void    removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
+void    removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
 
-static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) {
   assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
   return pWindowResInfo->pResult[slot];
 }
@@ -50,7 +50,7 @@ static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int
 #define curTimeWindowIndex(_winres)        ((_winres)->curIndex)
 #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
 
-bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
+bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot);
 
 int32_t initResultRow(SResultRow *pResultRow);
 

src/query/inc/tsqlfunction.h

@@ -152,6 +152,11 @@ typedef struct SResultRowCellInfo {
   uint32_t numOfRes;        // num of output result in current buffer
 } SResultRowCellInfo;
 
+typedef struct SPoint1 {
+  int64_t key;
+  double  val;
+} SPoint1;
+
 #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
 
 struct SQLFunctionCtx;
@@ -194,6 +199,8 @@ typedef struct SQLFunctionCtx {
   SResultRowCellInfo *resultInfo;
 
   SExtTagsInfo tagInfo;
+  SPoint1      start;
+  SPoint1      end;
 } SQLFunctionCtx;
 
 typedef struct SQLAggFuncElem {
@@ -243,21 +250,11 @@ enum {
 };
 
 typedef struct STwaInfo {
-  TSKEY   lastKey;
+  TSKEY       lastKey;
-  int8_t  hasResult;  // flag to denote has value
+  int8_t      hasResult;  // flag to denote has value
-  int16_t type;       // source data type
+  double      dOutput;
-  TSKEY   SKey;
+  double      lastValue;
-  TSKEY   EKey;
+  STimeWindow win;
-
-  union {
-    double  dOutput;
-    int64_t iOutput;
-  };
-
-  union {
-    double  dLastValue;
-    int64_t iLastValue;
-  };
 } STwaInfo;
 
 /* global sql function array */
@@ -276,8 +273,6 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
     (_r)->initialized = false; \
   } while (0)
 
-//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf);
-
 static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
   pResInfo->initialized = true;  // the this struct has been initialized flag
   

src/query/src/qAst.c

@@ -370,6 +370,66 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
 #endif
 }
 
+static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {
+  switch(type) {
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t* p = (int8_t*) dest;
+      int8_t* pSrc = (int8_t*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t* p = (int16_t*) dest;
+      int16_t* pSrc = (int16_t*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t* p = (int32_t*) dest;
+      int32_t* pSrc = (int32_t*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t* p = (int64_t*) dest;
+      int64_t* pSrc = (int64_t*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float* p = (float*) dest;
+      float* pSrc = (float*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double* p = (double*) dest;
+      double* pSrc = (double*) src;
+
+      for(int32_t i = 0; i < numOfRows; ++i) {
+        p[i] = pSrc[numOfRows - i - 1];
+      }
+      break;
+    }
+    default: assert(0);
+  }
+}
+
 void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
                                 char *(*getSourceDataBlock)(void *, const char*, int32_t)) {
   if (pExprs == NULL) {
@@ -387,6 +447,8 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
 
   /* the right output has result from the right child syntax tree */
   char *pRightOutput = malloc(sizeof(int64_t) * numOfRows);
+  char *pdata = malloc(sizeof(int64_t) * numOfRows);
+
   if (pRight->nodeType == TSQL_NODE_EXPR) {
     tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock);
   }
@@ -398,52 +460,75 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
        * the type of returned value of one expression is always double float precious
        */
       _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
-      fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, order);
+      fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
 
     } else if (pRight->nodeType == TSQL_NODE_COL) {  // exprLeft + columnRight
       _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr);
+
       // set input buffer
       char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
-      fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, order);
+      if (order == TSDB_ORDER_DESC) {
+        reverseCopy(pdata, pInputData, pRight->pSchema->type, numOfRows);
+        fp(pLeftOutput, pdata, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
+      } else {
+        fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
+      }
 
     } else if (pRight->nodeType == TSQL_NODE_VALUE) {  // exprLeft + 12
       _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr);
-      fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order);
+      fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
     }
   } else if (pLeft->nodeType == TSQL_NODE_COL) {
     // column data specified on left-hand-side
     char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId);
     if (pRight->nodeType == TSQL_NODE_EXPR) {  // columnLeft + expr2
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
-      fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, order);
+
+      if (order == TSDB_ORDER_DESC) {
+        reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows);
+        fp(pdata, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
+      } else {
+        fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
+      }
 
     } else if (pRight->nodeType == TSQL_NODE_COL) {  // columnLeft + columnRight
       // column data specified on right-hand-side
       char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
-
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr);
-      fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order);
 
+      // both columns are descending order, do not reverse the source data
+      fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order);
     } else if (pRight->nodeType == TSQL_NODE_VALUE) {  // columnLeft + 12
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr);
-      fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order);
+
+      if (order == TSDB_ORDER_DESC) {
+        reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows);
+        fp(pdata, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
+      } else {
+        fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
+      }
     }
   } else {
     // column data specified on left-hand-side
     if (pRight->nodeType == TSQL_NODE_EXPR) {  // 12 + expr2
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
-      fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, order);
+      fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC);
 
     } else if (pRight->nodeType == TSQL_NODE_COL) {  // 12 + columnRight
       // column data specified on right-hand-side
       char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
-      
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr);
-      fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order);
+
+      if (order == TSDB_ORDER_DESC) {
+        reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows);
+        fp(&pLeft->pVal->i64Key, pdata, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
+      } else {
+        fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, TSDB_ORDER_ASC);
+      }
 
     } else if (pRight->nodeType == TSQL_NODE_VALUE) {  // 12 + 12
       _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr);
-      fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, order);
+      fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC);
     }
   }
 

src/query/src/qParserImpl.c

@@ -405,14 +405,29 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+          int32_t bytes = -(int32_t)(type->type);
+          if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+            // we have to postpone reporting the error because it cannot be done here
+            // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow
+            bytes = TSDB_MAX_NCHAR_LEN + 1;
+          } else {
+            bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
+          }
+          pField->bytes = (int16_t)bytes;
         }
       } else if (i == TSDB_DATA_TYPE_BINARY) {
         /* for binary, the TOKENTYPE is the length of binary */
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
+          int32_t bytes = -(int32_t)(type->type);
+          if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
+            // refer comment for NCHAR above
+            bytes = TSDB_MAX_BINARY_LEN + 1;
+          } else {
+            bytes += VARSTR_HEADER_SIZE;
+          }
+          pField->bytes = (int16_t)bytes;
         } 
       }
       break;

src/query/src/qResultbuf.c

@@ -423,9 +423,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   unlink(pResultBuf->path);
   tfree(pResultBuf->path);
 
-  SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
+  SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
-  while(taosHashIterNext(iter)) {
+  while(p) {
-    SArray** p = (SArray**) taosHashIterGet(iter);
     size_t n = taosArrayGetSize(*p);
     for(int32_t i = 0; i < n; ++i) {
       SPageInfo* pi = taosArrayGetP(*p, i);
@@ -434,10 +433,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
     }
 
     taosArrayDestroy(*p);
+    p = taosHashIterate(pResultBuf->groupSet, p);
   }
 
-  taosHashDestroyIter(iter);
-
   tdListFree(pResultBuf->lruList);
   taosArrayDestroy(pResultBuf->emptyDummyIdList);
   taosHashCleanup(pResultBuf->groupSet);

src/query/src/qUtil.c

@@ -43,51 +43,48 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   return size;
 }
 
-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
+int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) {
-  pWindowResInfo->capacity = size;
+  pResultRowInfo->capacity = size;
-  pWindowResInfo->threshold = threshold;
-  
-  pWindowResInfo->type = type;
-  pWindowResInfo->curIndex = -1;
-  pWindowResInfo->size     = 0;
-  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
 
-  pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
+  pResultRowInfo->type = type;
-  if (pWindowResInfo->pResult == NULL) {
+  pResultRowInfo->curIndex = -1;
+  pResultRowInfo->size     = 0;
+  pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
+
+  pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES);
+  if (pResultRowInfo->pResult == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
+void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) {
-  if (pWindowResInfo == NULL) {
+  if (pResultRowInfo == NULL) {
     return;
   }
-  if (pWindowResInfo->capacity == 0) {
+  if (pResultRowInfo->capacity == 0) {
-    assert(pWindowResInfo->pResult == NULL);
+    assert(pResultRowInfo->pResult == NULL);
     return;
   }
 
-  if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) {
+  if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) {
-    for(int32_t i = 0; i < pWindowResInfo->size; ++i) {
+    for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
-      tfree(pWindowResInfo->pResult[i]->key);
+      tfree(pResultRowInfo->pResult[i]->key);
     }
   }
   
-  tfree(pWindowResInfo->pResult);
+  tfree(pResultRowInfo->pResult);
 }
 
-void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
+void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) {
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) {
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) {
     return;
   }
 
-//  assert(pWindowResInfo->size == 1);
+  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-
+    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
-  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
+    clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
-    SResultRow *pWindowRes = pWindowResInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -96,30 +93,30 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
     taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex)));
   }
   
-  pWindowResInfo->curIndex = -1;
+  pResultRowInfo->curIndex = -1;
-  pWindowResInfo->size = 0;
+  pResultRowInfo->size = 0;
   
-  pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
+  pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
-  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
+  pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
 
 void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) {
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) {
     return;
   }
   
-  int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
+  int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
   assert(num >= 0 && num <= numOfClosed);
 
-  int16_t type = pWindowResInfo->type;
+  int16_t type = pResultRowInfo->type;
   int64_t uid = getResultInfoUId(pRuntimeEnv);
 
   char    *key  = NULL;
   int16_t  bytes = -1;
 
   for (int32_t i = 0; i < num; ++i) {
-    SResultRow *pResult = pWindowResInfo->pResult[i];
+    SResultRow *pResult = pResultRowInfo->pResult[i];
     if (pResult->closed) {  // remove the window slot from hash table
       getResultRowKeyInfo(pResult, type, &key, &bytes);
       SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
@@ -129,23 +126,23 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     }
   }
   
-  int32_t remain = pWindowResInfo->size - num;
+  int32_t remain = pResultRowInfo->size - num;
   
   // clear all the closed windows from the window list
   for (int32_t k = 0; k < remain; ++k) {
-    copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type);
+    copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type);
   }
   
   // move the unclosed window in the front of the window list
-  for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
+  for (int32_t k = remain; k < pResultRowInfo->size; ++k) {
-    SResultRow *pWindowRes = pWindowResInfo->pResult[k];
+    SResultRow *pWindowRes = pResultRowInfo->pResult[k];
-    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
+    clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
   }
   
-  pWindowResInfo->size = remain;
+  pResultRowInfo->size = remain;
 
-  for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
+  for (int32_t k = 0; k < pResultRowInfo->size; ++k) {
-    SResultRow *pResult = pWindowResInfo->pResult[k];
+    SResultRow *pResult = pResultRowInfo->pResult[k];
     getResultRowKeyInfo(pResult, type, &key, &bytes);
     SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
 
@@ -153,43 +150,43 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     assert(p != NULL); 
 
     int32_t  v = (*p - num);
-    assert(v >= 0 && v <= pWindowResInfo->size);
+    assert(v >= 0 && v <= pResultRowInfo->size);
 
     SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
     taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
   }
   
-  pWindowResInfo->curIndex = -1;
+  pResultRowInfo->curIndex = -1;
 }
 
 void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) {
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) {
     return;
   }
   
-  int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
+  int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
   clearFirstNWindowRes(pRuntimeEnv, numOfClosed);
 }
 
-int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
+int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) {
   int32_t i = 0;
-  while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
+  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
     ++i;
   }
   
   return i;
 }
 
-void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
+void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) {
-  assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
+  assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
-  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
+  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    if (pWindowResInfo->pResult[i]->closed) {
+    if (pResultRowInfo->pResult[i]->closed) {
       continue;
     }
     
-    pWindowResInfo->pResult[i]->closed = true;
+    pResultRowInfo->pResult[i]->closed = true;
   }
 }
 
@@ -198,41 +195,41 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
  * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time.
  * NOTE: remove redundant, only when the result set order equals to traverse order
  */
-void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) {
+void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) {
-  assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
+  assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
-  if (pWindowResInfo->size <= 1) {
+  if (pResultRowInfo->size <= 1) {
     return;
   }
 
   // get the result order
-  int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
+  int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1;
   if (order != resultOrder) {
     return;
   }
 
   int32_t i = 0;
   if (order == QUERY_ASC_FORWARD_STEP) {
-    TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
+    TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey;
-    while (i < pWindowResInfo->size && (ekey < lastKey)) {
+    while (i < pResultRowInfo->size && (ekey < lastKey)) {
       ++i;
     }
   } else if (order == QUERY_DESC_FORWARD_STEP) {
-    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
+    while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) {
       ++i;
     }
   }
 
-  if (i < pWindowResInfo->size) {
+  if (i < pResultRowInfo->size) {
-    pWindowResInfo->size = (i + 1);
+    pResultRowInfo->size = (i + 1);
   }
 }
 
-bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
+bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) {
-  return (getResultRow(pWindowResInfo, slot)->closed == true);
+  return (getResultRow(pResultRowInfo, slot)->closed == true);
 }
 
-void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
+void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) {
-  getResultRow(pWindowResInfo, slot)->closed = true;
+  getResultRow(pResultRowInfo, slot)->closed = true;
 }
 
 void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) {
@@ -395,11 +392,10 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
   }
 
   SQuery* pQuery = pRuntimeEnv->pQuery;
-  if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) ||
+  if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) {
-      pRuntimeEnv->groupbyNormalCol) {
     return 0;
   }
 
-  STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current);
+  STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable);
   return id->uid;
 }

src/rpc/src/rpcMain.c

@@ -630,8 +630,16 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   } else {
     // if there is an outgoing message, free it
     if (pConn->outType && pConn->pReqMsg) {
-      if (pConn->pContext) pConn->pContext->pConn = NULL; 
+      SRpcReqContext *pContext = pConn->pContext;
-      taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
+      if (pContext->pRsp) {   
+        // for synchronous API, post semaphore to unblock app
+        pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR;
+        pContext->pRsp->pCont = NULL;
+        pContext->pRsp->contLen = 0;
+        tsem_post(pContext->pSem);
+      }
+      pContext->pConn = NULL; 
+      taosRemoveRef(tsRpcRefId, pContext->rid);
     }
   }
 

src/rpc/src/rpcTcp.c

@@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) {
 
 int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
   SFdObj *pFdObj = chandle;
-
   if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
+  SThreadObj *pThreadObj = pFdObj->pThreadObj;
 
-  return taosWriteMsg(pFdObj->fd, data, len);
+  int ret = taosWriteMsg(pFdObj->fd, data, len);
+  tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); 
+
+  return ret;
 }
 
 static void taosReportBrokenLink(SFdObj *pFdObj) {
@@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
 
   headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
   if (headLen != sizeof(SRpcHead)) {
-    tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
+    tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
     return -1; 
   }
 
@@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   } else {
-    tTrace("TCP malloc mem:%p size:%d", buffer, size);
+    tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer);
   }
 
   msg = buffer + tsRpcOverhead;
@@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pthread_mutex_unlock(&pThreadObj->mutex);
 
-  tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", 
+  tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", 
-          pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
+          pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds);
 
   tfree(pFdObj);
 }

src/sync/inc/syncInt.h

@@ -139,15 +139,14 @@ typedef struct SsyncPeer {
   char     id[TSDB_EP_LEN + 32];  // peer vgId + end point
   uint64_t version;
   uint64_t sversion;        // track the peer version in retrieve process
+  uint64_t lastFileVer;     // track the file version while retrieve
+  uint64_t lastWalVer;      // track the wal version while retrieve
   int32_t  syncFd;
   int32_t  peerFd;          // forward FD
   int32_t  numOfRetrieves;  // number of retrieves tried
   int32_t  fileChanged;     // a flag to indicate file is changed during retrieving process
   void *   timer;
   void *   pConn;
-  int32_t  notifyFd;
-  int32_t  watchNum;
-  int32_t *watchFd;
   int32_t  refCount;  // reference count
   struct   SSyncNode *pSyncNode;
 } SSyncPeer;
@@ -172,6 +171,7 @@ typedef struct SSyncNode {
   FNotifyRole       notifyRole;
   FNotifyFlowCtrl   notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
+  FGetVersion       getVersion;
   pthread_mutex_t   mutex;
 } SSyncNode;
 

src/sync/src/syncMain.c

@@ -196,6 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   pNode->confirmForward = pInfo->confirmForward;
   pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
   pNode->notifyFileSynced = pInfo->notifyFileSynced;
+  pNode->getVersion = pInfo->getVersion;
 
   pNode->selfIndex = -1;
   pNode->vgId = pInfo->vgId;
@@ -497,7 +498,6 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) {
     taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
 
     sDebug("%s, resource is freed", pPeer->id);
-    tfree(pPeer->watchFd);
     tfree(pPeer);
     return 0;
   }
@@ -540,7 +540,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   pPeer->ip = ip;
   pPeer->port = pInfo->nodePort;
   pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
-  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
+  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId);
 
   pPeer->peerFd = -1;
   pPeer->syncFd = -1;
@@ -1143,8 +1143,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
       pPeer->syncFd = connFd;
       syncCreateRestoreDataThread(pPeer);
     } else {
-      sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd,
+      sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
-             pPeer->syncFd);
       syncClosePeerConn(pPeer);
       pPeer->peerFd = connFd;
       pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);

src/sync/src/syncRestore.c

@@ -52,12 +52,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
 
 static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */  // master file info
+  SFileInfo  minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */  // slave file info
+  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileAck   fileAck; 
+  SFileAck   fileAck = {0};
   int32_t    code = -1;
   char       name[TSDB_FILENAME_LEN * 2] = {0};
-  uint32_t   pindex = 0;    // index in last restore
+  uint32_t   pindex = 0;  // index in last restore
   bool       fileChanged = false;
 
   *fversion = 0;
@@ -134,7 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
     // data file is changed, code shall be set to 1
     *fversion = minfo.fversion;
     code = 1;
-    sDebug("%s, file changed while restore file", pPeer->id);
+    sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
   }
 
   if (code < 0) {
@@ -160,7 +160,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
     }
 
     if (pHead->len == 0) {
-      sDebug("%s, wal is synced over", pPeer->id);
+      sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer);
       code = 0;
       break;
     }  // wal sync over

src/sync/src/syncRetrieve.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include <sys/inotify.h>
 #include "os.h"
+#include "taoserror.h"
 #include "tlog.h"
 #include "tutil.h"
 #include "tglobal.h"
@@ -25,110 +26,102 @@
 #include "tsync.h"
 #include "syncInt.h"
 
-static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
+static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
-  sDebug("%s, start to monitor:%s", pPeer->id, name);
+  uint64_t fver, wver;
-
+  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
-  if (pPeer->notifyFd <= 0) {
+  if (code != 0) {
-    pPeer->watchNum = 0;
+    sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
-    pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
-    if (pPeer->notifyFd < 0) {
-      sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno));
-      return -1;
-    }
-
-    if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
-    if (pPeer->watchFd == NULL) {
-      sError("%s, failed to allocate watchFd", pPeer->id);
-      return -1;
-    }
-
-    memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
-  }
-
-  int32_t *wd = pPeer->watchFd + pPeer->watchNum;
-
-  if (*wd >= 0) {
-    if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
-      sError("%s, failed to remove wd:%d since %s", pPeer->id, *wd, strerror(errno));
-      return -1;
-    }
-  }
-
-  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE);
-  if (*wd == -1) {
-    sError("%s, failed to add %s since %s", pPeer->id, name, strerror(errno));
     return -1;
-  } else {
-    sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum);
   }
 
-  pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles;
+  pPeer->lastWalVer = wver;
-
+  return code;
-  return 0;
 }
 
-static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
+static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
-  if (pPeer->notifyFd <= 0) return 0;
+  uint64_t fver, wver;
+  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  if (code != 0) {
+    sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
+    return true;
+  }
 
-  char    buf[2048];
+  if (wver != pPeer->lastWalVer) {
-  int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
+    sDebug("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer);
-  if (len < 0 && errno != EAGAIN) {
+    return true;
-    sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno));
+  }
+
+  return false;
+}
+
+static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
+  uint64_t fver, wver;
+  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  if (code != 0) {
+    sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     return -1;
   }
 
-  int32_t code = 0;
+  pPeer->lastFileVer = fver;
-  if (len > 0) {
+  return code;
-    const struct inotify_event *event;
+}
-    char *ptr;
+
-    for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
+static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
-      event = (const struct inotify_event *)ptr;
+  uint64_t fver, wver;
-      if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) {
+  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
-        sDebug("%s, processed file is changed", pPeer->id);
+  if (code != 0) {
-        pPeer->fileChanged = 1;
+    sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
-        code = 1;
+    pPeer->fileChanged = 1;
-        break;
+    return true;
-      }
-    }
   }
 
-  return code;
+  if (fver != pPeer->lastFileVer) {
+    sDebug("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer);
+    pPeer->fileChanged = 1;
+    return true;
+  }
+
+  pPeer->fileChanged = 0;
+  return false;
 }
 
 static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  fileInfo;
+  SFileInfo  fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
-  SFileAck   fileAck;
+  SFileAck   fileAck = {0};
   int32_t    code = -1;
   char       name[TSDB_FILENAME_LEN * 2] = {0};
 
-  memset(&fileInfo, 0, sizeof(fileInfo));
+  if (syncGetFileVersion(pNode, pPeer) < 0) return -1;
-  memset(&fileAck, 0, sizeof(fileAck));
 
   while (1) {
     // retrieve file info
     fileInfo.name[0] = 0;
+    fileInfo.size = 0;
     fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
                                            &fileInfo.size, &fileInfo.fversion);
     // fileInfo.size = htonl(size);
+    sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
 
     // send the file info
     int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
     if (ret < 0) {
+      code = -1;
       sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
       break;
     }
 
     // if no file anymore, break
     if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
-      sDebug("%s, no more files to sync", pPeer->id);
       code = 0;
+      sDebug("%s, no more files to sync", pPeer->id);
       break;
     }
 
     // wait for the ack from peer
     ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck));
     if (ret < 0) {
+      code = -1;
       sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
       break;
     }
@@ -136,15 +129,6 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
     // set the peer sync version
     pPeer->sversion = fileInfo.fversion;
 
-    // get the full path to file
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
-
-    // add the file into watch list
-    if (syncAddIntoWatchList(pPeer, name) < 0) {
-      sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
     // if sync is not required, continue
     if (fileAck.sync == 0) {
       fileInfo.index++;
@@ -152,9 +136,13 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
       continue;
     }
 
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+
     // send the file to peer
     int32_t sfd = open(name, O_RDONLY);
     if (sfd < 0) {
+      code = -1;
       sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
       break;
     }
@@ -162,138 +150,112 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
     ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
     close(sfd);
     if (ret < 0) {
+      code = -1;
       sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
       break;
     }
 
-    sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size);
+    sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
     fileInfo.index++;
 
     // check if processed files are modified
-    if (syncAreFilesModified(pPeer) != 0) {
+    if (syncAreFilesModified(pNode, pPeer)) {
-      sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
+      code = -1;
       break;
     }
   }
 
-  if (code < 0) {
+  if (code != TSDB_CODE_SUCCESS) {
-    sError("%s, failed to retrieve file", pPeer->id);
+    sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code);
   }
 
   return code;
 }
 
-/* if only a partial record is read out, set the IN_MODIFY flag in event,
+// if only a partial record is read out, upper layer will reload the file to get a complete record
-   so upper layer will reload the file to get a complete record */
+static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) {
-static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) {
+  int32_t ret = read(sfd, pHead, sizeof(SWalHead));
-  int32_t ret;
+  if (ret < 0) {
+    sError("sfd:%d, failed to read wal head since %s, ret:%d", sfd, strerror(errno), ret);
+    return -1;
+  }
 
-  ret = read(sfd, pHead, sizeof(SWalHead));
+  if (ret == 0) {
-  if (ret < 0) return -1;
+    sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret);
-  if (ret == 0) return 0;
+    return 0;
+  }
 
   if (ret != sizeof(SWalHead)) {
     // file is not at end yet, it shall be reloaded
-    *pEvent = *pEvent | IN_MODIFY;
+    sDebug("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret);
     return 0;
   }
 
   assert(pHead->len <= TSDB_MAX_WAL_SIZE);
 
   ret = read(sfd, pHead->cont, pHead->len);
-  if (ret < 0) return -1;
+  if (ret < 0) {
+    sError("sfd:%d, failed to read wal content since %s, ret:%d", sfd, strerror(errno), ret);
+    return -1;
+  }
 
   if (ret != pHead->len) {
     // file is not at end yet, it shall be reloaded
-    *pEvent = *pEvent | IN_MODIFY;
+    sDebug("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret);
     return 0;
   }
 
   return sizeof(SWalHead) + pHead->len;
 }
 
-static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
+static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) {
-  pPeer->watchNum = 0;
+  int32_t sfd = open(name, O_RDONLY);
-  taosClose(pPeer->notifyFd);
-  pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
-  if (pPeer->notifyFd < 0) {
-    sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno));
-    return -1;
-  }
-
-  if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
-  if (pPeer->watchFd == NULL) {
-    sError("%s, failed to allocate watchFd", pPeer->id);
-    return -1;
-  }
-
-  memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
-  int32_t *wd = pPeer->watchFd;
-
-  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
-  if (*wd == -1) {
-    sError("%s, failed to watch last wal since %s", pPeer->id, strerror(errno));
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
-  char    buf[2048];
-  int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
-  if (len < 0 && errno != EAGAIN) {
-    sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno));
-    return -1;
-  }
-
-  if (len == 0) return 0;
-
-  struct inotify_event *event;
-  for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
-    event = (struct inotify_event *)ptr;
-    if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
-    if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
-  }
-
-  if (pEvent != 0) sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
-
-  return 0;
-}
-
-static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
-  SWalHead *pHead = malloc(SYNC_MAX_SIZE);
-  int32_t   code = -1;
-  int32_t   bytes = 0;
-  int32_t   sfd;
-
-  sfd = open(name, O_RDONLY);
   if (sfd < 0) {
-    free(pHead);
+    sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno));
     return -1;
   }
 
-  (void)lseek(sfd, offset, SEEK_SET);
+  int32_t code = taosLSeek(sfd, offset, SEEK_SET);
-  sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion);
+  if (code < 0) {
+    sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno));
+    close(sfd);
+    return -1;
+  }
+
+  sDebug("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion);
+
+  SWalHead *pHead = malloc(SYNC_MAX_SIZE);
+  int32_t   bytes = 0;
 
   while (1) {
-    int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
+    code = syncReadOneWalRecord(sfd, pHead);
-    if (wsize < 0) break;
+    if (code < 0) {
-    if (wsize == 0) {
+      sError("%s, failed to read one record from wal:%s", pPeer->id, name);
-      code = 0;
+      break;
+    }
+
+    if (code == 0) {
+      code = bytes;
+      sDebug("%s, read to the end of wal, bytes:%d", pPeer->id, bytes);
       break;
     }
 
     sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version);
-    int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
-    if (ret != wsize) break;
-    pPeer->sversion = pHead->version;
 
+    int32_t wsize = code;
+    int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+    if (ret != wsize) {
+      code = -1;
+      sError("%s, failed to forward wal since %s, hver:%" PRIu64, pPeer->id, strerror(errno), pHead->version);
+      break;
+    }
+
+    pPeer->sversion = pHead->version;
     bytes += wsize;
 
     if (pHead->version >= fversion && fversion > 0) {
       code = 0;
-      bytes = 0;
+      sDebug("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion);
       break;
     }
   }
@@ -301,92 +263,62 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi
   free(pHead);
   close(sfd);
 
-  if (code == 0) return bytes;
+  return code;
-  return -1;
 }
 
 static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  int32_t    code = -1;
+  int32_t    once = 0;  // last WAL has once ever been processed
-  char       fname[TSDB_FILENAME_LEN * 2];  // full path to wal file
+  int64_t    offset = 0;
+  uint64_t   fversion = 0;
+  char       fname[TSDB_FILENAME_LEN * 2] = {0};  // full path to wal file
 
-  if (syncAreFilesModified(pPeer) != 0) return -1;
+  // get full path to wal file
+  snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+  sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
 
   while (1) {
-    int32_t  once = 0;  // last WAL has once ever been processed
+    if (syncAreFilesModified(pNode, pPeer)) return -1;
-    int64_t  offset = 0;
+    if (syncGetWalVersion(pNode, pPeer) < 0) return -1;
-    uint64_t fversion = 0;
-    uint32_t event = 0;
 
-    // get full path to wal file
+    int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset);
-    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+    if (bytes < 0) {
-    sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
+      sDebug("%s, failed to retrieve last wal", pPeer->id);
-
+      return bytes;
-    // monitor last wal
-    if (syncMonitorLastWal(pPeer, fname) < 0) break;
-
-    while (1) {
-      int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
-      if (bytes < 0) break;
-
-      // check file changes
-      if (syncCheckLastWalChanges(pPeer, &event) < 0) break;
-
-      // if file is not updated or updated once, set the fversion and sstatus
-      if (((event & IN_MODIFY) == 0) || once) {
-        if (fversion == 0) {
-          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
-          sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
-          fversion = nodeVersion;                   // must read data to fversion
-        }
-      }
-
-      // if all data up to fversion is read out, it is over
-      if (pPeer->sversion >= fversion && fversion > 0) {
-        code = 0;
-        sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
-        break;
-      }
-
-      // if all data are read out, and no update
-      if ((bytes == 0) && ((event & IN_MODIFY) == 0)) {
-        // wal file is closed, break
-        if (event & IN_CLOSE_WRITE) {
-          code = 0;
-          sDebug("%s, current wal is closed", pPeer->id);
-          break;
-        }
-
-        // wal not closed, it means some data not flushed to disk, wait for a while
-        usleep(10000);
-      }
-
-      // if bytes>0, file is updated, or fversion is not reached but file still open, read again
-      once = 1;
-      offset += bytes;
-      sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes);
-      event = event & (~IN_MODIFY);  // clear IN_MODIFY flag
     }
 
-    if (code < 0) break;
+    // check file changes
-    if (pPeer->sversion >= fversion && fversion > 0) break;
+    bool walModified = syncIsWalModified(pNode, pPeer);
 
-    index++;
+    // if file is not updated or updated once, set the fversion and sstatus
-    wname[0] = 0;
+    if (!walModified || once) {
-    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
+      if (fversion == 0) {
-    if (code < 0) break;
+        pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
-    if (wname[0] == 0) {
+        fversion = nodeVersion;                   // must read data to fversion
-      code = 0;
+        sDebug("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion);
-      break;
+      }
     }
 
-    // current last wal is closed, there is a new one
+    // if all data up to fversion is read out, it is over
-    sDebug("%s, last wal is closed, try new one", pPeer->id);
+    if (pPeer->sversion >= fversion && fversion > 0) {
+      sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d sver:%" PRIu64, pPeer->id, fversion, bytes,
+             pPeer->sversion);
+      return 0;
+    }
+
+    // if all data are read out, and no update
+    if (bytes == 0 && !walModified) {
+      // wal not closed, it means some data not flushed to disk, wait for a while
+      usleep(10000);
+    }
+
+    // if bytes > 0, file is updated, or fversion is not reached but file still open, read again
+    once = 1;
+    offset += bytes;
+    sDebug("%s, continue retrieve last wal, bytes:%d offset:%" PRId64, pPeer->id, bytes, offset);
   }
 
-  taosClose(pPeer->notifyFd);
+  return -1;
-
-  return code;
 }
 
 static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
@@ -394,7 +326,6 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   char        fname[TSDB_FILENAME_LEN * 3];
   char        wname[TSDB_FILENAME_LEN * 2];
   int32_t     size;
-  struct stat fstat;
   int32_t     code = -1;
   int64_t     index = 0;
 
@@ -402,9 +333,14 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
     // retrieve wal info
     wname[0] = 0;
     code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
-    if (code < 0) break;  // error
+    if (code < 0) {
+      sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code);
+      break;
+    }
+
     if (wname[0] == 0) {  // no wal file
-      sDebug("%s, no wal file", pPeer->id);
+      code = 0;
+      sDebug("%s, no wal file anymore", pPeer->id);
       break;
     }
 
@@ -416,22 +352,35 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
     // get the full path to wal file
     snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
 
-    // send wal file,
+    // send wal file, old wal file won't be modified, even remove is ok
-    // inotify is not required, old wal file won't be modified, even remove is ok
+    struct stat fstat;
-    if (stat(fname, &fstat) < 0) break;
+    if (stat(fname, &fstat) < 0) {
-    size = fstat.st_size;
+      code = -1;
+      sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
+      break;
+    }
 
+    size = fstat.st_size;
     sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
+
     int32_t sfd = open(fname, O_RDONLY);
-    if (sfd < 0) break;
+    if (sfd < 0) {
+      code = -1;
+      sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
+      break;
+    }
 
     code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
     close(sfd);
-    if (code < 0) break;
+    if (code < 0) {
+      sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
+      break;
+    }
 
-    index++;
+    if (syncAreFilesModified(pNode, pPeer)) {
-
+      code = -1;
-    if (syncAreFilesModified(pPeer) != 0) break;
+      break;
+    }
   }
 
   if (code == 0) {
@@ -440,9 +389,9 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
 
     SWalHead walHead;
     memset(&walHead, 0, sizeof(walHead));
-    code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
+    taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
   } else {
-    sError("%s, failed to send wal since %s", pPeer->id, strerror(errno));
+    sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code);
   }
 
   return code;
@@ -481,18 +430,19 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
 
   pPeer->sversion = 0;
   pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
-  sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+  sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
-  if (syncRetrieveFile(pPeer) < 0) {
+  if (syncRetrieveFile(pPeer) != 0) {
-    sError("%s, failed to retrieve file", pPeer->id);
+    sError("%s, failed to retrieve files", pPeer->id);
     return -1;
   }
 
   // if no files are synced, there must be wal to sync, sversion must be larger than one
   if (pPeer->sversion == 0) pPeer->sversion = 1;
 
-  sInfo("%s, start to retrieve wal", pPeer->id);
+  sInfo("%s, start to retrieve wals", pPeer->id);
-  if (syncRetrieveWal(pPeer) < 0) {
+  int32_t code = syncRetrieveWal(pPeer);
-    sError("%s, failed to retrieve wal", pPeer->id);
+  if (code != 0) {
+    sError("%s, failed to retrieve wals, code:0x%x", pPeer->id, code);
     return -1;
   }
 
@@ -506,7 +456,6 @@ void *syncRetrieveData(void *param) {
 
   if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
 
-  pPeer->fileChanged = 0;
   pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (pPeer->syncFd < 0) {
     sError("%s, failed to open socket to sync", pPeer->id);
@@ -529,7 +478,6 @@ void *syncRetrieveData(void *param) {
   }
 
   pPeer->fileChanged = 0;
-  taosClose(pPeer->notifyFd);
   taosClose(pPeer->syncFd);
   syncDecPeerRef(pPeer);
 

src/tsdb/src/tsdbFile.c

@@ -248,8 +248,13 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
   if (pGroup == NULL) {  // if not exists, create one
     pFGroup->fileId = fid;
     for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-      if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
+      if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) {
-        goto _err;
+        for (int i = type; i >= 0; i--) {
+          remove(pFGroup->files[i].fname);
+        }
+
+        return NULL;
+      }
     }
 
     pthread_rwlock_wrlock(&pFileH->fhlock);
@@ -261,10 +266,6 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
   }
 
   return pGroup;
-
-_err:
-  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
-  return NULL;
 }
 
 void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) {

src/tsdb/src/tsdbRWHelper.c

@@ -917,6 +917,8 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
     ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst);
   }
 
+  ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last));
+
   tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
             blkIdx);
 
@@ -1042,6 +1044,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
   pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast;
   pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last;
 
+  ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last));
+
   tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
             blkIdx);
 
@@ -1622,11 +1626,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
                                   pCfg->update);
 
         if (pDataCols->numOfRows == 0) break;
-        if (tblkIdx == pIdx->numOfBlocks - 1) {
+        if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
-          if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
-        } else {
-          if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
-        }
 
         if (round == 0) {
           if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;

src/util/inc/hash.h

@@ -31,16 +31,18 @@ extern "C" {
 typedef void (*_hash_free_fn_t)(void *param);
 
 typedef struct SHashNode {
-//  char             *key;
   struct SHashNode *next;
   uint32_t          hashVal;  // the hash value of key
   uint32_t          keyLen;   // length of the key
-//  char             *data;
+  size_t            dataLen;  // length of data
+  int8_t            count;    // reference count
+  int8_t            removed;  // flag to indicate removed
+  char              data[];
 } SHashNode;
 
-#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode))
+#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
-#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen)
+#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
-
+#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode));
 typedef enum SHashLockTypeE {
   HASH_NO_LOCK     = 0,
   HASH_ENTRY_LOCK  = 1,
@@ -65,15 +67,6 @@ typedef struct SHashObj {
   SArray         *pMemBlock;    // memory block allocated for SHashEntry
 } SHashObj;
 
-typedef struct SHashMutableIterator {
-  SHashObj  *pHashObj;
-  int32_t    entryIndex;
-  SHashNode *pCur;
-  SHashNode *pNext;           // current node can be deleted for mutable iterator, so keep the next one before return current
-  size_t     numOfChecked;    // already check number of elements in hash table
-  size_t     numOfEntries;    // number of entries while the iterator is created
-} SHashMutableIterator;
-
 /**
  * init the hash table
  *
@@ -142,33 +135,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
  */
 void taosHashCleanup(SHashObj *pHashObj);
 
-/**
+/*
- *
+void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *);
- * @param pHashObj
+*/
- * @return
- */
-SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj);
-
-/**
- *
- * @param iter
- * @return
- */
-bool taosHashIterNext(SHashMutableIterator *iter);
-
-/**
- *
- * @param iter
- * @return
- */
-void *taosHashIterGet(SHashMutableIterator *iter);
-
-/**
- *
- * @param iter
- * @return
- */
-void* taosHashDestroyIter(SHashMutableIterator* iter);
 
 /**
  *
@@ -179,6 +148,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
 
 size_t taosHashGetMemSize(const SHashObj *pHashObj);
 
+void *taosHashIterate(SHashObj *pHashObj, void *p);
+void  taosHashCancelIterate(SHashObj *pHashObj, void *p);
+
 #ifdef __cplusplus
 }
 #endif

src/util/inc/tfile.h

@@ -20,23 +20,26 @@
 extern "C" {
 #endif
 
-#include <unistd.h>
-
 // init taos file module
-int32_t tfinit();
+int32_t tfInit();
 
 // clean up taos file module
-void tfcleanup();
+void tfCleanup();
 
 // the same syntax as UNIX standard open/close/read/write
 // but FD is int64_t and will never be reused
-int64_t tfopen(const char *pathname, int32_t flags);
+int64_t tfOpen(const char *pathname, int32_t flags);
-int64_t tfclose(int64_t tfd);
+int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode);
-int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfClose(int64_t tfd);
-int64_t tfread(int64_t tfd, void *buf, int64_t count);
+int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfRead(int64_t tfd, void *buf, int64_t count);
+int32_t tfFsync(int64_t tfd);
+bool    tfValid(int64_t tfd);
+int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
+int32_t tfFtruncate(int64_t tfd, int64_t length);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_TREF_H
+#endif  // TDENGINE_TFILE_H

src/util/src/hash.c

@@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
 static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
  * @param dsize   size of actual data
  * @return        hash node
  */
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
   assert(pNode->keyLen == pNewNode->keyLen);
+
+  pNode->count--;
   if (prev != NULL) {
     prev->next = pNewNode;
   } else {
     pe->next = pNewNode;
   }
 
-  pNewNode->next = pNode->next;
+  if (pNode->count <= 0) {
+    pNewNode->next = pNode->next;
+    DO_FREE_HASH_NODE(pNode);
+  } else {
+    pNewNode->next = pNode;    
+    pe->num++;
+    atomic_add_fetch_64(&pHashObj->size, 1);
+  }
+  
   return pNewNode;
 }
 
@@ -139,11 +149,11 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
  * @param pIter
  * @return
  */
-static SHashNode *getNextHashNode(SHashMutableIterator *pIter);
 
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
-  if (capacity == 0 || fn == NULL) {
+  assert(fn != NULL);
-    return NULL;
+  if (capacity == 0) {
+    capacity = 4;
   }
 
   SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj));
@@ -213,7 +223,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 
   SHashNode* prev = NULL;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -244,8 +254,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
   } else {
     // not support the update operation, return error
     if (pHashObj->enableUpdate) {
-      doUpdateHashNode(pe, prev, pNode, pNewNode);
+      doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode);
-      DO_FREE_HASH_NODE(pNode);
     } else {
       DO_FREE_HASH_NODE(pNewNode);
     }
@@ -335,22 +344,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
   int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[slot];
 
-  // no data, return directly
-  if (pe->num == 0) {
-    __rd_unlock(&pHashObj->lock, pHashObj->type);
-    return -1;
-  }
-
   if (pHashObj->type == HASH_ENTRY_LOCK) {
     taosWLockLatch(&pe->latch);
   }
 
-  if (pe->num == 0) {
-    assert(pe->next == NULL);
-  } else {
-    assert(pe->next != NULL);
-  }
-
   // double check after locked
   if (pe->num == 0) {
     assert(pe->next == NULL);
@@ -360,37 +357,37 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
     return -1;
   }
 
+  int code = -1;
   SHashNode *pNode = pe->next;
-  SHashNode *pRes = NULL;
+  SHashNode *prevNode = NULL;
 
-  // remove it
+  while (pNode) {
-  if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) 
-    pe->num -= 1;
+      break;
-    pRes = pNode;
+
-    pe->next = pNode->next;
+    prevNode = pNode;
-  } else {
+    pNode = pNode->next;
-    while (pNode->next != NULL) {
+  }
-      if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) {
+
-        assert((pNode->next)->hashVal == hashVal);
+  if (pNode) {
-        break;
+    code = 0;  // it is found
+
+    pNode->count--;
+    pNode->removed = 1;
+    if (pNode->count <= 0) {
+      if (prevNode) {
+        prevNode->next = pNode->next;
+      } else {
+        pe->next = pNode->next;
       }
+   
+      if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
 
-      pNode = pNode->next;
+      pe->num--;
+      atomic_sub_fetch_64(&pHashObj->size, 1);
+      FREE_HASH_NODE(pHashObj, pNode);
     }
-
+  } 
-
-    if (pNode->next != NULL) {
-      pe->num -= 1;
-      pRes = pNode->next;
-      pNode->next = pNode->next->next;
-    }
-  }
-
-  if (pe->num == 0) {
-    assert(pe->next == NULL);
-  } else {
-    assert(pe->next != NULL);
-  }
 
   if (pHashObj->type == HASH_ENTRY_LOCK) {
     taosWUnLockLatch(&pe->latch);
@@ -398,17 +395,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
 
   __rd_unlock(&pHashObj->lock, pHashObj->type);
 
-  if (data != NULL && pRes != NULL) {
+  return code;
-    memcpy(data, GET_HASH_NODE_DATA(pRes), dsize);
-  }
-
-  if (pRes != NULL) {
-    atomic_sub_fetch_64(&pHashObj->size, 1);
-    FREE_HASH_NODE(pHashObj, pRes);
-    return 0;
-  } else {
-    return -1;
-  }
 }
 
 int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) {
@@ -531,98 +518,6 @@ void taosHashCleanup(SHashObj *pHashObj) {
   free(pHashObj);
 }
 
-SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
-  SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator));
-  if (pIter == NULL) {
-    return NULL;
-  }
-
-  pIter->pHashObj = pHashObj;
-
-  // keep it in local variable, in case the resize operation expand the size
-  pIter->numOfEntries = pHashObj->capacity;
-  return pIter;
-}
-
-bool taosHashIterNext(SHashMutableIterator *pIter) {
-  if (pIter == NULL) {
-    return false;
-  }
-
-  size_t size = taosHashGetSize(pIter->pHashObj);
-  if (size == 0) {
-    return false;
-  }
-
-  // check the first one
-  if (pIter->numOfChecked == 0) {
-    assert(pIter->pCur == NULL && pIter->pNext == NULL);
-
-    while (1) {
-      SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
-      if (pEntry->num == 0) {
-        assert(pEntry->next == NULL);
-
-        pIter->entryIndex++;
-        continue;
-      }
-
-      if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-        taosRLockLatch(&pEntry->latch);
-      }
-
-      pIter->pCur = pEntry->next;
-
-      if (pIter->pCur->next) {
-        pIter->pNext = pIter->pCur->next;
-
-        if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-          taosRUnLockLatch(&pEntry->latch);
-        }
-      } else {
-        if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-          taosRUnLockLatch(&pEntry->latch);
-        }
-
-        pIter->pNext = getNextHashNode(pIter);
-      }
-
-      break;
-    }
-
-    pIter->numOfChecked++;
-    return true;
-  } else {
-    assert(pIter->pCur != NULL);
-    if (pIter->pNext) {
-      pIter->pCur = pIter->pNext;
-    } else {  // no more data in the hash list
-      return false;
-    }
-
-    pIter->numOfChecked++;
-
-    if (pIter->pCur->next) {
-      pIter->pNext = pIter->pCur->next;
-    } else {
-      pIter->pNext = getNextHashNode(pIter);
-    }
-
-    return true;
-  }
-}
-
-void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); }
-
-void *taosHashDestroyIter(SHashMutableIterator *iter) {
-  if (iter == NULL) {
-    return NULL;
-  }
-
-  free(iter);
-  return NULL;
-}
-
 // for profile only
 int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
   if (pHashObj == NULL || pHashObj->size == 0) {
@@ -759,6 +654,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
 
   pNewNode->keyLen = (uint32_t)keyLen;
   pNewNode->hashVal = hashVal;
+  pNewNode->dataLen = dsize;
+  pNewNode->count = 1;
 
   memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
   memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
@@ -775,35 +672,6 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) {
   pEntry->num += 1;
 }
 
-SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
-  assert(pIter != NULL);
-
-  pIter->entryIndex++;
-  SHashNode *p = NULL;
-
-  while (pIter->entryIndex < pIter->numOfEntries) {
-    SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
-    if (pEntry->num == 0) {
-      pIter->entryIndex++;
-      continue;
-    }
-
-    if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-      taosRLockLatch(&pEntry->latch);
-    }
-
-    p = pEntry->next;
-
-    if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-      taosRUnLockLatch(&pEntry->latch);
-    }
-
-    return p;
-  }
-
-  return NULL;
-}
-
 size_t taosHashGetMemSize(const SHashObj *pHashObj) {
   if (pHashObj == NULL) {
     return 0;
@@ -811,3 +679,129 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
 
   return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
 }
+
+// release the pNode, return next pNode, and lock the current entry
+static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
+
+  SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p);
+  SHashNode *prevNode = NULL;
+
+  *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity);
+  SHashEntry *pe = pHashObj->hashList[*slot];
+
+  // lock entry
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosWLockLatch(&pe->latch);
+  }
+
+  SHashNode *pNode = pe->next;
+
+  while (pNode) {
+    if (pNode == pOld) 
+      break;
+
+    prevNode = pNode;
+    pNode = pNode->next;
+  }
+
+  if (pNode) { 
+    pNode = pNode->next;
+    while (pNode) {
+      if (pNode->removed == 0) break;
+      pNode = pNode->next;
+    }
+
+    pOld->count--;
+    if (pOld->count <=0) {
+      if (prevNode) {
+        prevNode->next = pOld->next;
+      } else {
+        pe->next = pOld->next;
+      }
+   
+      pe->num--;
+      atomic_sub_fetch_64(&pHashObj->size, 1);
+      FREE_HASH_NODE(pHashObj, pOld);
+    } 
+  } else {
+    uError("pNode:%p data:%p is not there!!!", pNode, p);
+  }
+
+  return pNode;
+}
+
+void *taosHashIterate(SHashObj *pHashObj, void *p) {
+  if (pHashObj == NULL) return NULL; 
+
+  int  slot = 0;
+  char *data = NULL;
+
+  // only add the read lock to disable the resize process
+  __rd_lock(&pHashObj->lock, pHashObj->type);
+
+  SHashNode *pNode = NULL;
+  if (p) {
+    pNode = taosHashReleaseNode(pHashObj, p, &slot);
+    if (pNode == NULL) {
+      SHashEntry *pe = pHashObj->hashList[slot];
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWUnLockLatch(&pe->latch);
+      } 
+
+      slot = slot + 1;
+    }
+  }
+
+  if (pNode == NULL) {
+    for (; slot < pHashObj->capacity; ++slot) {
+      SHashEntry *pe = pHashObj->hashList[slot];
+
+      // lock entry
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWLockLatch(&pe->latch);
+      }
+
+      pNode = pe->next;
+      while (pNode) {
+        if (pNode->removed == 0) break;
+        pNode = pNode->next;
+      }
+
+      if (pNode) break;
+
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWUnLockLatch(&pe->latch);
+      } 
+    }
+  }
+
+  if (pNode) {
+    SHashEntry *pe = pHashObj->hashList[slot];
+    pNode->count++;
+    data = GET_HASH_NODE_DATA(pNode);
+    if (pHashObj->type == HASH_ENTRY_LOCK) {
+      taosWUnLockLatch(&pe->latch);
+    } 
+  }
+
+  __rd_unlock(&pHashObj->lock, pHashObj->type);
+  return data;
+
+}
+
+void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
+  if (pHashObj == NULL || p == NULL) return;
+
+  // only add the read lock to disable the resize process
+  __rd_lock(&pHashObj->lock, pHashObj->type);
+
+  int slot;
+  taosHashReleaseNode(pHashObj, p, &slot);
+
+  SHashEntry *pe = pHashObj->hashList[slot];
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosWUnLockLatch(&pe->latch);
+  } 
+
+  __rd_unlock(&pHashObj->lock, pHashObj->type);
+}

src/util/src/tfile.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
 #include "tulog.h"
@@ -21,40 +22,52 @@
 
 static int32_t tsFileRsetId = -1;
 
-static void taosCloseFile(void *p) {
+static void tfCloseFile(void *p) {
   close((int32_t)(uintptr_t)p);
 }
 
-int32_t tfinit() {
+int32_t tfInit() {
-  tsFileRsetId = taosOpenRef(2000, taosCloseFile);
+  tsFileRsetId = taosOpenRef(2000, tfCloseFile);
-  return tsFileRsetId;
+  if (tsFileRsetId > 0) {
+    return 0;
+  } else {
+    return -1;
+  }
 }
 
-void tfcleanup() {
+void tfCleanup() {
   if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
   tsFileRsetId = -1;
 }
 
-int64_t tfopen(const char *pathname, int32_t flags) {
+static int64_t tfOpenImp(int32_t fd) {
-  int32_t fd = open(pathname, flags);
-
   if (fd < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
-  } 
+  }
 
-  void *p = (void *)(int64_t)fd;
+  void *  p = (void *)(int64_t)fd;
   int64_t rid = taosAddRef(tsFileRsetId, p);
   if (rid < 0) close(fd);
 
   return rid;
 }
 
-int64_t tfclose(int64_t tfd) {
+int64_t tfOpen(const char *pathname, int32_t flags) {
+  int32_t fd = open(pathname, flags);
+  return tfOpenImp(fd);
+}
+
+int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) {
+  int32_t fd = open(pathname, flags, mode);
+  return tfOpenImp(fd);
+}
+
+int64_t tfClose(int64_t tfd) {
   return taosRemoveRef(tsFileRsetId, tfd);
 }
 
-int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
+int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
   void *p = taosAcquireRef(tsFileRsetId, tfd);
   if (p == NULL) return -1;
 
@@ -67,7 +80,7 @@ int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
   return ret;
 }
 
-int64_t tfread(int64_t tfd, void *buf, int64_t count) {
+int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
   void *p = taosAcquireRef(tsFileRsetId, tfd);
   if (p == NULL) return -1;
 
@@ -79,3 +92,44 @@ int64_t tfread(int64_t tfd, void *buf, int64_t count) {
   taosReleaseRef(tsFileRsetId, tfd);
   return ret;
 }
+
+int32_t tfFsync(int64_t tfd) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  int32_t code = fsync(fd);
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return code;
+}
+
+bool tfValid(int64_t tfd) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return false;
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return true;
+}
+
+int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  int64_t ret = taosLSeek(fd, offset, whence);
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return ret;
+}
+
+int32_t tfFtruncate(int64_t tfd, int64_t length) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  int32_t code = taosFtruncate(fd, length);
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return code;
+}