Merge branch 'develop' into test/jenkins

CMakeLists.txt

@@ -13,7 +13,6 @@ ENDIF ()
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
-SET(TD_SYNC TRUE)
 SET(TD_MQTT TRUE)
 SET(TD_TSDB_PLUGINS FALSE)
 

Jenkinsfile

@@ -14,10 +14,12 @@ pipeline {
             sh '''
             date
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -39,11 +41,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -65,11 +69,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -108,11 +114,13 @@ pipeline {
           steps {
             sh '''
             cd ${WKC}
+            git reset --hard
             git checkout develop
             git pull
               
             git submodule update
             cd ${WK}
+            git reset --hard
             git checkout develop
             git pull
             export TZ=Asia/Harbin
@@ -167,7 +175,47 @@ pipeline {
           
          }
        }
-
+       stage('arm64_build'){
+                agent{label 'arm64'}
+                steps{
+                    sh '''
+                    cd ${WK}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    cd ${WKC}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    git submodule update                    
+                    cd ${WKC}/packaging
+                    ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
+                    
+                    '''
+                }
+            }
+            stage('arm32_build'){
+                agent{label 'arm32'}
+                steps{
+                    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                        sh '''
+                        cd ${WK}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        cd ${WKC}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        git submodule update
+                        cd ${WKC}/packaging
+                        ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
+                        
+                        '''
+                    }
+                    
+                }
+            }
       }
     }
 

cmake/define.inc

@@ -13,10 +13,6 @@ IF (TD_GRANT)
   ADD_DEFINITIONS(-D_GRANT)
 ENDIF ()
 
-IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
 IF (TD_MQTT)
   ADD_DEFINITIONS(-D_MQTT)
 ENDIF ()

cmake/input.inc

@@ -47,11 +47,6 @@ IF (${MQTT} MATCHES "false")
   MESSAGE(STATUS "build without mqtt module")
 ENDIF ()
 
-IF (${SYNC} MATCHES "false")
-  SET(TD_SYNC FALSE)
-  MESSAGE(STATUS "build without sync module")
-ENDIF ()
-
 IF (${RANDOM_FILE_FAIL} MATCHES "true")
   SET(TD_RANDOM_FILE_FAIL TRUE)
   MESSAGE(STATUS "build with random-file-fail enabled")

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.8.0")
+  SET(TD_VER_NUMBER "2.0.9.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

documentation20/webdocs/markdowndocs/Model-ch.md

@@ -12,7 +12,7 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个
 CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4;
 ```
 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL</a>  
-  
+
 创建库之后，需要使用SQL命令USE将当前库切换过来，例如：
 
 ```cmd
@@ -20,7 +20,7 @@ USE power;
 ```
 
 就当前连接里操作的库换为power，否则对具体表操作前，需要使用“库名.表名”来指定库的名字。  
-  
+
 **注意：**
 
 - 任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。
@@ -44,15 +44,17 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);
 ```
 其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。
 
+**注意：**目前 TDengine 没有从技术层面限制使用一个 database （dbA）的超级表作为模板建立另一个 database （dbB）的子表，后续会禁止这种用法，不建议使用这种方法建表。
+
 TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列号）。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。  
-  
+
 **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如：
 
 ```cmd
 INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);
 ```
 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。  
-  
+
 ## 多列模型 vs 单列模型
 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。
 

documentation20/webdocs/markdowndocs/administrator-ch.md

@@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 - logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。
 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
+- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
 
 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。
 

documentation20/webdocs/markdowndocs/cluster-ch.md

@@ -37,7 +37,7 @@ fqdn                  h1.taosdata.com
 // 配置本数据节点的端口号，缺省是6030
 serverPort            6030
 
-// 服务端节点数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分
+// 使用场景，请参考《Arbitrator的使用》的部分
 arbitrator            ha.taosdata.com:6042
 ```
 

documentation20/webdocs/markdowndocs/connector-ch.md

@@ -142,7 +142,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
   获取最近一次API调用失败的原因，返回值为错误代码。
 
 
-**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该连接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。
+**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
 
 
 ### 异步查询API

documentation20/webdocs/markdowndocs/faq-ch.md

@@ -36,16 +36,20 @@
 
 5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
 
-6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的
+6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的
 
 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
 
 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
 
-9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
+9. 如果仍不能排除连接故障
-   检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
+   
-   检查服务器侧TCP端口连接是否工作：`nc -l {port}`
+   * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
-   检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}`
+     检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
+     检查服务器侧TCP端口连接是否工作：`nc -l {port}`
+     检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}`
+   
+   * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
    
 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。
 
@@ -101,7 +105,20 @@ Connection = DriverManager.getConnection(url, properties);
   <version>2.0.4</version>
 </dependency>
 ```
-## 14. 怎么报告问题？
+## 14. taos connect failed, reason: invalid timestamp 
+
+常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。
+
+
+
+## 15. 表名显示不全
+
+由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
+
+
+
+## 16. 怎么报告问题？
+
 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包:
 1. /var/log/taos
 2. /etc/taos 

packaging/cfg/taos.cfg

@@ -236,7 +236,7 @@
 # httpDebugFlag         131
 
 # debug flag for monitor
-# monitorDebugFlag      131
+# monDebugFlag          131
 
 # debug flag for query
 # qDebugflag            131
@@ -251,7 +251,7 @@
 # cqDebugFlag           131
 
 # enable/disable recording the SQL in taos client
-# tscEnableRecordSql    0
+# enableRecordSql    0
 
 # generate core file when service crash
 # enableCoreFile        1
@@ -260,4 +260,10 @@
 # maxBinaryDisplayWidth 30
 
 # enable/disable telemetry reporting
-# telemetryReporting    1
+# telemetryReporting    1
+
+# enable/disable stream (continuous query)
+# stream                1
+
+# only 50% CPU resources will be used in query processing
+# halfCoresForQuery     0

packaging/tools/install.sh

@@ -172,6 +172,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos     || :
     ${csudo} rm -f ${bin_link_dir}/taosd    || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
@@ -182,6 +183,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                      || :
     [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd                   || :
     [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo          || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump          || :
     [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos          || :
     [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core    || :
     [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :

packaging/tools/install_client.sh

@@ -84,8 +84,9 @@ function install_main_path() {
 function install_bin() {
   # Remove links
   ${csudo} rm -f ${bin_link_dir}/taos         || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+      ${csudo} rm -f ${bin_link_dir}/taosdump || :
   fi
   ${csudo} rm -f ${bin_link_dir}/rmtaos       || :
   ${csudo} rm -f ${bin_link_dir}/set_core     || :
@@ -94,8 +95,9 @@ function install_bin() {
 
   #Make link
   [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                 || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
+      [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
   fi
   [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

packaging/tools/install_client_power.sh

@@ -84,8 +84,9 @@ function install_main_path() {
 function install_bin() {
   # Remove links
   ${csudo} rm -f ${bin_link_dir}/power         || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       ${csudo} rm -f ${bin_link_dir}/powerdemo || :
+      ${csudo} rm -f ${bin_link_dir}/powerdump || :
   fi
   ${csudo} rm -f ${bin_link_dir}/rmpower       || :
   ${csudo} rm -f ${bin_link_dir}/set_core      || :
@@ -94,8 +95,9 @@ function install_bin() {
 
   #Make link
   [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power                 || :
-  if [ "$osType" == "Darwin" ]; then
+  if [ "$osType" != "Darwin" ]; then
       [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
+      [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
   fi
   [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

packaging/tools/install_power.sh

@@ -172,6 +172,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/power     || :
     ${csudo} rm -f ${bin_link_dir}/powerd    || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump || :
     ${csudo} rm -f ${bin_link_dir}/rmpower   || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator   || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
@@ -182,6 +183,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power                        || :
     [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd                     || :
     [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo            || :
+    [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump            || :
     [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower  || :
     [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core         || :
     [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator      || :

packaging/tools/post.sh

@@ -92,6 +92,7 @@ function install_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos     || :
     ${csudo} rm -f ${bin_link_dir}/taosd    || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
     ${csudo} rm -f ${bin_link_dir}/set_core || :
 
@@ -101,6 +102,7 @@ function install_bin() {
     [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos             || :
     [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd          || :
     [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || :
+    [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || :
     [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || :
 }
 

packaging/tools/remove.sh

@@ -72,6 +72,7 @@ function clean_bin() {
     ${csudo} rm -f ${bin_link_dir}/taos        || :
     ${csudo} rm -f ${bin_link_dir}/taosd       || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo    || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump    || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos      || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator || :
     ${csudo} rm -f ${bin_link_dir}/set_core    || :
@@ -222,4 +223,4 @@ elif  echo $osinfo | grep -qwi "centos" ; then
 fi
 
 echo -e "${GREEN}TDengine is removed successfully!${NC}"
-echo 
+echo 

packaging/tools/remove_client.sh

@@ -38,6 +38,7 @@ function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/taos      || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo  || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump  || :
     ${csudo} rm -f ${bin_link_dir}/rmtaos    || :
     ${csudo} rm -f ${bin_link_dir}/set_core  || :
 }

packaging/tools/remove_client_power.sh

@@ -38,6 +38,7 @@ function clean_bin() {
     # Remove link
     ${csudo} rm -f ${bin_link_dir}/power      || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo  || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump  || :
     ${csudo} rm -f ${bin_link_dir}/rmpower    || :
     ${csudo} rm -f ${bin_link_dir}/set_core   || :
 }

packaging/tools/remove_power.sh

@@ -72,6 +72,7 @@ function clean_bin() {
     ${csudo} rm -f ${bin_link_dir}/power        || :
     ${csudo} rm -f ${bin_link_dir}/powerd       || :
     ${csudo} rm -f ${bin_link_dir}/powerdemo    || :
+    ${csudo} rm -f ${bin_link_dir}/powerdump    || :
     ${csudo} rm -f ${bin_link_dir}/rmpower      || :
     ${csudo} rm -f ${bin_link_dir}/tarbitrator  || :
     ${csudo} rm -f ${bin_link_dir}/set_core     || :
@@ -223,4 +224,4 @@ fi
 #fi
 
 echo -e "${GREEN}PowerDB is removed successfully!${NC}"
-echo 
+echo 

snap/snapcraft.yaml

@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: '2.0.8.0'
+version: '2.0.9.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.8.0
+      - usr/lib/libtaos.so.2.0.9.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/CMakeLists.txt

@@ -10,9 +10,7 @@ ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)
 ADD_SUBDIRECTORY(kit)
 ADD_SUBDIRECTORY(plugins)
-IF (TD_SYNC)
+ADD_SUBDIRECTORY(sync)
-  ADD_SUBDIRECTORY(sync)
-ENDIF ()
 ADD_SUBDIRECTORY(balance)
 ADD_SUBDIRECTORY(mnode)
 ADD_SUBDIRECTORY(vnode)

src/balance/inc/bnInt.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_INT_H
+#define TDENGINE_BALANCE_INT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeInt.h"
+#include "mnodeDef.h"
+#include "mnodeDnode.h"
+
+typedef struct {
+  int32_t     size;
+  int32_t     maxSize;
+  SDnodeObj **list;
+} SBnDnodes;
+
+typedef struct {
+  void *          timer;
+  bool            stop;
+  pthread_mutex_t mutex;
+  pthread_cond_t  cond;
+  pthread_t       thread;
+} SBnThread;
+
+typedef struct {
+  pthread_mutex_t mutex;
+} SBnMgmt;
+
+int32_t bnInit();
+void    bnCleanUp();
+bool    bnStart();
+void    bnCheckStatus();
+void    bnCheckModules();
+
+extern SBnDnodes tsBnDnodes;
+extern void *tsMnodeTmr;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/inc/bnScore.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_SCORE_H
+#define TDENGINE_BALANCE_SCORE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "bnInt.h"
+
+void  bnInitDnodes();
+void  bnCleanupDnodes();
+void  bnAccquireDnodes();
+void  bnReleaseDnodes();
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/inc/bnThread.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_THREAD_H
+#define TDENGINE_BALANCE_THREAD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "bnInt.h"
+
+int32_t bnInitThread();
+void    bnCleanupThread();
+void    bnNotify();
+void    bnStartTimer(int64_t mseconds);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/balance/src/bnMain.c

@@ -15,17 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tutil.h"
+#include "tref.h"
-#include "tbalance.h"
 #include "tsync.h"
-#include "ttimer.h"
 #include "tglobal.h"
-#include "tdataformat.h"
 #include "dnode.h"
-#include "mnode.h"
+#include "bnInt.h"
-#include "mnodeDef.h"
+#include "bnScore.h"
-#include "mnodeInt.h"
+#include "bnThread.h"
-#include "mnodeDnode.h"
 #include "mnodeDb.h"
 #include "mnodeMnode.h"
 #include "mnodeSdb.h"
@@ -33,36 +29,20 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-/*
+extern int64_t tsDnodeRid;
- * once sdb work as mater, then tsAccessSquence reset to zero
+extern int64_t tsSdbRid;
- * increase tsAccessSquence every balance interval
+static SBnMgmt tsBnMgmt;
- */
+static void  bnMonitorDnodeModule();
-extern void *       tsMnodeTmr;
-static void *       tsBalanceTimer = NULL;
-static int32_t      tsBalanceDnodeListSize = 0;
-static SDnodeObj ** tsBalanceDnodeList = NULL;
-static int32_t      tsBalanceDnodeListMallocSize = 16;
-static pthread_mutex_t tsBalanceMutex;
 
-static void  balanceStartTimer(int64_t mseconds);
+static void bnLock() {
-static void  balanceInitDnodeList();
+  pthread_mutex_lock(&tsBnMgmt.mutex);
-static void  balanceCleanupDnodeList();
-static void  balanceAccquireDnodeList();
-static void  balanceReleaseDnodeList();
-static void  balanceMonitorDnodeModule();
-static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-
-static void balanceLock() {
-  pthread_mutex_lock(&tsBalanceMutex);
 }
 
-static void balanceUnLock() {
+static void bnUnLock() {
-  pthread_mutex_unlock(&tsBalanceMutex);
+  pthread_mutex_unlock(&tsBnMgmt.mutex);
 }
 
-static bool balanceCheckFree(SDnodeObj *pDnode) {
+static bool bnCheckFree(SDnodeObj *pDnode) {
   if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
     return false;
@@ -86,7 +66,7 @@ static bool balanceCheckFree(SDnodeObj *pDnode) {
   return true;
 }
 
-static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
 
   SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
@@ -111,27 +91,26 @@ static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mnodeUpdateVgroup(pVgroup);
 }
 
-static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
   // SVnodeGid tmp = *pVnodeGid1;
   // *pVnodeGid1 = *pVnodeGid2;
   // *pVnodeGid2 = tmp;
 }
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t bnAllocVnodes(SVgObj *pVgroup) {
   static int32_t randIndex = 0;
   int32_t dnode = 0;
   int32_t vnodes = 0;
 
-  balanceLock();
+  bnLock();
-
+  bnAccquireDnodes();
-  balanceAccquireDnodeList();
 
   mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
-         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
+         mnodeGetDnodesNum(), tsBnDnodes.size);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
+    for (; dnode < tsBnDnodes.size; ++dnode) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
+      SDnodeObj *pDnode = tsBnDnodes.list[dnode];
-      if (balanceCheckFree(pDnode)) {
+      if (bnCheckFree(pDnode)) {
         SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
         pVnodeGid->dnodeId = pDnode->dnodeId;
         pVnodeGid->pDnode = pDnode;
@@ -148,8 +127,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   }
 
   if (vnodes != pVgroup->numOfVnodes) {
-    balanceReleaseDnodeList();
+    bnReleaseDnodes();
-    balanceUnLock();
+    bnUnLock();
 
     mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
 
@@ -162,7 +141,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
              pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
       mnodeDecDnodeRef(pDnode);
     }
-    sdbFreeIter(pIter);
 
     if (mnodeGetOnlineDnodesNum() == 0) {
       return TSDB_CODE_MND_NOT_READY;
@@ -180,33 +158,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes == 1) {
   } else if (pVgroup->numOfVnodes == 2) {
     if (randIndex++ % 2 == 0) {
-      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
     }
   } else {
     int32_t randVal = randIndex++ % 6;
     if (randVal == 1) {  // 1, 0, 2
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
     } else if (randVal == 2) {  // 1, 2, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else if (randVal == 3) {  // 2, 1, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
     } else if (randVal == 4) {  // 2, 0, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     }
     if (randVal == 5) {  // 0, 2, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else {
     }  // 0, 1, 2
   }
 
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-  balanceUnLock();
+  bnUnLock();
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
   if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
     return false;
   }
@@ -233,7 +211,7 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
  * desc: remove one vnode from vgroup
  * all vnodes in vgroup should in ready state, except the balancing one
  **/
-static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
+static int32_t bnRemoveVnode(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes <= 1) return -1;
 
   SVnodeGid *pRmVnode = NULL;
@@ -275,17 +253,17 @@ static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
     pSelVnode = pRmVnode;
   }
 
-  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+  if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
     mDebug("vgId:%d, is not ready", pVgroup->vgId);
     return -1;
   } else {
     mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
-    balanceDiscardVnode(pVgroup, pSelVnode);
+    bnDiscardVnode(pVgroup, pSelVnode);
     return TSDB_CODE_SUCCESS;
   }
 }
 
-static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pGid = &pVgroup->vnodeGid[i];
     if (pGid->dnodeId == 0) break;
@@ -300,13 +278,13 @@ static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
 /**
  * desc: add vnode to vgroup, find a new one if dest dnode is null
  **/
-static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
   if (pDestDnode == NULL) {
-    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+      SDnodeObj *pDnode = tsBnDnodes.list[i];
       if (pDnode == pSrcDnode) continue;
-      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
-      if (!balanceCheckFree(pDnode)) continue;
+      if (!bnCheckFree(pDnode)) continue;
       
       pDestDnode = pDnode;
       mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
@@ -334,25 +312,25 @@ static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceMonitorBalance() {
+static bool bnMonitorBalance() {
-  if (tsBalanceDnodeListSize < 2) return false;
+  if (tsBnDnodes.size < 2) return false;
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
+  for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[src];
+    SDnodeObj *pDnode = tsBnDnodes.list[src];
-    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
            pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
            pDnode->openVnodes);
   }
 
-  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
   if (scoresDiff < 0.01) {
-    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
     return false;
   }
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
+  for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) {
-    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
+    SDnodeObj *pSrcDnode = tsBnDnodes.list[src];
-    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+    float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1);
     if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
       continue;
     }
@@ -363,29 +341,27 @@ static bool balanceMonitorBalance() {
       pIter = mnodeGetNextVgroup(pIter, &pVgroup);
       if (pVgroup == NULL) break;
 
-      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+      if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
         for (int32_t dest = 0; dest < src; dest++) {
-          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
+          SDnodeObj *pDestDnode = tsBnDnodes.list[dest];
-          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+          if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
 
-          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          float destScore = bnTryCalcDnodeScore(pDestDnode, 1);
           if (srcScore + 0.0001 < destScore) continue;
-          if (!balanceCheckFree(pDestDnode)) continue;
+          if (!bnCheckFree(pDestDnode)) continue;
           
           mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
                  pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
                  srcScore, pDestDnode->score, destScore);
-          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
           mnodeDecVgroupRef(pVgroup);
-          sdbFreeIter(pIter);
+          mnodeCancelGetNextVgroup(pIter);
           return true;
         }
       }
 
       mnodeDecVgroupRef(pVgroup);
     }
-
-    sdbFreeIter(pIter);
   }
 
   return false;
@@ -395,7 +371,7 @@ static bool balanceMonitorBalance() {
 // 1. reset balanceAccessSquence to zero
 // 2. reset state of dnodes to offline
 // 3. reset lastAccess of dnodes to zero
-void balanceReset() {
+void bnReset() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   while (1) {
@@ -413,12 +389,10 @@ void balanceReset() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   tsAccessSquence = 0;
 }
 
-static int32_t balanceMonitorVgroups() {
+static int32_t bnMonitorVgroups() {
   void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
   bool    hasUpdatingVgroup = false;
@@ -434,25 +408,24 @@ static int32_t balanceMonitorVgroups() {
     if (vgReplica > dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceRemoveVnode(pVgroup);
+      code = bnRemoveVnode(pVgroup);
     } else if (vgReplica < dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceAddVnode(pVgroup, NULL, NULL);
+      code = bnAddVnode(pVgroup, NULL, NULL);
     }
 
     mnodeDecVgroupRef(pVgroup);
     if (code == TSDB_CODE_SUCCESS) {
+      mnodeCancelGetNextVgroup(pIter);
       break;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return hasUpdatingVgroup;
 }
 
-static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) {
   mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
 
   void *  pIter = NULL;
@@ -462,14 +435,15 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
-    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup);
     mnodeDecVgroupRef(pVgroup);
 
-    if (hasThisDnode) break;
+    if (hasThisDnode) {
+      mnodeCancelGetNextVgroup(pIter);
+      break;
+    }
   }
 
-  sdbFreeIter(pIter);
-
   if (!hasThisDnode) {
     mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
     mnodeDropDnode(pDnode, NULL);
@@ -479,7 +453,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
   return false;
 }
 
-static bool balanceMontiorDropping() {
+static bool bnMontiorDropping() {
   void *pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
@@ -499,50 +473,46 @@ static bool balanceMontiorDropping() {
       pDnode->status = TAOS_DN_STATUS_DROPPING;
       mnodeUpdateDnode(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return true;
     }
 
     if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
-      bool ret = balanceMonitorDnodeDropping(pDnode);
+      bool ret = bnMonitorDnodeDropping(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return ret;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return false;
 }
 
-static bool balanceStart() {
+bool bnStart() {
   if (!sdbIsMaster()) return false;
 
-  balanceLock();
+  bnLock();
+  bnAccquireDnodes();
 
-  balanceAccquireDnodeList();
+  bnMonitorDnodeModule();
 
-  balanceMonitorDnodeModule();
+  bool updateSoon = bnMontiorDropping();
-
-  bool updateSoon = balanceMontiorDropping();
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorVgroups();
+    updateSoon = bnMonitorVgroups();
   }
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorBalance();
+    updateSoon = bnMonitorBalance();
   }
  
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-
+  bnUnLock();
-  balanceUnLock();
 
   return updateSoon;
 }
 
-static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+static void bnSetVgroupOffline(SDnodeObj* pDnode) {
   void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup;
@@ -556,14 +526,15 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
     }
     mnodeDecVgroupRef(pVgroup);
   }
-
-  sdbFreeIter(pIter);
 }
 
-static void balanceCheckDnodeAccess() {
+void bnCheckStatus() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
+  void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid);
+  if (dnodeSdb == NULL) return;
+
   while (1) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
@@ -573,85 +544,41 @@ static void balanceCheckDnodeAccess() {
         pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
         mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
               pDnode->lastAccess, pDnode->status);
-        balanceSetVgroupOffline(pDnode);
+        bnSetVgroupOffline(pDnode);
       }
     }
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
+  taosReleaseRef(tsSdbRid, tsDnodeRid);
 }
 
-static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
+void bnCheckModules() {
-  if (!sdbIsMaster()) return;
-
-  tsBalanceTimer = NULL;
-  tsAccessSquence ++;
-
-  balanceCheckDnodeAccess();  
-  bool updateSoon = false;
-
-  if (handle == NULL) {
-    if (tsAccessSquence % tsBalanceInterval == 0) {
-      mDebug("balance function is scheduled by timer");
-      updateSoon = balanceStart();
-    }
-  } else {
-    int64_t mseconds = (int64_t)handle;
-    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
-    updateSoon = balanceStart();
-  }
-
-  if (updateSoon) {
-    balanceStartTimer(1000);
-  } else {
-    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
-  }
-}
-
-static void balanceStartTimer(int64_t mseconds) {
-  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
-}
-
-void balanceSyncNotify() {
   if (sdbIsMaster()) {
-    balanceLock();
+    bnLock();
-    balanceAccquireDnodeList();
+    bnAccquireDnodes();
-    balanceMonitorDnodeModule();
+    bnMonitorDnodeModule();
-    balanceReleaseDnodeList();
+    bnReleaseDnodes();
-    balanceUnLock();
+    bnUnLock();
   }
 }
 
-void balanceAsyncNotify() {
+int32_t bnInit() {
-  balanceStartTimer(500); 
+  pthread_mutex_init(&tsBnMgmt.mutex, NULL);
-}
+  bnInitDnodes();
-
+  bnInitThread();
-int32_t balanceInit() {
+  bnReset();
-  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
-  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
-  
-  pthread_mutex_init(&tsBalanceMutex, NULL);
-  balanceInitDnodeList();
-  balanceStartTimer(2000);
-  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
-
-  balanceReset();
   
   return 0;
 }
 
-void balanceCleanUp() {
+void bnCleanUp() {
-  if (tsBalanceTimer != NULL) {
+  bnCleanupThread();
-    taosTmrStopA(&tsBalanceTimer);
+  bnCleanupDnodes();
-    pthread_mutex_destroy(&tsBalanceMutex);
+  pthread_mutex_destroy(&tsBnMgmt.mutex);
-    tsBalanceTimer = NULL;
-    mDebug("stop balance timer");
-  }
-  balanceCleanupDnodeList();
 }
 
-int32_t balanceDropDnode(SDnodeObj *pDnode) {
+int32_t bnDropDnode(SDnodeObj *pDnode) {
   int32_t    totalFreeVnodes = 0;
   void *     pIter = NULL;
   SDnodeObj *pTempDnode = NULL;
@@ -660,15 +587,13 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
     pIter = mnodeGetNextDnode(pIter, &pTempDnode);
     if (pTempDnode == NULL) break;
 
-    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+    if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) {
       totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
     }
 
     mnodeDecDnodeRef(pTempDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (pDnode->openVnodes > totalFreeVnodes) {
     mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;
@@ -677,296 +602,17 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
   pDnode->status = TAOS_DN_STATUS_DROPPING;
   mnodeUpdateDnode(pDnode);
   
-  balanceStartTimer(1100);
+  bnStartTimer(1100);
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
+static void bnMonitorDnodeModule() {
-  if (pDnode->cpuAvgUsage < 80)
-    return 0;
-  else if (pDnode->cpuAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
-  if (pDnode->memoryAvgUsage < 80)
-    return 0;
-  else if (pDnode->memoryAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
-  if (pDnode->diskAvgUsage < 80)
-    return 0;
-  else if (pDnode->diskAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
-  if (pDnode->bandwidthUsage < 30)
-    return 0;
-  else if (pDnode->bandwidthUsage < 80)
-    return 10;
-  else
-    return 50;
-}
-
-static float balanceCalcModuleScore(SDnodeObj *pDnode) {
-  if (pDnode->numOfCores <= 0) return 0;
-  if (pDnode->isMgmt) {
-     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
-  }
-  return 0;
-}
-
-static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
-  if (pDnode->numOfCores <= 0) return 0;
-  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
-}
-
-/**
- * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
- * 1. get the score config
- * 2. if the value is out of range, use border data
- * 3. otherwise use interpolation method
- **/
-void balanceCalcDnodeScore(SDnodeObj *pDnode) {
-  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
-                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
-}
-
-float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                        balanceCalcBandwidthScore(pDnode);
-  float moduleScore = balanceCalcModuleScore(pDnode);
-  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
-
-  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
-  return score;
-}
-
-static void balanceInitDnodeList() {
-  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
-}
-
-static void balanceCleanupDnodeList() {
-  if (tsBalanceDnodeList != NULL) {
-    free(tsBalanceDnodeList);
-    tsBalanceDnodeList = NULL;
-  }
-}
-
-static void balanceCheckDnodeListSize(int32_t dnodesNum) {
-  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
-    tsBalanceDnodeListMallocSize = dnodesNum * 2;
-    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
-  }
-}
-
-void balanceAccquireDnodeList() {
-  int32_t dnodesNum = mnodeGetDnodesNum();
-  balanceCheckDnodeListSize(dnodesNum);
-
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  int32_t    dnodeIndex = 0;
-
-  while (1) {  
-    if (dnodeIndex >= dnodesNum) break;
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-      mnodeDecDnodeRef(pDnode);
-      continue;
-    }
-
-    balanceCalcDnodeScore(pDnode);
-    
-    int32_t orderIndex = dnodeIndex;
-    for (; orderIndex > 0; --orderIndex) {
-      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
-        break;
-      }
-      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
-    }
-    tsBalanceDnodeList[orderIndex] = pDnode;
-    dnodeIndex++;
-  }
-
-  sdbFreeIter(pIter);
-
-  tsBalanceDnodeListSize = dnodeIndex;
-}
-
-void balanceReleaseDnodeList() {
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
-    if (pDnode != NULL) {
-      mnodeDecDnodeRef(pDnode);
-    }
-  }
-}
-
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mnodeGetUserFromConn(pConn);
-  if (pUser == NULL) return 0;
-
-  if (strcmp(pUser->pAcct->user, "root") != 0)  {
-    mnodeDecUserRef(pUser);
-    return TSDB_CODE_MND_NO_RIGHTS;
-  }
-
-  int32_t cols = 0;
-  SSchema *pSchema = pMeta->schema;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "system scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "custom scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "module scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "vnode scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "total scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "open vnodes");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "cpu cores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "balance state");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htons(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->numOfRows = mnodeGetDnodesNum();
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pIter = NULL;
-
-  mnodeDecUserRef(pUser);
-
-  return 0;
-}
-
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
-  int32_t    numOfRows = 0;
-  SDnodeObj *pDnode = NULL;
-  char *     pWrite;
-  int32_t    cols = 0;
-  
-  while (numOfRows < rows) {
-    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                      balanceCalcBandwidthScore(pDnode);
-    float moduleScore = balanceCalcModuleScore(pDnode);
-    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
-
-    cols = 0;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->dnodeId;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = systemScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = pDnode->customScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)moduleScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)vnodeScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->openVnodes;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->numOfCores;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
-    cols++;
-
-    numOfRows++;
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  pShow->numOfReads += numOfRows;
-  return numOfRows;
-}
-
-static void balanceMonitorDnodeModule() {
   int32_t numOfMnodes = mnodeGetMnodesNum();
   if (numOfMnodes >= tsNumOfMnodes) return;
 
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
     if (pDnode == NULL) break;
 
     if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@@ -990,7 +636,7 @@ static void balanceMonitorDnodeModule() {
   }
 }
 
-int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
   if (!sdbIsMaster()) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
@@ -1014,29 +660,29 @@ int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
   }
 
-  balanceLock();
+  bnLock();
-  balanceAccquireDnodeList();
+  bnAccquireDnodes();
 
   int32_t code = TSDB_CODE_SUCCESS;
-  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+  if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, pSrcDnode->dnodeId);
     code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
-  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+  } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, dnodeId);
     code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
-  } else if (!balanceCheckFree(pDestDnode)) {
+  } else if (!bnCheckFree(pDestDnode)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
            dnodeId);
     code = TSDB_CODE_MND_DNODE_NOT_FREE;
   } else {
-    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
     mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
   }
 
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
-  balanceUnLock();
+  bnUnLock();
 
   mnodeDecVgroupRef(pVgroup);
   mnodeDecDnodeRef(pDestDnode);

src/balance/src/bnScore.c

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "bnScore.h"
+
+SBnDnodes tsBnDnodes;
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static int32_t bnCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcBandScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float bnCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+    return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+static void bnCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                  bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) +
+                  pDnode->customScore;
+}
+
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                        bnCalcBandScore(pDnode);
+  float moduleScore = bnCalcModuleScore(pDnode);
+  float vnodeScore = bnCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+void bnInitDnodes() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
+
+  memset(&tsBnDnodes, 0, sizeof(SBnDnodes));
+  tsBnDnodes.maxSize = 16;
+  tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *));
+}
+
+void bnCleanupDnodes() {
+  if (tsBnDnodes.list != NULL) {
+    free(tsBnDnodes.list);
+    tsBnDnodes.list = NULL;
+  }
+}
+
+static void bnCheckDnodesSize(int32_t dnodesNum) {
+  if (tsBnDnodes.maxSize <= dnodesNum) {
+    tsBnDnodes.maxSize = dnodesNum * 2;
+    tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *));
+  }
+}
+
+void bnAccquireDnodes() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  bnCheckDnodesSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {
+    if (dnodeIndex >= dnodesNum) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
+
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    bnCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) {
+        break;
+      }
+      tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1];
+    }
+    tsBnDnodes.list[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  tsBnDnodes.size = dnodeIndex;
+}
+
+void bnReleaseDnodes() {
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0) {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode);
+    float moduleScore = bnCalcModuleScore(pDnode);
+    float vnodeScore = bnCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}

src/balance/src/bnThread.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "mnodeSdb.h"
+#include "bnThread.h"
+
+static SBnThread tsBnThread;
+
+static void *bnThreadFunc(void *arg) {
+  while (1) {
+    pthread_mutex_lock(&tsBnThread.mutex);
+    if (tsBnThread.stop) {
+      pthread_mutex_unlock(&tsBnThread.mutex);
+      break;
+    }
+
+    pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
+    bool updateSoon = bnStart();
+    bnStartTimer(updateSoon ? 1000 : -1);
+    pthread_mutex_unlock(&(tsBnThread.mutex));
+  }
+
+  mDebug("balance thread is stopped");
+  return NULL;
+}
+
+int32_t bnInitThread() {
+  memset(&tsBnThread, 0, sizeof(SBnThread));
+  tsBnThread.stop = false;
+  pthread_mutex_init(&tsBnThread.mutex, NULL);
+  pthread_cond_init(&tsBnThread.cond, NULL);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    mError("failed to create balance thread since %s", strerror(errno));
+    return -1;
+  }
+
+  bnStartTimer(2000);
+  mDebug("balance thread is created");
+  return 0;
+}
+
+void bnCleanupThread() {
+  mDebug("balance thread will be cleanup");
+
+  if (tsBnThread.timer != NULL) {
+    taosTmrStopA(&tsBnThread.timer);
+    tsBnThread.timer = NULL;
+    mDebug("stop balance timer");
+  }
+
+  pthread_mutex_lock(&tsBnThread.mutex);
+  tsBnThread.stop = true;
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+  pthread_join(tsBnThread.thread, NULL);
+
+  pthread_cond_destroy(&tsBnThread.cond);
+  pthread_mutex_destroy(&tsBnThread.mutex);
+}
+
+static void bnPostSignal() {
+  if (tsBnThread.stop) return;
+
+  pthread_mutex_lock(&tsBnThread.mutex);
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+}
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+
+static void bnProcessTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+  if (tsBnThread.stop) return;
+
+  tsBnThread.timer = NULL;
+  tsAccessSquence++;
+
+  bnCheckStatus();
+  bnStartTimer(-1);
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      bnPostSignal();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    bnPostSignal();
+  }
+}
+
+void bnStartTimer(int64_t mseconds) {
+  if (tsBnThread.stop) return;
+
+  bool updateSoon = (mseconds != -1);
+  if (updateSoon) {
+    taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
+  } else {
+    taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
+  }
+}
+
+void bnNotify() {
+  bnStartTimer(500); 
+}

src/client/inc/tscLocalMerge.h

@@ -56,7 +56,6 @@ typedef struct SLocalReducer {
   tFilePage *            pTempBuffer;
   struct SQLFunctionCtx *pCtx;
   int32_t                rowSize;      // size of each intermediate result.
-  int32_t                finalRowSize; // final result row size
   int32_t                status;       // denote it is in reduce process, in reduce process, it
   bool                   hasPrevRow;   // cannot be released
   bool                   hasUnprocessedRow;

src/client/inc/tscUtil.h

@@ -92,7 +92,7 @@ typedef struct SVgroupTableInfo {
 } SVgroupTableInfo;
 
 static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
-  assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE);
+  assert(pCmd != NULL && subClauseIndex >= 0);
 
   if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) {
     return NULL;
@@ -278,7 +278,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql);
 void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
-int  tscSetMgmtEpSetFromCfg(const char *first, const char *second);
+int  tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
 
 bool tscSetSqlOwner(SSqlObj* pSql);
 void tscClearSqlOwner(SSqlObj* pSql);

src/client/inc/tsclient.h

@@ -246,11 +246,14 @@ typedef struct SQueryInfo {
   int16_t          fillType;      // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
-  struct STSBuf *  tsBuf;
+  struct STSBuf   *tsBuf;
   int64_t *        fillVal;       // default value for fill
   char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;   // limit for current sub clause
+
   int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
+  int64_t          tableLimit;    // table limit in case of super table projection query + global order + limit
+
   int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
   int16_t          resColumnId;   // result column id
 } SQueryInfo;
@@ -282,7 +285,7 @@ typedef struct {
 
   int8_t       dataSourceType;     // load data from file or not
   int8_t       submitSchema; // submit block is built with table schema
-  STagData     tagData;
+  STagData    *pTagData;     // NOTE: pTagData->data is used as a variant length array
   SHashObj    *pTableList;   // referred table involved in sql
   SArray      *pDataBlocks;  // SArray<STableDataBlocks*> submit data blocks after parsing sql
 } SSqlCmd;
@@ -333,9 +336,10 @@ typedef struct STscObj {
   char               superAuth : 1;
   uint32_t           connId;
   uint64_t           rid;      // ref ID returned by taosAddRef
-  struct SSqlObj *   pHb;
+  int64_t            hbrid;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
+  SRpcCorEpSet       *tscCorMgmtEpSet;
   void*              pDnodeConn;
   pthread_mutex_t    mutex;
   T_REF_DECLARE()
@@ -373,7 +377,7 @@ typedef struct SSqlObj {
   struct SSqlObj **pSubs;
 
   struct SSqlObj  *prev, *next;
-  struct SSqlObj **self;
+  int64_t          self;
 } SSqlObj;
 
 typedef struct SSqlStream {
@@ -507,7 +511,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
 }
 
 extern SCacheObj*    tscMetaCache;
-extern SCacheObj*    tscObjCache;
+extern int           tscObjRef;
 extern void *    tscTmr;
 extern void *    tscQhandle;
 extern int       tscKeepConn[];
@@ -515,7 +519,6 @@ extern int       tsInsertHeadSize;
 extern int       tscNumOfThreads;
 extern int       tscRefId;
   
-extern SRpcCorEpSet tscMgmtEpSet;
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 

src/client/src/tscFunctionImpl.c

@@ -54,8 +54,8 @@
 
 #define DO_UPDATE_TAG_COLUMNS(ctx, ts)                                           \
   do {                                                                           \
-    for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) {                  \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) {                  \
-      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];                     \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];                     \
       if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) {                             \
         __ctx->tag.i64Key = (ts); \
         __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
@@ -64,13 +64,13 @@
     }                                                                            \
   } while (0);
 
-#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
+#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                   \
-do {\
+  do {                                                          \
-for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) {                  \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
-      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];                     \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];    \
-      aAggs[TSDB_FUNC_TAG].xFunction(__ctx);                                     \
+      aAggs[TSDB_FUNC_TAG].xFunction(__ctx);                    \
-    }     \
+    }                                                           \
-} while(0);
+  } while (0);
 
 void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {}
 void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {}
@@ -305,7 +305,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interBytes = dataBytes;
+    *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
   } else if (functionId == TSDB_FUNC_SPREAD) {
     *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
@@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   }
   
   SET_VAL(pCtx, 1, 1);
-  
+  *((int64_t *)pCtx->aOutputBuf) += pCtx->size;
-  *((int64_t *)pCtx->aOutputBuf) += 1;
   
   // do not need it actually
   SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
@@ -1170,8 +1169,8 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp
         if ((*(int32_t *)output < v) ^ isMin) {
           *(int32_t *)output = v;
           
-          for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) {
+          for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) {
-            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i];
+            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j];
             aAggs[TSDB_FUNC_TAG].xFunction(__ctx);
           }
           
@@ -1680,16 +1679,35 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
-  SET_VAL(pCtx, 1, 1);
+  // the scan order is not the required order, ignore it
-  memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
+  if (pCtx->order != pCtx->param[0].i64Key) {
-  
+    return;
-  TSKEY ts = pCtx->ptsList[index];
+  }
-  DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+
-  
+  if (pCtx->order == TSDB_ORDER_DESC) {
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    SET_VAL(pCtx, 1, 1);
-  pResInfo->hasResult = DATA_SET_FLAG;
+    memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
-  pResInfo->complete = true;  // set query completed
+
+    TSKEY ts = pCtx->ptsList[index];
+    DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    pResInfo->complete = true;  // set query completed
+  } else { // in case of ascending order check, all data needs to be checked
+    SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx);
+    TSKEY ts = pCtx->ptsList[index];
+
+    char* buf = GET_ROWCELL_INTERBUF(pResInfo);
+    if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) {
+      pResInfo->hasResult = DATA_SET_FLAG;
+      memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
+
+      *(TSKEY*)buf = ts;
+      DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+    }
+  }
 }
 
 static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) {
@@ -1712,7 +1730,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind
 
 static void last_dist_function(SQLFunctionCtx *pCtx) {
   /*
-   * 1. for scan data in asc order, no need to check data
+   * 1. for scan data is not the required order
    * 2. for data blocks that are not loaded, no need to check data
    */
   if (pCtx->order != pCtx->param[0].i64Key) {
@@ -2448,7 +2466,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
 static void percentile_function(SQLFunctionCtx *pCtx) {
   int32_t notNullElems = 0;
   
-  SResultRowCellInfo *    pResInfo = GET_RES_INFO(pCtx);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
 
   // the first stage, only acquire the min/max value
@@ -2549,12 +2567,14 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
   double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
   
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket;
+  SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo);
-  
+
-  if (pMemBucket->total > 0) {  // check for null
+  tMemBucket * pMemBucket = ppInfo->pMemBucket;
-    *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
+  if (pMemBucket == NULL || pMemBucket->total == 0) {  // check for null
-  } else {
+    assert(ppInfo->numOfElems == 0);
     setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+  } else {
+    *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
   }
   
   tMemBucketDestroy(pMemBucket);
@@ -3624,94 +3644,158 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
     return false;
   }
   
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);  //->aOutputBuf + pCtx->outputBytes;
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  STwaInfo *   pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
-  
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-  pInfo->lastKey = INT64_MIN;
+  pInfo->lastKey  = INT64_MIN;
-  pInfo->type = pCtx->inputType;
+  pInfo->win      = TSWINDOW_INITIALIZER;
-  
   return true;
 }
 
-static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, int32_t i, STwaInfo *pInfo) {
+static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
-  switch (pCtx->inputType) {
+  int32_t notNullElems = 0;
-    case TSDB_DATA_TYPE_INT:
+  TSKEY *primaryKey = pCtx->ptsList;
-      pInfo->iLastValue = GET_INT32_VAL(data + pCtx->inputBytes * i);
+  
-      break;
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-    case TSDB_DATA_TYPE_TINYINT:
+
-      pInfo->iLastValue = GET_INT8_VAL(data + pCtx->inputBytes * i);
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-      break;
+
-    case TSDB_DATA_TYPE_SMALLINT:
+  int32_t i = index;
-      pInfo->iLastValue = GET_INT16_VAL(data + pCtx->inputBytes * i);
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
-      break;
+
-    case TSDB_DATA_TYPE_BIGINT:
+  if (pCtx->start.key != INT64_MIN) {
-      pInfo->iLastValue = GET_INT64_VAL(data + pCtx->inputBytes * i);
+    assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
-      break;
+               (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
-    case TSDB_DATA_TYPE_FLOAT:
+
-      pInfo->dLastValue = GET_FLOAT_VAL(data + pCtx->inputBytes * i);
+    assert(pInfo->lastKey == INT64_MIN);
-      break;
+
-    case TSDB_DATA_TYPE_DOUBLE:
+    pInfo->lastKey = primaryKey[tsIndex + i];
-      pInfo->dLastValue = GET_DOUBLE_VAL(data + pCtx->inputBytes * i);
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
-      break;
+
-    default:
+    pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
-      assert(0);
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += step;
+  } else if (pInfo->lastKey == INT64_MIN) {
+    pInfo->lastKey = primaryKey[tsIndex + i];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pInfo->lastKey;
+    notNullElems++;
+    i += step;
   }
+
+  // calculate the value of
+  switch(pCtx->inputType) {
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = (double) val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
+      }
+      break;
+    }
+    default: assert(0);
+  }
+
+  // the last interpolated time window value
+  if (pCtx->end.key != INT64_MIN) {
+    pInfo->dOutput  += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
+    pInfo->lastValue = pCtx->end.val;
+    pInfo->lastKey   = pCtx->end.key;
+  }
+
+  pInfo->win.ekey  = pInfo->lastKey;
+  return notNullElems;
 }
 
 static void twa_function(SQLFunctionCtx *pCtx) {
   void * data = GET_INPUT_CHAR(pCtx);
-  TSKEY *primaryKey = pCtx->ptsList;
+
-  
-  int32_t notNullElems = 0;
-  
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   STwaInfo *   pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   
-  int32_t i = 0;
-  
   // skip null value
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1);
   while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
-    i++;
+    i += step;
   }
-  
+
-  if (i >= pCtx->size) {
+  int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size);
-    return;
-  }
-  
-  if (pInfo->lastKey == INT64_MIN) {
-    pInfo->lastKey = pCtx->nStartQueryTimestamp;
-    setTWALastVal(pCtx, data, i, pInfo);
-    
-    pInfo->hasResult = DATA_SET_FLAG;
-  }
-  
-  notNullElems++;
-  
-  if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
-    pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
-  } else {
-    pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
-  }
-  
-  pInfo->lastKey = primaryKey[i];
-  setTWALastVal(pCtx, data, i, pInfo);
-  
-  for (++i; i < pCtx->size; i++) {
-    if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
-      continue;
-    }
-    
-    notNullElems++;
-    if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
-      pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
-    } else {
-      pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
-    }
-    
-    pInfo->lastKey = primaryKey[i];
-    setTWALastVal(pCtx, data, i, pInfo);
-  }
-  
   SET_VAL(pCtx, notNullElems, 1);
   
   if (notNullElems > 0) {
@@ -3721,8 +3805,6 @@ static void twa_function(SQLFunctionCtx *pCtx) {
   if (pCtx->stableQuery) {
     memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo));
   }
-  
-  //  pCtx->numOfIteratedElems += notNullElems;
 }
 
 static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
@@ -3730,34 +3812,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
-  SET_VAL(pCtx, 1, 1);
+  int32_t notNullElems = 0;
-  
   TSKEY *primaryKey = pCtx->ptsList;
-  
+
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
   STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-  
+  int32_t i = pCtx->startOffset;
-  if (pInfo->lastKey == INT64_MIN) {
+  int32_t size = pCtx->size;
-    pInfo->lastKey = pCtx->nStartQueryTimestamp;
+
-    setTWALastVal(pCtx, pData, 0, pInfo);
+  if (pCtx->start.key != INT64_MIN) {
-    
+    assert(pInfo->lastKey == INT64_MIN);
+
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
+
     pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += 1;
+  } else if (pInfo->lastKey == INT64_MIN) {
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pInfo->lastKey;
+    notNullElems++;
+    i += 1;
   }
-  
+
-  if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+  // calculate the value of
-    pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey);
+  switch(pCtx->inputType) {
-  } else {
+    case TSDB_DATA_TYPE_TINYINT: {
-    pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey);
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = (double) val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    default: assert(0);
   }
-  
+
-  // record the last key/value
+  // the last interpolated time window value
-  pInfo->lastKey = primaryKey[index];
+  if (pCtx->end.key != INT64_MIN) {
-  setTWALastVal(pCtx, pData, 0, pInfo);
+    pInfo->dOutput  += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
-  
+    pInfo->lastValue = pCtx->end.val;
-  //  pCtx->numOfIteratedElems += 1;
+    pInfo->lastKey   = pCtx->end.key;
-  pResInfo->hasResult = DATA_SET_FLAG;
+  }
-  
+
+  pInfo->win.ekey  = pInfo->lastKey;
+
+  SET_VAL(pCtx, notNullElems, 1);
+
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+
   if (pCtx->stableQuery) {
     memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
   }
@@ -3778,16 +3962,10 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
     }
     
     numOfNotNull++;
-    if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
+    pBuf->dOutput += pInput->dOutput;
-      pBuf->iOutput += pInput->iOutput;
+
-    } else {
+    pBuf->win = pInput->win;
-      pBuf->dOutput += pInput->dOutput;
-    }
-    
-    pBuf->SKey = pInput->SKey;
-    pBuf->EKey = pInput->EKey;
     pBuf->lastKey = pInput->lastKey;
-    pBuf->iLastValue = pInput->iLastValue;
   }
   
   SET_VAL(pCtx, numOfNotNull, 1);
@@ -3814,21 +3992,17 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   
   STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
-  assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
+  assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
   
   if (pInfo->hasResult != DATA_SET_FLAG) {
     setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
     return;
   }
   
-  if (pInfo->SKey == pInfo->EKey) {
+  if (pInfo->win.ekey == pInfo->win.skey) {
-    *(double *)pCtx->aOutputBuf = 0;
+    *(double *)pCtx->aOutputBuf = pInfo->lastValue;
-  } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) {
-    pInfo->iOutput += pInfo->iLastValue * (pInfo->EKey - pInfo->lastKey);
-    *(double *)pCtx->aOutputBuf = pInfo->iOutput / (double)(pInfo->EKey - pInfo->SKey);
   } else {
-    pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey);
+    *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey);
-    *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey);
   }
   
   GET_RES_INFO(pCtx)->numOfRes = 1;
@@ -4387,11 +4561,11 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
  *
  */
 int32_t functionCompatList[] = {
-    // count,       sum,      avg,       min,      max,  stddev,    percentile, apercentile, first,   last
+    // count,   sum,      avg,       min,      max,    stddev,    percentile,   apercentile, first,   last
-    1,          1,        1,         1,        1,      1,          1,           1,        1,      1,
+    1,          1,        1,         1,        1,      1,          1,           1,           1,      1,
-    // last_row,    top,    bottom,     spread,    twa,  leastsqr,     ts,       ts_dummy, tag_dummy, ts_z
+    // last_row,top,      bottom,    spread,   twa,    leastsqr,   ts,          ts_dummy, tag_dummy, ts_z
     4,         -1,       -1,         1,        1,      1,          1,           1,        1,     -1,
-    //  tag,       colprj,  tagprj,   arithmetic, diff, first_dist, last_dist,    interp      rate   irate
+    //  tag,    colprj,   tagprj,    arithmetic, diff, first_dist, last_dist,   interp    rate    irate
     1,          1,        1,         1,       -1,      1,          1,           5,        1,      1,
     // sum_rate, sum_irate, avg_rate, avg_irate
     1,          1,        1,         1,

src/client/src/tscLocal.c

@@ -825,8 +825,11 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) {
 static int32_t tscProcessServStatus(SSqlObj *pSql) {
   STscObj* pObj = pSql->pTscObj;
 
-  if (pObj->pHb != NULL) {
+  SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-    if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+  if (pHb != NULL) {
+    int32_t code = pHb->res.code;
+    taosReleaseRef(tscObjRef, pObj->hbrid);
+    if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
       pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
       return pSql->res.code;
     }

src/client/src/tscLocalMerge.c

@@ -198,6 +198,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   if (numOfFlush == 0 || numOfBuffer == 0) {
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer);
+    pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty
     tscDebug("%p retrieved no data", pSql);
     return;
   }
@@ -330,22 +331,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16;
   pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage));
 
-  pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList);
   pReducer->resColModel = finalmodel;
   pReducer->resColModel->capacity = pReducer->nResultBufSize;
-
   pReducer->finalModel = pFFModel;
 
-  assert(pReducer->finalRowSize > 0);
+  if (finalmodel->rowSize > 0) {
-  if (pReducer->finalRowSize > 0) {
+    pReducer->resColModel->capacity /= finalmodel->rowSize;
-    pReducer->resColModel->capacity /= pReducer->finalRowSize;
   }
-  assert(pReducer->finalRowSize <= pReducer->rowSize);
 
+  assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pReducer->rowSize);
   pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
 
   if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
-      /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
+      pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
     tfree(pReducer->pTempBuffer);
     tfree(pReducer->discardData);
     tfree(pReducer->pResultBuf);
@@ -723,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   // final result depends on the fields number
   memset(pSchema, 0, sizeof(SSchema) * size);
+
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
 
-    SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    SSchema p1 = {0};
+    if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) {
+      p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    } else {
+      p1 = tGetTableNameColumnSchema();
+    }
 
     int32_t inter = 0;
     int16_t type = -1;
@@ -745,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
         functionId = TSDB_FUNC_LAST;
       }
 
-      getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      assert(ret == TSDB_CODE_SUCCESS);
     }
 
     pSchema[i].type = (uint8_t)type;
@@ -920,7 +925,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
     savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo);
   }
 
-  memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalRowSize));
+  memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize));
 
   pRes->numOfClauseTotal += pRes->numOfRows;
   pBeforeFillData->num = 0;
@@ -1256,7 +1261,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
   tColModelCompact(pModel, pResBuf, pModel->capacity);
 
   if (tscIsSecondStageQuery(pQueryInfo)) {
-    pLocalReducer->finalRowSize = doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize);
+    doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize);
   }
 
 #ifdef _DEBUG_VIEW
@@ -1627,7 +1632,8 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
 }
 
 int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
-  char* pbuf = calloc(1, pOutput->num * rowSize);
+  int32_t maxRowSize = MAX(rowSize, finalRowSize);
+  char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize));
 
   size_t size = tscNumOfFields(pQueryInfo);
   SArithmeticSupport arithSup = {0};
@@ -1654,17 +1660,16 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
       tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc);
     } else {
       SSqlExpr* pExpr = pSup->pSqlExpr;
-      memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num);
+      memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num));
     }
 
     offset += pSup->field.bytes;
   }
 
-  assert(finalRowSize <= rowSize);
+  memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset));
-  memcpy(pOutput->data, pbuf, pOutput->num * offset);
 
   tfree(pbuf);
   tfree(arithSup.data);
 
   return offset;
-}
+}

src/client/src/tscParseInsert.c

@@ -630,11 +630,17 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
   return TSDB_CODE_SUCCESS;
 }
 
-static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
+static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
   pBlocks->tid = pTableMeta->id.tid;
   pBlocks->uid = pTableMeta->id.uid;
   pBlocks->sversion = pTableMeta->sversion;
-  pBlocks->numOfRows += numOfRows;
+
+  if (pBlocks->numOfRows + numOfRows >= INT16_MAX) {
+    return TSDB_CODE_TSC_INVALID_SQL;
+  } else {
+    pBlocks->numOfRows += numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
 }
 
 // data block is disordered, sort it in ascending order
@@ -722,7 +728,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   }
 
   SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData);
-  tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", *str);
+    return code;
+  }
 
   dataBuf->vgId = pTableMeta->vgroupInfo.vgId;
   dataBuf->numOfTables = 1;
@@ -790,9 +800,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     sql += index;
 
     tscAllocPayload(pCmd, sizeof(STagData));
-    STagData *pTag = &pCmd->tagData;
-
-    memset(pTag, 0, sizeof(STagData));
     
     //the source super table is moved to the secondary position of the pTableMetaInfo list
     if (pQueryInfo->numOfTables < 2) {
@@ -805,7 +812,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return code;
     }
 
+    STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memset(pTag, 0, offsetof(STagData, data));
     tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
+    pCmd->pTagData = pTag;
+
     code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -934,7 +948,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     tdSortKVRowByColIdx(row);
-    pTag->dataLen = kvRowLen(row);
+
+    pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    pCmd->pTagData = pTag;
+    pTag->dataLen = htonl(kvRowLen(row));
     kvRowCpy(pTag->data, row);
     free(row);
 
@@ -945,8 +965,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z);
     }
 
-    pTag->dataLen = htonl(pTag->dataLen);
-
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }
@@ -1376,7 +1394,10 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
   STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
 
   SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData);
-  tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  if (code != TSDB_CODE_SUCCESS) {
+    return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL);
+  }
 
   if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
     return code;

src/client/src/tscProfile.c

@@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) {
     tscSlowQueryConnInitialized = true;
     tscSaveSlowQueryFp(sql, NULL);
   }
+  taos_free_result(result);
 }
 
 void tscAddIntoSqlList(SSqlObj *pSql) {
@@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) {
   } else {
     tscDebug("success to save slow query, code:%d", code);
   }
+  taos_free_result(result);
 }
 
 void tscSaveSlowQueryFp(void *handle, void *tmrId) {

src/client/src/tscSQLParser.c

@@ -2835,6 +2835,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
 
     if (functionCompatList[functionId] != factor) {
       return false;
+    } else {
+      if (factor == -1) { // two functions with the same -1 flag
+        return false;
+      }
     }
 
     if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) {
@@ -4248,7 +4252,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     tExprTreeDestroy(&p, NULL);
     
     taosArrayDestroy(colList);
-    if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+    if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table");
     }
   }
@@ -4256,6 +4260,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
   pCondExpr->pTagCond = NULL;
   return ret;
 }
+
 int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) {
   if (pExpr == NULL) {
     return TSDB_CODE_SUCCESS;
@@ -5102,7 +5107,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
   const int tokenDebugFlagEnd = 20;
   const SDNodeDynConfOption cfgOptions[] = {
       {"resetLog", 8},    {"resetQueryCache", 15},  {"balance", 7},     {"monitor", 7},
-      {"debugFlag", 9},   {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10},
+      {"debugFlag", 9},   {"monDebugFlag", 12},     {"vDebugFlag", 10}, {"mDebugFlag", 10},
       {"cDebugFlag", 10}, {"httpDebugFlag", 13},    {"qDebugflag", 10}, {"sdbDebugFlag", 12},
       {"uDebugFlag", 10}, {"tsdbDebugFlag", 13},    {"sDebugflag", 10}, {"rpcDebugFlag", 12},
       {"dDebugFlag", 10}, {"mqttDebugFlag", 13},    {"wDebugFlag", 10}, {"tmrDebugFlag", 12},
@@ -5306,15 +5311,18 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
 
     // keep original limitation value in globalLimit
     pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
-    pQueryInfo->prjOffset = pQueryInfo->limit.offset;
+    pQueryInfo->prjOffset   = pQueryInfo->limit.offset;
+    pQueryInfo->tableLimit  = -1;
 
     if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
       /*
-       * the limitation/offset value should be removed during retrieve data from virtual node,
+       * the offset value should be removed during retrieve data from virtual node, since the
-       * since the global order are done in client side, so the limitation should also
+       * global order are done in client side, so the offset is applied at the client side
-       * be done at the client side.
+       * However, note that the maximum allowed number of result for each table should be less
+       * than or equal to the value of limit.
        */
       if (pQueryInfo->limit.limit > 0) {
+        pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset;
         pQueryInfo->limit.limit = -1;
       }
 
@@ -6648,7 +6656,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
       
       return TSDB_CODE_SUCCESS;
     } else {
-      return TSDB_CODE_TSC_INVALID_SQL;
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
     }
     
   } else {

src/client/src/tscServer.c

@@ -26,7 +26,7 @@
 #include "ttimer.h"
 #include "tlockfree.h"
 
-SRpcCorEpSet  tscMgmtEpSet;
+///SRpcCorEpSet  tscMgmtEpSet;
 
 int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
 
@@ -73,10 +73,11 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
   assert(hasFqdn);
 }
 
-static void tscDumpMgmtEpSet(SRpcEpSet *epSet) {
+static void tscDumpMgmtEpSet(SSqlObj *pSql) {
-  taosCorBeginRead(&tscMgmtEpSet.version);
+  SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet;
-  *epSet = tscMgmtEpSet.epSet;
+  taosCorBeginRead(&pCorEpSet->version);
-  taosCorEndRead(&tscMgmtEpSet.version);
+  pSql->epSet = pCorEpSet->epSet;
+  taosCorEndRead(&pCorEpSet->version);
 }  
 static void tscEpSetHtons(SRpcEpSet *s) {
    for (int32_t i = 0; i < s->numOfEps; i++) {
@@ -94,11 +95,12 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) {
    }
    return true;
 }
-void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
+void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) {
   // no need to update if equal
-  taosCorBeginWrite(&tscMgmtEpSet.version);
+  SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet;
-  tscMgmtEpSet.epSet = *pEpSet;
+  taosCorBeginWrite(&pCorEpSet->version);
-  taosCorEndWrite(&tscMgmtEpSet.version);
+  pCorEpSet->epSet = *pEpSet;
+  taosCorEndWrite(&pCorEpSet->version);
 }
 static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
   if (pVgroupInfo == NULL) { return;}
@@ -133,18 +135,6 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
   taosCorEndWrite(&pVgroupInfo->version);
 }
 
-void tscPrintMgmtEp() {
-  SRpcEpSet dump;
-  tscDumpMgmtEpSet(&dump);
-  if (dump.numOfEps <= 0) {
-    tscError("invalid mnode EP list:%d", dump.numOfEps);
-  } else {
-    for (int i = 0; i < dump.numOfEps; ++i) {
-      tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]);
-    }
-  }
-}
-
 void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
   STscObj *pObj = (STscObj *)param;
   if (pObj == NULL) return;
@@ -162,7 +152,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
     SRpcEpSet *    epSet = &pRsp->epSet;
     if (epSet->numOfEps > 0) {
       tscEpSetHtons(epSet);
-      tscUpdateMgmtEpSet(epSet);
+      tscUpdateMgmtEpSet(pSql, epSet);
     }
 
     pSql->pTscObj->connId = htonl(pRsp->connId);
@@ -175,10 +165,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
       if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
     }
   } else {
-    tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
+    tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code));
   }
 
-  if (pObj->pHb != NULL) {
+  if (pObj->hbrid != 0) {
     int32_t waitingDuring = tsShellActivityTimer * 500;
     tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
 
@@ -191,22 +181,21 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
 void tscProcessActivityTimer(void *handle, void *tmrId) {
   int64_t rid = (int64_t) handle;
   STscObj *pObj = taosAcquireRef(tscRefId, rid);
-  if (pObj == NULL) return; 
+  if (pObj == NULL) {
-
-  SSqlObj* pHB = pObj->pHb;
-
-  void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
-  if (p == NULL) {
-    tscWarn("%p HB object has been released already", pHB);
-    taosReleaseRef(tscRefId, pObj->rid);
     return;
   }
 
-  assert(*pHB->self == pHB);
+  SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid);
+  if (pHB == NULL) {
+    taosReleaseRef(tscRefId, rid);
+    return;
+  }
+
+  assert(pHB->self == pObj->hbrid);
 
   pHB->retry = 0;
   int32_t code = tscProcessSql(pHB);
-  taosCacheRelease(tscObjCache, (void**) &p, false);
+  taosReleaseRef(tscObjRef, pObj->hbrid);
 
   if (code != TSDB_CODE_SUCCESS) {
     tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
@@ -227,7 +216,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 
   // set the mgmt ip list
   if (pSql->cmd.command >= TSDB_SQL_MGMT) {
-    tscDumpMgmtEpSet(&pSql->epSet);
+    tscDumpMgmtEpSet(pSql);
   }
 
   memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
@@ -236,7 +225,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .msgType = pSql->cmd.msgType,
       .pCont   = pMsg,
       .contLen = pSql->cmd.payloadLen,
-      .ahandle = pSql,
+      .ahandle = (void*)pSql->self,
       .handle  = NULL,
       .code    = 0
   };
@@ -247,26 +236,24 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle;
-  void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(TSDB_CACHE_PTR_TYPE));
+  SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle);
-  if (p == NULL) {
+  if (pSql == NULL) {
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
-
+  assert(pSql->self == handle);
-  SSqlObj* pSql = *p;
-  assert(pSql != NULL);
 
   STscObj *pObj = pSql->pTscObj;
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
 
-  assert(*pSql->self == pSql);
   pSql->rpcRid = -1;
 
   if (pObj->signature != pObj) {
     tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
 
-    taosCacheRelease(tscObjCache, (void**) &p, true);
+    taosRemoveRef(tscObjRef, pSql->self);
+    taosReleaseRef(tscObjRef, pSql->self);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
@@ -276,10 +263,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
         pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
 
-    void** p1 = p;
+    taosRemoveRef(tscObjRef, pSql->self);
-    taosCacheRelease(tscObjCache, (void**) &p1, false);
+    taosReleaseRef(tscObjRef, pSql->self);
-
-    taosCacheRelease(tscObjCache, (void**) &p, true);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
@@ -289,7 +274,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
       if (pCmd->command < TSDB_SQL_MGMT) {
         tscUpdateVgroupInfo(pSql, pEpSet);
       } else {
-        tscUpdateMgmtEpSet(pEpSet);
+        tscUpdateMgmtEpSet(pSql, pEpSet);
       }
     }
   }
@@ -322,7 +307,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
 
       // if there is an error occurring, proceed to the following error handling procedure.
       if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
-        taosCacheRelease(tscObjCache, (void**) &p, false);
+        taosReleaseRef(tscObjRef, pSql->self);
         rpcFreeCont(rpcMsg->pCont);
         return;
       }
@@ -390,11 +375,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
   }
 
-  void** p1 = p;
+  taosReleaseRef(tscObjRef, pSql->self);
-  taosCacheRelease(tscObjCache, (void**) &p1, false);
 
   if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
-    taosCacheRelease(tscObjCache, (void **)&p, true);
+    taosRemoveRef(tscObjRef, pSql->self);
     tscDebug("%p sqlObj is automatically freed", pSql);
   }
 
@@ -697,6 +681,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
   pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->queryType      = htonl(pQueryInfo->type);
+  pQueryMsg->tableLimit     = htobe64(pQueryInfo->tableLimit);
   
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number
@@ -1565,11 +1550,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
 
-  size_t len = htonl(pCmd->tagData.dataLen);
+  if (pCmd->autoCreated && pCmd->pTagData != NULL) {
-  if (pSql->cmd.autoCreated) {
+    int len = htonl(pCmd->pTagData->dataLen);
     if (len > 0) {
-      len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen);
+      len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen);
-      memcpy(pInfoMsg->tags, &pCmd->tagData, len);
+      memcpy(pInfoMsg->tags, pCmd->pTagData, len);
       pMsg += len;
     }
   }
@@ -2020,7 +2005,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
 
 // TODO multithread problem
 static void createHBObj(STscObj* pObj) {
-  if (pObj->pHb != NULL) {
+  if (pObj->hbrid != 0) {
     return;
   }
 
@@ -2052,7 +2037,7 @@ static void createHBObj(STscObj* pObj) {
   registerSqlObj(pSql);
   tscDebug("%p HB is allocated, pObj:%p", pSql, pObj);
 
-  pObj->pHb = pSql;
+  pObj->hbrid = pSql->self;
 }
 
 int tscProcessConnectRsp(SSqlObj *pSql) {
@@ -2070,7 +2055,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   
   if (pConnect->epSet.numOfEps > 0) {
     tscEpSetHtons(&pConnect->epSet);
-    tscUpdateMgmtEpSet(&pConnect->epSet);
+    tscUpdateMgmtEpSet(pSql, &pConnect->epSet);
   } 
 
   strcpy(pObj->sversion, pConnect->serverVersion);
@@ -2239,8 +2224,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->signature = pNew;
   pNew->cmd.command = TSDB_SQL_META;
 
-  registerSqlObj(pNew);
-
   tscAddSubqueryInfo(&pNew->cmd);
 
   SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
@@ -2248,8 +2231,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->cmd.autoCreated = pSql->cmd.autoCreated;  // create table if not exists
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
     tscError("%p malloc failed for payload to get table meta", pSql);
-    free(pNew);
+    tscFreeSqlObj(pNew);
-
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -2257,12 +2239,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
 
   tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
-  memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p malloc failed for new tag data to get table meta", pSql);
+      tscFreeSqlObj(pNew);
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
+
   tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
 
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
 
+  registerSqlObj(pNew);
+
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated

src/client/src/tscSql.c

@@ -58,6 +58,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH;
     return NULL;
   }
+  SRpcCorEpSet corMgmtEpSet;
 
   char secretEncrypt[32] = {0};
   int  secretEncryptLen = 0;
@@ -82,11 +83,13 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     }
     secretEncryptLen = outlen;
   }
-
+  
   if (ip) {
-    if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL;
+    if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL;
-    if (port) tscMgmtEpSet.epSet.port[0] = port;
+    if (port) corMgmtEpSet.epSet.port[0] = port;
-  } 
+  } else {
+    if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL;
+  }
  
   void *pDnodeConn = NULL;
   if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) {
@@ -100,10 +103,20 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     rpcClose(pDnodeConn);
     return NULL;
   }
+  // set up tscObj's mgmtEpSet
+  pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet));
+  if (NULL == pObj->tscCorMgmtEpSet) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    rpcClose(pDnodeConn);
+    free(pObj->tscCorMgmtEpSet);
+    free(pObj);
+  }
+  memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet));
 
   pObj->signature = pObj;
   pObj->pDnodeConn = pDnodeConn;
   T_REF_INIT_VAL(pObj, 1);
+  
 
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@@ -115,6 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     if (len >= TSDB_DB_NAME_LEN) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       rpcClose(pDnodeConn);
+      free(pObj->tscCorMgmtEpSet);
       free(pObj);
       return NULL;
     }
@@ -132,6 +146,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
   if (NULL == pSql) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
+    free(pObj->tscCorMgmtEpSet);
     free(pObj);
     return NULL;
   }
@@ -149,6 +164,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
     free(pSql);
+    free(pObj->tscCorMgmtEpSet);
     free(pObj);
     return NULL;
   }
@@ -276,15 +292,18 @@ void taos_close(TAOS *taos) {
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
 
-  SSqlObj* pHb = pObj->pHb;
+  if (pObj->hbrid > 0) {
-  if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
+    SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
+    if (pHb != NULL) {
-      rpcCancelRequest(pHb->rpcRid);
+      if (pHb->rpcRid > 0) {  // wait for rsp from dnode
-      pHb->rpcRid = -1;
+        rpcCancelRequest(pHb->rpcRid);
-    }
+        pHb->rpcRid = -1;
+      }
 
-    tscDebug("%p HB is freed", pHb);
+      tscDebug("%p HB is freed", pHb);
-    taos_free_result(pHb);
+      taos_free_result(pHb);
+      taosReleaseRef(tscObjRef, pHb->self);
+    }
   }
 
   int32_t ref = T_REF_DEC(pObj);
@@ -606,8 +625,7 @@ void taos_free_result(TAOS_RES *res) {
   bool freeNow = tscKillQueryInDnode(pSql);
   if (freeNow) {
     tscDebug("%p free sqlObj in cache", pSql);
-    SSqlObj** p = pSql->self;
+    taosReleaseRef(tscObjRef, pSql->self);
-    taosCacheRelease(tscObjCache, (void**) &p, true);
   }
 }
 
@@ -700,13 +718,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
       continue;
     }
 
-    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
+    SSqlObj* pSubObj = pSub;
-    if (p == NULL) {
-      continue;
-    }
-
-    SSqlObj* pSubObj = (SSqlObj*) (*p);
-    assert(pSubObj->self == (SSqlObj**) p);
 
     pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
     if (pSubObj->rpcRid > 0) {
@@ -715,7 +727,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
     }
 
     tscQueueAsyncRes(pSubObj);
-    taosCacheRelease(tscObjCache, (void**) &p, false);
+    taosReleaseRef(tscObjRef, pSubObj->self);
   }
 
   tscDebug("%p super table query cancelled", pSql);

src/client/src/tscSub.c

@@ -157,7 +157,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
 
   registerSqlObj(pSql);
 
-  code = tsParseSql(pSql, false);
+  code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     tsem_wait(&pSub->sem);
     code = pSql->res.code;
@@ -168,7 +168,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
     goto fail;
   }
 
-  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+  if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
     line = __LINE__;
     code = TSDB_CODE_TSC_INVALID_SQL;
     goto fail;
@@ -179,10 +179,10 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
 fail:
   tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code));
   if (pSql != NULL) {
-    if (pSql->self != NULL) {
+    if (pSql->self != 0) {
-      taos_free_result(pSql);
+      taosReleaseRef(tscObjRef, pSql->self);
     } else {
-      tscFreeSqlObj(pSql);
+    tscFreeSqlObj(pSql);
     }
 
     pSql = NULL;
@@ -401,9 +401,11 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
     tscLoadSubscriptionProgress(pSub);
   }
 
-  if (!tscUpdateSubscription(pObj, pSub)) {
+  if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) {
-    taos_unsubscribe(pSub, 1);
+    if (!tscUpdateSubscription(pObj, pSub)) {
-    return NULL;
+      taos_unsubscribe(pSub, 1);
+      return NULL;
+    }
   }
 
   pSub->interval = interval;
@@ -417,10 +419,80 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
   return pSub;
 }
 
+SSqlObj* recreateSqlObj(SSub* pSub) {
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  if (pSql == NULL) {
+    return NULL;
+  }
+
+  pSql->signature = pSql;
+  pSql->pTscObj = pSub->taos;
+
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pSql->param = pSub;
+  pSql->maxRetry = TSDB_MAX_REPLICA;
+  pSql->fp = asyncCallback;
+  pSql->fetchFp = asyncCallback;
+  pSql->sqlstr = strdup(pSub->pSql->sqlstr);
+  if (pSql->sqlstr == NULL) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pRes->qhandle = 0;
+  pRes->numOfRows = 1;
+
+  int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  registerSqlObj(pSql);
+
+  code = tsParseSql(pSql, true);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    tsem_wait(&pSub->sem);
+    code = pSql->res.code;
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  return pSql;
+}
+
 TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   SSub *pSub = (SSub *)tsub;
   if (pSub == NULL) return NULL;
 
+  if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
+    SSqlObj* pSql = recreateSqlObj(pSub);
+    if (pSql == NULL) {
+      return NULL;
+    }
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
+    pSub->pSql = pSql;
+    pSql->pSubscription = pSub;
+  }
+
   tscSaveSubscriptionProgress(pSub);
 
   SSqlObj *pSql = pSub->pSql;
@@ -442,7 +514,12 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
 
   size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo);
   size += sizeof(SQueryTableMsg) + 4096;
-  tscAllocPayload(&pSql->cmd, (int)size);
+  int code = tscAllocPayload(&pSql->cmd, (int)size);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscError("failed to alloc payload");
+    return NULL;
+  }
+
   for (int retry = 0; retry < 3; retry++) {
     tscRemoveFromSqlList(pSql);
 
@@ -507,10 +584,13 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   if (pSub->pSql != NULL) {
-    taos_free_result(pSub->pSql);
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
   }
 
-  tscFreeSqlObj(pSub->pSql);
   taosArrayDestroy(pSub->progress);
   tsem_destroy(&pSub->sem);
   memset(pSub, 0, sizeof(*pSub));

src/client/src/tscSubquery.c

@@ -2198,6 +2198,9 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
   STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
   int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
 
+  // free the data block created from insert sql string
+  pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
+
   if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
     tscQueueAsyncRes(pSql);
     return code;  // here the pSql may have been released already.

src/client/src/tscSystem.c

@@ -15,7 +15,7 @@
 
 #include "os.h"
 #include "taosmsg.h"
-#include "tcache.h"
+#include "tref.h"
 #include "trpc.h"
 #include "tsystem.h"
 #include "ttimer.h"
@@ -31,7 +31,7 @@
 
 // global, not configurable
 SCacheObj*  tscMetaCache;
-SCacheObj*  tscObjCache;
+int     tscObjRef = -1;
 void *  tscTmr;
 void *  tscQhandle;
 void *  tscCheckDiskUsageTmr;
@@ -116,11 +116,6 @@ void taos_init_imp(void) {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
   }
 
-  if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) {
-    tscError("failed to init mnode EP list");
-    return;
-  } 
-
   tscInitMsgsFp();
   int queueSize = tsMaxConnections*2;
 
@@ -144,7 +139,7 @@ void taos_init_imp(void) {
   int64_t refreshTime = 10; // 10 seconds by default
   if (tscMetaCache == NULL) {
     tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
+    tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
   }
 
   tscRefId = taosOpenRef(200, tscCloseTscObj);
@@ -167,9 +162,9 @@ void taos_cleanup(void) {
     taosCacheCleanup(m);
   }
 
-  m = tscObjCache;
+  int refId = atomic_exchange_32(&tscObjRef, -1);
-  if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+  if (refId != -1) {
-    taosCacheCleanup(m);
+    taosCloseRef(refId);
   }
 
   m = tscQhandle;

src/client/src/tscUtil.c

@@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
   pCmd->pTableList = NULL;
   
   pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-  
+
   tscFreeQueryInfo(pCmd, removeFromCache);
 }
 
@@ -447,20 +447,18 @@ static void tscFreeSubobj(SSqlObj* pSql) {
 void tscFreeRegisteredSqlObj(void *pSql) {
   assert(pSql != NULL);
 
-  SSqlObj** p = (SSqlObj**)pSql;
+  SSqlObj* p = *(SSqlObj**)pSql;
-  STscObj* pTscObj = (*p)->pTscObj;
+  STscObj* pTscObj = p->pTscObj;
 
-  assert((*p)->self != 0 && (*p)->self == (p));
+  assert(p->self != 0);
-
+  tscFreeSqlObj(p);
-  SSqlObj* ptr = *p;
-  tscFreeSqlObj(*p);
 
   int32_t ref = T_REF_DEC(pTscObj);
   assert(ref >= 0);
 
-  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref);
+  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref);
   if (ref == 0) {
-    tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj);
+    tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj);
     taosRemoveRef(tscRefId, pTscObj->rid);
   }
 }
@@ -512,6 +510,8 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   tscResetSqlCmdObj(pCmd, false);
 
+  tfree(pCmd->pTagData);
+  
   memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
   pCmd->allocSize = 0;
@@ -838,7 +838,6 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
 
     // the length does not include the SSubmitBlk structure
     pBlocks->dataLen = htonl(finalLen);
-
     dataBuf->numOfTables += 1;
   }
 
@@ -865,7 +864,7 @@ void tscCloseTscObj(void *param) {
     rpcClose(pObj->pDnodeConn);
     pObj->pDnodeConn = NULL;
   }
-
+  tfree(pObj->tscCorMgmtEpSet);
   pthread_mutex_destroy(&pObj->mutex);
 
   tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
@@ -1563,19 +1562,6 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) {
   }
 }
 
-void tscSetFreeHeatBeat(STscObj* pObj) {
-  if (pObj == NULL || pObj->signature != pObj || pObj->pHb == NULL) {
-    return;
-  }
-
-  SSqlObj* pHeatBeat = pObj->pHb;
-  assert(pHeatBeat == pHeatBeat->signature);
-
-  // to denote the heart-beat timer close connection and free all allocated resources
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHeatBeat->cmd, 0);
-  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-}
-
 /*
  * the following four kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
@@ -1594,7 +1580,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
   }
   
   STscObj* pTscObj = pSql->pTscObj;
-  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
+  if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) {
     return false;
   }
 
@@ -1886,13 +1872,10 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
 }
 
 void registerSqlObj(SSqlObj* pSql) {
-  int32_t DEFAULT_LIFE_TIME = 2 * 600 * 1000;  // 1200 sec
-
   int32_t ref = T_REF_INC(pSql->pTscObj);
   tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref);
 
-  TSDB_CACHE_PTR_TYPE p = (TSDB_CACHE_PTR_TYPE) pSql;
+  pSql->self = taosAddRef(tscObjRef, pSql);
-  pSql->self = taosCachePut(tscObjCache, &p, sizeof(TSDB_CACHE_PTR_TYPE), &p, sizeof(TSDB_CACHE_PTR_TYPE), DEFAULT_LIFE_TIME);
 }
 
 SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {
@@ -1909,7 +1892,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   pCmd->command = cmd;
   pCmd->parseFinished = 1;
   pCmd->autoCreated = pSql->cmd.autoCreated;
-  memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0);
+      free(pNew);
+      return NULL;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
@@ -2447,10 +2440,10 @@ char* strdup_throw(const char* str) {
   return p;
 }
 
-int tscSetMgmtEpSetFromCfg(const char *first, const char *second) {
+int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) {
+  corMgmtEpSet->version = 0;
   // init mgmt ip set 
-  tscMgmtEpSet.version = 0;
+  SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet);
-  SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet);
   mgmtEpSet->numOfEps = 0;
   mgmtEpSet->inUse = 0;
 

src/common/inc/tglobal.h

@@ -35,6 +35,7 @@ extern int32_t  tsNumOfMnodes;
 extern int32_t  tsEnableVnodeBak;
 extern int32_t  tsEnableTelemetryReporting;
 extern char     tsEmail[];
+extern char     tsArbitrator[];
 
 // common
 extern int      tsRpcTimer;
@@ -56,6 +57,7 @@ extern char     tsTempDir[];
 
 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer for each data node during query processing
+extern int32_t  tsHalfCoresForQuery;         // only 50% will be used in query processing
 
 // client
 extern int32_t tsTableMetaKeepTimer;
@@ -125,6 +127,9 @@ extern char    tsMonitorDbName[];
 extern char    tsInternalPass[];
 extern int32_t tsMonitorInterval;
 
+// stream
+extern int32_t tsEnableStream;
+
 // internal
 extern int32_t tsPrintAuth;
 extern int32_t tscEmbedded;
@@ -176,7 +181,7 @@ extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
 extern int32_t mqttDebugFlag;
-extern int32_t monitorDebugFlag;
+extern int32_t monDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t odbcDebugFlag;

src/common/inc/tname.h

@@ -25,6 +25,8 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
+size_t tableIdPrefix(const char* name, char* prefix, int32_t len);
+
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
 SSchema tGetTableNameColumnSchema();

src/common/src/tglobal.c

@@ -107,6 +107,9 @@ int64_t tsMaxRetentWindow = 24 * 3600L;  // maximum time window tolerance
 // positive value (in MB)
 int32_t tsQueryBufferSize = -1;
 
+// only 50% cpu will be used in query processing in dnode
+int32_t tsHalfCoresForQuery = 0;
+
 // db parameters
 int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
 int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
@@ -161,6 +164,9 @@ char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+// stream
+int32_t tsEnableStream = 1;
+
 // internal
 int32_t tsPrintAuth = 0;
 int32_t tscEmbedded = 0;
@@ -200,13 +206,13 @@ int32_t tsNumOfLogLines = 10000000;
 int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
 int32_t dDebugFlag = 135;
-int32_t vDebugFlag = 131;
+int32_t vDebugFlag = 135;
 int32_t cDebugFlag = 131;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
 int32_t mqttDebugFlag = 131;
-int32_t monitorDebugFlag = 131;
+int32_t monDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;
@@ -216,9 +222,9 @@ int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 135;
 
-int32_t (*monitorStartSystemFp)() = NULL;
+int32_t (*monStartSystemFp)() = NULL;
-void (*monitorStopSystemFp)() = NULL;
+void (*monStopSystemFp)() = NULL;
-void (*monitorExecuteSQLFp)(char *sql) = NULL;
+void (*monExecuteSQLFp)(char *sql) = NULL;
 
 char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
 
@@ -235,7 +241,7 @@ void taosSetAllDebugFlag() {
     odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
     mqttDebugFlag = debugFlag;
-    monitorDebugFlag = debugFlag;
+    monDebugFlag = debugFlag;
     qDebugFlag = debugFlag;    
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -276,15 +282,15 @@ bool taosCfgDynamicOptions(char *msg) {
 
     if (strncasecmp(cfg->option, "monitor", olen) == 0) {
       if (1 == vint) {
-        if (monitorStartSystemFp) {
+        if (monStartSystemFp) {
-          (*monitorStartSystemFp)();
+          (*monStartSystemFp)();
           uInfo("monitor is enabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
         }
       } else {
-        if (monitorStopSystemFp) {
+        if (monStopSystemFp) {
-          (*monitorStopSystemFp)();
+          (*monStopSystemFp)();
           uInfo("monitor is disabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
@@ -307,8 +313,8 @@ bool taosCfgDynamicOptions(char *msg) {
   }
 
   if (strncasecmp(option, "resetQueryCache", 15) == 0) {
-    if (monitorExecuteSQLFp) {
+    if (monExecuteSQLFp) {
-      (*monitorExecuteSQLFp)("resetQueryCache");
+      (*monExecuteSQLFp)("resetQueryCache");
       uInfo("resetquerycache is executed");
     } else {
       uError("resetquerycache can't be executed, for monitor not started");
@@ -881,6 +887,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_BYTE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "halfCoresForQuery";
+  cfg.ptr = &tsHalfCoresForQuery;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // locale & charset
   cfg.option = "timezone";
   cfg.ptr = tsTimezone;
@@ -1015,6 +1031,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "stream";
+  cfg.ptr = &tsEnableStream;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "httpEnableRecordSql";
   cfg.ptr = &tsHttpEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1227,8 +1253,8 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "monitorDebugFlag";
+  cfg.option = "monDebugFlag";
-  cfg.ptr = &monitorDebugFlag;
+  cfg.ptr = &monDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
@@ -1277,7 +1303,7 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "tscEnableRecordSql";
+  cfg.option = "enableRecordSql";
   cfg.ptr = &tsTscEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;

src/common/src/tname.c

@@ -39,6 +39,13 @@ char* extractDBName(const char* tableId, char* name) {
   return strncpy(name, &tableId[offset1 + 1], len);
 }
 
+size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
+  tstrncpy(prefix, name, len);
+  strcat(prefix, TS_PATH_DELIMITER);
+
+  return strlen(prefix);
+}
+
 SSchema tGetTableNameColumnSchema() {
   SSchema s = {0};
   s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE;
@@ -198,4 +205,4 @@ SSchema tscGetTbnameColumnSchema() {
 
   strcpy(s.name, TSQL_TBNAME_L);
   return s;
-}
+}

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.13</version>
+  <version>2.0.14</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.13</version>
+    <version>2.0.14</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>

src/connector/python/linux/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python2/taos/__init__.py

@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 

src/connector/python/linux/python2/taos/cinterface.py

@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):

src/connector/python/linux/python2/taos/cursor.py

@@ -26,7 +26,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -234,7 +234,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)

src/connector/python/linux/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python3/taos/__init__.py

@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 
@@ -21,4 +21,4 @@ def connect(*args, **kwargs):
 
     @rtype: TDengineConnector
     """
-    return TDengineConnection(*args, **kwargs)
+    return TDengineConnection(*args, **kwargs)

src/connector/python/linux/python3/taos/cinterface.py

@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):

src/connector/python/linux/python3/taos/cursor.py

@@ -27,7 +27,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -242,7 +242,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)

src/connector/python/windows/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python2/taos/__init__.py

@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 

src/connector/python/windows/python2/taos/cinterface.py

@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):

src/connector/python/windows/python2/taos/cursor.py

@@ -27,7 +27,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -193,7 +193,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)

src/connector/python/windows/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python3/taos/__init__.py

@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 

src/connector/python/windows/python3/taos/cinterface.py

@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -108,6 +109,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):

src/connector/python/windows/python3/taos/cursor.py

@@ -28,7 +28,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -194,7 +194,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)

src/cq/src/cqMain.c

@@ -40,15 +40,14 @@
 
 typedef struct {
   int32_t  vgId;
+  int32_t  master;
+  int32_t  num;      // number of continuous streams
   char     user[TSDB_USER_LEN];
   char     pass[TSDB_PASSWORD_LEN];
   char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
-  void    *ahandle;
-  int32_t  num;      // number of continuous streams
   struct SCqObj *pHead;
   void    *dbConn;
-  int32_t  master;
   void    *tmrCtrl;
   pthread_mutex_t mutex;
 } SCqContext;
@@ -70,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
 static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
 
 void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = calloc(sizeof(SCqContext), 1);
   if (pContext == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -90,7 +92,6 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   tstrncpy(pContext->db, db, sizeof(pContext->db));
   pContext->vgId = pCfg->vgId;
   pContext->cqWrite = pCfg->cqWrite;
-  pContext->ahandle = ahandle;
   tscEmbedded = 1;
 
   pthread_mutex_init(&pContext->mutex, NULL);
@@ -101,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
 }
 
 void cqClose(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (handle == NULL) return;
 
@@ -131,6 +135,9 @@ void cqClose(void *handle) {
 }
 
 void cqStart(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (pContext->dbConn || pContext->master) return;
 
@@ -149,6 +156,9 @@ void cqStart(void *handle) {
 }
 
 void cqStop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   cInfo("vgId:%d, stop all CQs", pContext->vgId);
   if (pContext->dbConn == NULL || pContext->master == 0) return;
@@ -176,6 +186,9 @@ void cqStop(void *handle) {
 }
 
 void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = handle;
 
   SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -205,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
 }
 
 void cqDrop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqObj *pObj = handle;
   SCqContext *pContext = pObj->pContext;
 
@@ -241,8 +257,12 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
   SCqObj* pObj = (SCqObj*)param;
   SCqContext* pContext = pObj->pContext;
   SSqlObj* pSql = (SSqlObj*)result;
-  pContext->dbConn = pSql->pTscObj;
+  if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) {
+    taos_close(pSql->pTscObj);
+  }
+  pthread_mutex_lock(&pContext->mutex);
   cqCreateStream(pContext, pObj);
+  pthread_mutex_unlock(&pContext->mutex);
 }
 
 static void cqProcessCreateTimer(void *param, void *tmrId) {
@@ -253,7 +273,9 @@ static void cqProcessCreateTimer(void *param, void *tmrId) {
     cDebug("vgId:%d, try connect to TDengine", pContext->vgId);
     taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL);
   } else {
+    pthread_mutex_lock(&pContext->mutex);
     cqCreateStream(pContext, pObj);
+    pthread_mutex_unlock(&pContext->mutex);
   }
 }
 
@@ -267,12 +289,14 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
   }
   pObj->tmrId = 0;
 
-  pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
+  if (pObj->pStream == NULL) {
-  if (pObj->pStream) {
+    pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
-    pContext->num++;
+    if (pObj->pStream) {
-    cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
+      pContext->num++;
-  } else {
+      cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
-    cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
+    } else {
+      cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
+    }
   }
 }
 
@@ -334,7 +358,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   pHead->version = 0;
 
   // write into vnode write queue
-  pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
+  pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL);
   free(buffer);
 }
 

src/cq/test/cqtest.c

@@ -24,7 +24,7 @@
 int64_t  ver = 0;
 void    *pCq = NULL;
 
-int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { 
+int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) { 
   return 0;
 }
 

src/dnode/CMakeLists.txt

@@ -12,7 +12,7 @@ AUX_SOURCE_DIRECTORY(src SRC)
 
 IF (TD_LINUX)
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
 
   IF (TD_SOMODE_STATIC)
     TARGET_LINK_LIBRARIES(taosd taos_static)
@@ -32,10 +32,6 @@ IF (TD_LINUX)
     TARGET_LINK_LIBRARIES(taosd mqtt)
   ENDIF ()
 
-  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
   SET(PREPARE_ENV_CMD "prepare_env_cmd")
   SET(PREPARE_ENV_TARGET "prepare_env_target")
   ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

src/dnode/inc/dnodeCfg.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCfg();
 void    dnodeCleanupCfg();

src/dnode/inc/dnodeCheck.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCheck();
 void    dnodeCleanupCheck();

src/dnode/inc/dnodeEps.h

@@ -19,8 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#include "dnodeInt.h"
-#include "taosmsg.h"
 
 int32_t dnodeInitEps();
 void    dnodeCleanupEps();

src/dnode/inc/dnodeInt.h

@@ -19,8 +19,13 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#include "taoserror.h"
+#include "taosmsg.h"
 #include "tlog.h"
+#include "trpc.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "vnode.h"
 
 extern int32_t dDebugFlag;
 

src/dnode/inc/dnodeMInfos.h

@@ -19,16 +19,19 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#include "dnodeInt.h"
-#include "taosmsg.h"
 
 int32_t dnodeInitMInfos();
 void    dnodeCleanupMInfos();
-void    dnodeUpdateMInfos(SMnodeInfos *minfos);
+void    dnodeUpdateMInfos(SMInfos *pMinfos);
-void    dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet);
-void    dnodeGetMInfos(SMnodeInfos *minfos);
+void    dnodeGetMInfos(SMInfos *pMinfos);
 bool    dnodeIsMasterEp(char *ep);
 
+void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
+
 #ifdef __cplusplus
 }
 #endif

src/dnode/inc/dnodeMPeer.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMPeer();
 void    dnodeCleanupMPeer();

src/dnode/inc/dnodeMRead.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMRead();
 void    dnodeCleanupMRead();

src/dnode/inc/dnodeMWrite.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMWrite();
 void    dnodeCleanupMWrite();

src/dnode/inc/dnodeMain.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitSystem();
 void    dnodeCleanUpSystem();

src/dnode/inc/dnodeModule.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitModules();
 void    dnodeStartModules();

src/dnode/inc/dnodePeer.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitServer();
 void    dnodeCleanupServer();

src/dnode/inc/dnodeShell.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitShell();
 void    dnodeCleanupShell();

src/dnode/inc/dnodeStep.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_STEP_H
+#define TDENGINE_DNODE_STEP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
+void    dnodeStepCleanup(SStep *pSteps, int32_t stepSize);
+void    dnodeReportStep(char *name, char *desc, int8_t finished);
+void    dnodeSendStartupStep(SRpcMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/inc/dnodeTelemetry.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitTelemetry();
 void    dnodeCleanupTelemetry();

src/dnode/inc/dnodeVMgmt.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_VMGMT_H
+#define TDENGINE_DNODE_VMGMT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeInitVMgmt();
+void    dnodeCleanupVMgmt();
+void    dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/inc/dnodeVRead.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVRead();
 void    dnodeCleanupVRead();

src/dnode/inc/dnodeVWrite.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVWrite();
 void    dnodeCleanupVWrite();

src/dnode/inc/dnodeVnodes.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_VNODES_H
+#define TDENGINE_DNODE_VNODES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeInitVnodes();
+void    dnodeCleanupVnodes();
+int32_t dnodeInitTimer();
+void    dnodeCleanupTimer();
+void    dnodeSendStatusMsgToMnode();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/src/dnodeCfg.c

@@ -16,9 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeCfg.h"
 
 static SDnodeCfg tsCfg = {0};

src/dnode/src/dnodeCheck.c

@@ -15,8 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
-#include "dnodeInt.h"
 #include "dnodeCheck.h"
 
 typedef struct {

src/dnode/src/dnodeEps.c

@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "hash.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeEps.h"
 
 static SDnodeEps *tsEps = NULL;
@@ -236,7 +233,14 @@ PRASE_EPS_OVER:
   dnodeResetEps(eps);
   if (eps) free(eps);
 
+#if 0
   dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+#else
+  if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) {
+    dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
+    return -1;
+  }
+#endif
 
   terrno = 0;
   return 0;

src/dnode/src/dnodeMInfos.c

@@ -16,18 +16,15 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeMInfos.h"
 
-static SMnodeInfos tsMInfos;
+static SMInfos   tsMInfos;
-static SRpcEpSet   tsMEpSet;
+static SRpcEpSet tsMEpSet;
 static pthread_mutex_t tsMInfosMutex;
 
-static void    dnodeResetMInfos(SMnodeInfos *minfos);
+static void    dnodeResetMInfos(SMInfos *minfos);
-static void    dnodePrintMInfos(SMnodeInfos *minfos);
+static void    dnodePrintMInfos(SMInfos *minfos);
 static int32_t dnodeReadMInfos();
 static int32_t dnodeWriteMInfos();
 
@@ -44,14 +41,14 @@ int32_t dnodeInitMInfos() {
 
 void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
 
-void dnodeUpdateMInfos(SMnodeInfos *minfos) {
+void dnodeUpdateMInfos(SMInfos *pMinfos) {
-  if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
+  if (pMinfos->mnodeNum <= 0 || pMinfos->mnodeNum > 3) {
-    dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+    dError("invalid mnode infos, mnodeNum:%d", pMinfos->mnodeNum);
     return;
   }
 
-  for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
+  for (int32_t i = 0; i < pMinfos->mnodeNum; ++i) {
-    SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+    SMInfo *minfo = &pMinfos->mnodeInfos[i];
     minfo->mnodeId = htonl(minfo->mnodeId);
     if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
       dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
@@ -60,14 +57,14 @@ void dnodeUpdateMInfos(SMnodeInfos *minfos) {
   }
 
   pthread_mutex_lock(&tsMInfosMutex);
-  if (minfos->mnodeNum != tsMInfos.mnodeNum) {
+  if (pMinfos->mnodeNum != tsMInfos.mnodeNum) {
-    dnodeResetMInfos(minfos);
+    dnodeResetMInfos(pMinfos);
     dnodeWriteMInfos();
     sdbUpdateAsync();
   } else {
-    int32_t size = sizeof(SMnodeInfos);
+    int32_t size = sizeof(SMInfos);
-    if (memcmp(minfos, &tsMInfos, size) != 0) {
+    if (memcmp(pMinfos, &tsMInfos, size) != 0) {
-      dnodeResetMInfos(minfos);
+      dnodeResetMInfos(pMinfos);
       dnodeWriteMInfos();
       sdbUpdateAsync();
     }
@@ -99,11 +96,11 @@ bool dnodeIsMasterEp(char *ep) {
   return isMaster;
 }
 
-void dnodeGetMInfos(SMnodeInfos *minfos) {
+void dnodeGetMInfos(SMInfos *pMinfos) {
   pthread_mutex_lock(&tsMInfosMutex);
-  memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+  memcpy(pMinfos, &tsMInfos, sizeof(SMInfos));
   for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
-    minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+    pMinfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
   }
   pthread_mutex_unlock(&tsMInfosMutex);
 }
@@ -123,15 +120,15 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
   pthread_mutex_unlock(&tsMInfosMutex);
 }
 
-static void dnodePrintMInfos(SMnodeInfos *minfos) {
+static void dnodePrintMInfos(SMInfos *pMinfos) {
-  dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
+  dInfo("print minfos, mnodeNum:%d inUse:%d", pMinfos->mnodeNum, pMinfos->inUse);
-  for (int32_t i = 0; i < minfos->mnodeNum; i++) {
+  for (int32_t i = 0; i < pMinfos->mnodeNum; i++) {
-    dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+    dInfo("mnode index:%d, %s", pMinfos->mnodeInfos[i].mnodeId, pMinfos->mnodeInfos[i].mnodeEp);
   }
 }
 
-static void dnodeResetMInfos(SMnodeInfos *minfos) {
+static void dnodeResetMInfos(SMInfos *pMinfos) {
-  if (minfos == NULL) {
+  if (pMinfos == NULL) {
     tsMEpSet.numOfEps = 1;
     taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
 
@@ -142,10 +139,10 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) {
     return;
   }
 
-  if (minfos->mnodeNum == 0) return;
+  if (pMinfos->mnodeNum == 0) return;
 
-  int32_t size = sizeof(SMnodeInfos);
+  int32_t size = sizeof(SMInfos);
-  memcpy(&tsMInfos, minfos, size);
+  memcpy(&tsMInfos, pMinfos, size);
 
   tsMEpSet.inUse = tsMInfos.inUse;
   tsMEpSet.numOfEps = tsMInfos.mnodeNum;
@@ -153,16 +150,17 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) {
     taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
   }
 
-  dnodePrintMInfos(minfos);
+  dnodePrintMInfos(pMinfos);
 }
 
 static int32_t dnodeReadMInfos() {
-  int32_t     len = 0;
+  int32_t len = 0;
-  int32_t     maxLen = 2000;
+  int32_t maxLen = 2000;
-  char *      content = calloc(1, maxLen + 1);
+  char *  content = calloc(1, maxLen + 1);
-  cJSON *     root = NULL;
+  cJSON * root = NULL;
-  FILE *      fp = NULL;
+  FILE *  fp = NULL;
-  SMnodeInfos minfos = {0};
+  SMInfos minfos = {0};
+  bool    nodeChanged = false;
 
   char file[TSDB_FILENAME_LEN + 20] = {0};
   sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
@@ -221,14 +219,19 @@ static int32_t dnodeReadMInfos() {
       dError("failed to read mnodeEpSet.json, nodeId not found");
       goto PARSE_MINFOS_OVER;
     }
-    minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
 
     cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
     if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
       dError("failed to read mnodeEpSet.json, nodeName not found");
       goto PARSE_MINFOS_OVER;
     }
-    strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+    SMInfo *pMinfo = &minfos.mnodeInfos[i];
+    pMinfo->mnodeId = nodeId->valueint;
+    tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+    bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp);
+    if (changed) nodeChanged = changed;
   }
 
   dInfo("read file %s successed", file);
@@ -241,10 +244,15 @@ PARSE_MINFOS_OVER:
   terrno = 0;
 
   for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
-    SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+    SMInfo *mInfo = &minfos.mnodeInfos[i];
     dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
   }
   dnodeResetMInfos(&minfos);
+
+  if (nodeChanged) {
+    dnodeWriteMInfos();
+  }
+
   return 0;
 }
 
@@ -286,3 +294,25 @@ static int32_t dnodeWriteMInfos() {
   dInfo("successed to write %s", file);
   return 0;
 }
+
+void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
+  SRpcConnInfo connInfo = {0};
+  rpcGetConnInfo(rpcMsg->handle, &connInfo);
+
+  SRpcEpSet epSet = {0};
+  if (forShell) {
+    dnodeGetEpSetForShell(&epSet);
+  } else {
+    dnodeGetEpSetForPeer(&epSet);
+  }
+
+  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
+         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
+
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
+    epSet.port[i] = htons(epSet.port[i]);
+  }
+
+  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
+}

src/dnode/src/dnodeMPeer.c

@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
+#include "dnodeVMgmt.h"
-#include "dnodeInt.h"
+#include "dnodeMInfos.h"
-#include "dnodeMgmt.h"
 #include "dnodeMWrite.h"
 
 typedef struct {

src/dnode/src/dnodeMRead.c

@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
+#include "dnodeVMgmt.h"
-#include "dnodeInt.h"
+#include "dnodeMInfos.h"
-#include "dnodeMgmt.h"
 #include "dnodeMRead.h"
 
 typedef struct {

src/dnode/src/dnodeMWrite.c

@@ -15,17 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "ttimer.h"
 #include "tqueue.h"
-#include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
+#include "dnodeVMgmt.h"
-#include "dnodeInt.h"
+#include "dnodeMInfos.h"
-#include "dnodeMgmt.h"
 #include "dnodeMWrite.h"
 
 typedef struct {

src/dnode/src/dnodeMain.c

@@ -16,14 +16,12 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taos.h"
-#include "tutil.h"
 #include "tconfig.h"
-#include "tglobal.h"
+#include "tfile.h"
 #include "twal.h"
-#include "trpc.h"
+// #include "tfs.h"
-#include "dnode.h"
+#include "tsync.h"
-#include "dnodeInt.h"
+#include "dnodeStep.h"
-#include "dnodeMgmt.h"
 #include "dnodePeer.h"
 #include "dnodeModule.h"
 #include "dnodeEps.h"
@@ -32,6 +30,8 @@
 #include "dnodeCheck.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
+#include "dnodeVMgmt.h"
+#include "dnodeVnodes.h"
 #include "dnodeMRead.h"
 #include "dnodeMWrite.h"
 #include "dnodeMPeer.h"
@@ -44,37 +44,32 @@ static int32_t dnodeInitStorage();
 static void    dnodeCleanupStorage();
 static void    dnodeSetRunStatus(SRunStatus status);
 static void    dnodeCheckDataDirOpenned(char *dir);
-static int32_t dnodeInitComponents();
-static void    dnodeCleanupComponents(int32_t stepId);
 static int     dnodeCreateDir(const char *dir);
 
-typedef struct {
+static SStep tsDnodeSteps[] = {
-  const char *const name;
+  {"dnode-tfile",     tfInit,              tfCleanup},
-  int32_t (*init)();
+  {"dnode-rpc",       rpcInit,             rpcCleanup},
-  void (*cleanup)();
+  {"dnode-globalcfg", taosCheckGlobalCfg,  NULL},
-} SDnodeComponent;
+  {"dnode-storage",   dnodeInitStorage,    dnodeCleanupStorage},
-
+  {"dnode-cfg",       dnodeInitCfg,        dnodeCleanupCfg},
-static const SDnodeComponent tsDnodeComponents[] = {
+  {"dnode-eps",       dnodeInitEps,        dnodeCleanupEps},
-  {"rpc",       rpcInit,             rpcCleanup},
+  {"dnode-minfos",    dnodeInitMInfos,     dnodeCleanupMInfos},
-  {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
+  {"dnode-wal",       walInit,             walCleanUp},
-  {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
+  {"dnode-sync",      syncInit,            syncCleanUp},
-  {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
+  {"dnode-check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
-  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
+  {"dnode-vread",     dnodeInitVRead,      dnodeCleanupVRead},
-  {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
+  {"dnode-vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
-  {"wal",       walInit,             walCleanUp},
+  {"dnode-vmgmt",     dnodeInitVMgmt,      dnodeCleanupVMgmt},
-  {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
+  {"dnode-mread",     dnodeInitMRead,      dnodeCleanupMRead},
-  {"vread",     dnodeInitVRead,      dnodeCleanupVRead},
+  {"dnode-mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
-  {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
+  {"dnode-mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
-  {"mread",     dnodeInitMRead,      dnodeCleanupMRead},
+  {"dnode-client",    dnodeInitClient,     dnodeCleanupClient},
-  {"mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
+  {"dnode-server",    dnodeInitServer,     dnodeCleanupServer},
-  {"mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
+  {"dnode-vnodes",    dnodeInitVnodes,     dnodeCleanupVnodes},
-  {"client",    dnodeInitClient,     dnodeCleanupClient},
+  {"dnode-modules",   dnodeInitModules,    dnodeCleanupModules},
-  {"server",    dnodeInitServer,     dnodeCleanupServer},
+  {"dnode-tmr",       dnodeInitTimer,      dnodeCleanupTimer},
-  {"mgmt",      dnodeInitMgmt,       dnodeCleanupMgmt},
+  {"dnode-shell",     dnodeInitShell,      dnodeCleanupShell},
-  {"modules",   dnodeInitModules,    dnodeCleanupModules},
+  {"dnode-telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
-  {"mgmt-tmr",  dnodeInitMgmtTimer,  dnodeCleanupMgmtTimer},
-  {"shell",     dnodeInitShell,      dnodeCleanupShell},
-  {"telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
 };
 
 static int dnodeCreateDir(const char *dir) {
@@ -85,24 +80,14 @@ static int dnodeCreateDir(const char *dir) {
   return 0;
 }
 
-static void dnodeCleanupComponents(int32_t stepId) {
+static void dnodeCleanupComponents() {
-  for (int32_t i = stepId; i >= 0; i--) {
+  int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
-    if (tsDnodeComponents[i].cleanup) {
+  dnodeStepCleanup(tsDnodeSteps, stepSize);
-      (*tsDnodeComponents[i].cleanup)();
-    }
-  }
 }
 
 static int32_t dnodeInitComponents() {
-  int32_t code = 0;
+  int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
-  for (int32_t i = 0; i < sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]); i++) {
+  return dnodeStepInit(tsDnodeSteps, stepSize);
-    if (tsDnodeComponents[i].init() != 0) {
-      dnodeCleanupComponents(i);
-      code = -1;
-      break;
-    }
-  }
-  return code;
 }
 
 int32_t dnodeInitSystem() {
@@ -149,7 +134,7 @@ int32_t dnodeInitSystem() {
 void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
-    dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
+    dnodeCleanupComponents();
     taos_cleanup();
     taosCloseLog();
   }

src/dnode/src/dnodeMgmt.c

@@ -1,572 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "cJSON.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "ttimer.h"
-#include "tsdb.h"
-#include "twal.h"
-#include "tqueue.h"
-#include "tsync.h"
-#include "ttimer.h"
-#include "tbalance.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "vnode.h"
-#include "mnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeEps.h"
-#include "dnodeCfg.h"
-#include "dnodeMInfos.h"
-#include "dnodeVRead.h"
-#include "dnodeVWrite.h"
-#include "dnodeModule.h"
-
-typedef struct {
-  pthread_t thread;
-  int32_t   threadIndex;
-  int32_t   failed;
-  int32_t   opened;
-  int32_t   vnodeNum;
-  int32_t * vnodeList;
-} SOpenVnodeThread;
-
-typedef struct {
-  SRpcMsg rpcMsg;
-  char    pCont[];
-} SMgmtMsg;
-
-void *            tsDnodeTmr = NULL;
-static void *     tsStatusTimer = NULL;
-static uint32_t   tsRebootTime;
-static taos_qset  tsMgmtQset = NULL;
-static taos_queue tsMgmtQueue = NULL;
-static pthread_t  tsQthread;
-
-static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
-static void   dnodeSendStatusMsg(void *handle, void *tmrId);
-static void  *dnodeProcessMgmtQueue(void *param);
-
-static int32_t  dnodeOpenVnodes();
-static void     dnodeCloseVnodes();
-static int32_t  dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
-static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
-
-int32_t dnodeInitMgmt() {
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeProcessAlterVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeProcessDropVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
-
-  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
-  tsRebootTime = taosGetTimestampSec();
-
-  int32_t code = vnodeInitResources();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  // create the queue and thread to handle the message 
-  tsMgmtQset = taosOpenQset();
-  if (tsMgmtQset == NULL) {
-    dError("failed to create the mgmt queue set");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  tsMgmtQueue = taosOpenQueue();
-  if (tsMgmtQueue == NULL) {
-    dError("failed to create the mgmt queue");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  taosAddIntoQset(tsMgmtQset, tsMgmtQueue, NULL);
-
-  pthread_attr_t thAttr;
-  pthread_attr_init(&thAttr);
-  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-
-  code = pthread_create(&tsQthread, &thAttr, dnodeProcessMgmtQueue, NULL);
-  pthread_attr_destroy(&thAttr);
-  if (code != 0) {
-    dError("failed to create thread to process mgmt queue, reason:%s", strerror(errno));
-    dnodeCleanupMgmt();
-    return -1; 
-  }
-
-  code = dnodeOpenVnodes();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  dInfo("dnode mgmt is initialized");
- 
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t dnodeInitMgmtTimer() {
-  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
-  if (tsDnodeTmr == NULL) {
-    dError("failed to init dnode timer");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  dInfo("dnode mgmt timer is initialized");
-  return TSDB_CODE_SUCCESS;
-}
-
-void dnodeSendStatusMsgToMnode() {
-  if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
-    dInfo("force send status msg to mnode");
-    taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
-  }
-}
-
-void dnodeCleanupMgmtTimer() {
-  if (tsStatusTimer != NULL) {
-    taosTmrStopA(&tsStatusTimer);
-    tsStatusTimer = NULL;
-  }
-
-  if (tsDnodeTmr != NULL) {
-    taosTmrCleanUp(tsDnodeTmr);
-    tsDnodeTmr = NULL;
-  }
-}
-
-void dnodeCleanupMgmt() {
-  dnodeCleanupMgmtTimer();
-  dnodeCloseVnodes();
-
-  if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);
-  if (tsQthread) pthread_join(tsQthread, NULL);
-
-  if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue);
-  if (tsMgmtQset) taosCloseQset(tsMgmtQset);
-  tsMgmtQset = NULL;
-  tsMgmtQueue = NULL;
-
-  vnodeCleanupResources();
-}
-
-static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
-  int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
-  SMgmtMsg *pMgmt = taosAllocateQitem(size);
-  if (pMgmt == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-
-  pMgmt->rpcMsg = *pMsg;
-  pMgmt->rpcMsg.pCont = pMgmt->pCont;
-  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
-  taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
-  int32_t code = dnodeWriteToMgmtQueue(pMsg);
-  if (code != TSDB_CODE_SUCCESS) {
-    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-}
-
-static void *dnodeProcessMgmtQueue(void *param) {
-  SMgmtMsg *pMgmt;
-  SRpcMsg * pMsg;
-  SRpcMsg   rsp = {0};
-  int32_t   qtype;
-  void *    handle;
-
-  while (1) {
-    if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
-      dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
-      break;
-    }
-
-    pMsg = &pMgmt->rpcMsg;
-    dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
-    if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
-      rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
-    } else {
-      rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-    }
-
-    rsp.handle = pMsg->handle;
-    rsp.pCont = NULL;
-    rpcSendResponse(&rsp);
-
-    taosFreeQitem(pMsg);
-  }
-
-  return NULL;
-}
-
-static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  DIR *dir = opendir(tsVnodeDir);
-  if (dir == NULL) {
-    return TSDB_CODE_DND_NO_WRITE_ACCESS;
-  }
-
-  *numOfVnodes = 0;
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-    if (de->d_type & DT_DIR) {
-      if (strncmp("vnode", de->d_name, 5) != 0) continue;
-      int32_t vnode = atoi(de->d_name + 5);
-      if (vnode == 0) continue;
-
-      (*numOfVnodes)++;
-
-      if (*numOfVnodes >= TSDB_MAX_VNODES) {
-        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
-        continue;
-      } else {
-        vnodeList[*numOfVnodes - 1] = vnode;
-      }
-    }
-  }
-  closedir(dir);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void *dnodeOpenVnode(void *param) {
-  SOpenVnodeThread *pThread = param;
-  char vnodeDir[TSDB_FILENAME_LEN * 3];
-
-  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
-
-  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
-    int32_t vgId = pThread->vnodeList[v];
-    snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId);
-    if (vnodeOpen(vgId, vnodeDir) < 0) {
-      dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
-      pThread->failed++;
-    } else {
-      dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
-      pThread->opened++;
-    }
-  }
-
-  dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
-         pThread->failed);
-  return NULL;
-}
-
-static int32_t dnodeOpenVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
-  int32_t numOfVnodes = 0;
-  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return status;
-  }
-
-  int32_t threadNum = tsNumOfCores;
-  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
-  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
-  for (int32_t t = 0; t < threadNum; ++t) {
-    threads[t].threadIndex = t;
-    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
-  }
-
-  for (int32_t v = 0; v < numOfVnodes; ++v) {
-    int32_t t = v % threadNum;
-    SOpenVnodeThread *pThread = &threads[t];
-    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
-  }
-
-  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
-
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum == 0) continue;
-
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-    if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
-      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
-    }
-
-    pthread_attr_destroy(&thAttr);
-  }
-
-  int32_t openVnodes = 0;
-  int32_t failedVnodes = 0;
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum > 0 && pThread->thread) {
-      pthread_join(pThread->thread, NULL);
-    }
-    openVnodes += pThread->opened;
-    failedVnodes += pThread->failed;
-    free(pThread->vnodeList);
-  }
-
-  free(threads);
-  dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeCloseVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES]= {0};
-  int32_t numOfVnodes = 0;
-  int32_t status;
-
-  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return;
-  }
-
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    vnodeClose(vnodeList[i]);
-  }
-
-  dInfo("total vnodes:%d are all closed", numOfVnodes);
-}
-
-static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
-  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
-  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
-  pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
-  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
-  pCreate->cfg.cacheBlockSize      = htonl(pCreate->cfg.cacheBlockSize);
-  pCreate->cfg.totalBlocks         = htonl(pCreate->cfg.totalBlocks);
-  pCreate->cfg.daysPerFile         = htonl(pCreate->cfg.daysPerFile);
-  pCreate->cfg.daysToKeep1         = htonl(pCreate->cfg.daysToKeep1);
-  pCreate->cfg.daysToKeep2         = htonl(pCreate->cfg.daysToKeep2);
-  pCreate->cfg.daysToKeep          = htonl(pCreate->cfg.daysToKeep);
-  pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
-  pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
-  pCreate->cfg.fsyncPeriod         = htonl(pCreate->cfg.fsyncPeriod);
-  pCreate->cfg.commitTime          = htonl(pCreate->cfg.commitTime);
-
-  for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
-    pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
-  }
-
-  return pCreate;
-}
-
-static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
-  SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
-
-  void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
-  if (pVnode != NULL) {
-    dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
-    vnodeRelease(pVnode);
-    return TSDB_CODE_SUCCESS;
-  } else {
-    dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
-    return vnodeCreate(pCreate);
-  }
-}
-
-static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
-  SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
-
-  void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
-  if (pVnode != NULL) {
-    dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
-    int32_t code = vnodeAlter(pVnode, pAlter);
-    vnodeRelease(pVnode);
-    return code;
-  } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
-    return TSDB_CODE_VND_INVALID_VGROUP_ID;
-  }
-}
-
-static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
-  SDropVnodeMsg *pDrop = rpcMsg->pCont;
-  pDrop->vgId = htonl(pDrop->vgId);
-
-  return vnodeDrop(pDrop->vgId);
-}
-
-static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-//  SAlterStreamMsg *pStream = pCont;
-//  pStream->uid    = htobe64(pStream->uid);
-//  pStream->stime  = htobe64(pStream->stime);
-//  pStream->vnode  = htonl(pStream->vnode);
-//  pStream->sid    = htonl(pStream->sid);
-//  pStream->status = htonl(pStream->status);
-//
-//  int32_t code = dnodeCreateStream(pStream);
-
-  return 0;
-}
-
-static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
-  SCfgDnodeMsg *pCfg = pMsg->pCont;
-  return taosCfgDynamicOptions(pCfg->config);
-}
-
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
-  SCreateMnodeMsg *pCfg = pMsg->pCont;
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  if (pCfg->dnodeId != dnodeGetDnodeId()) {
-    dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
-    return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
-  }
-
-  if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
-    dError("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
-    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
-  }
-
-  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
-  for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
-    pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
-    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
-  }
-
-  dnodeStartMnode(&pCfg->mnodes);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  if (pMsg->code != TSDB_CODE_SUCCESS) {
-    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
-
-  SStatusRsp *pStatusRsp = pMsg->pCont;
-  SMnodeInfos *minfos = &pStatusRsp->mnodes;
-  dnodeUpdateMInfos(minfos);
-
-  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
-  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  dnodeUpdateCfg(pCfg);
-
-  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
-
-  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
-  dnodeUpdateEps(pEps);
-
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-}
-
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  if (tsDnodeTmr == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
-
-  if (tsStatusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to start status timer");
-    return;
-  }
-
-  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SStatusMsg *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to malloc status message");
-    return;
-  }
-
-  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
-  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
-  pStatus->version          = htonl(tsVersion);
-  pStatus->lastReboot       = htonl(tsRebootTime);
-  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
-  pStatus->diskAvailable    = tsAvailDataDirGB;
-  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
-
-  // fill cluster cfg parameters
-  pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
-  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
-  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
-  pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
-  pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);
-  pStatus->clusterCfg.maxtablesPerVnode  = htonl(tsMaxTablePerVnode);
-  pStatus->clusterCfg.maxVgroupsPerDb    = htonl(tsMaxVgroupsPerDb);
-  tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
-  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
-  pStatus->clusterCfg.checkTime = 0;
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
-  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
-  
-  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
-  pStatus->openVnodes = htons(pStatus->openVnodes);
-  
-  SRpcMsg rpcMsg = {
-    .pCont   = pStatus,
-    .contLen = contLen,
-    .msgType = TSDB_MSG_TYPE_DM_STATUS
-  };
-
-  SRpcEpSet epSet;
-  dnodeGetEpSetForPeer(&epSet);
-  dnodeSendMsgToDnode(&epSet, &rpcMsg);
-}
-
-void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
-  SRpcConnInfo connInfo = {0};
-  rpcGetConnInfo(rpcMsg->handle, &connInfo);
-
-  SRpcEpSet epSet = {0};
-  if (forShell) {
-    dnodeGetEpSetForShell(&epSet);
-  } else {
-    dnodeGetEpSetForPeer(&epSet);
-  }
-  
-  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
-         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
-
-  for (int i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
-    epSet.port[i] = htons(epSet.port[i]);
-  }
-
-  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
-}

src/dnode/src/dnodeModule.c

@@ -15,15 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
 #include "http.h"
 #include "tmqtt.h"
 #include "monitor.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeModule.h"
 
 typedef struct {
@@ -78,10 +73,10 @@ static void dnodeAllocModules() {
 
   tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
-  tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
+  tsModule[TSDB_MOD_MONITOR].initFp    = monInitSystem;
-  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem;
+  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem;
-  tsModule[TSDB_MOD_MONITOR].startFp   = monitorStartSystem;
+  tsModule[TSDB_MOD_MONITOR].startFp   = monStartSystem;
-  tsModule[TSDB_MOD_MONITOR].stopFp    = monitorStopSystem;
+  tsModule[TSDB_MOD_MONITOR].stopFp    = monStopSystem;
   if (tsEnableMonitorModule) {
     dnodeSetModuleStatus(TSDB_MOD_MONITOR);
   }
@@ -147,8 +142,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
   }
 }
 
-bool dnodeStartMnode(SMnodeInfos *minfos) {
+bool dnodeStartMnode(SMInfos *pMinfos) {
-  SMnodeInfos *mnodes = minfos;
+  SMInfos *pMnodes = pMinfos;
 
   if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
     dDebug("mnode module is already started, module status:%d", tsModuleStatus);
@@ -159,7 +154,7 @@ bool dnodeStartMnode(SMnodeInfos *minfos) {
   dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
   dnodeProcessModuleStatus(moduleStatus);
 
-  sdbUpdateSync(mnodes);
+  sdbUpdateSync(pMnodes);
 
   return true;
 }

src/dnode/src/dnodePeer.c

@@ -21,15 +21,12 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
+#include "dnodeVMgmt.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMPeer.h"
 #include "dnodeMInfos.h"
+#include "dnodeStep.h"
 
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
@@ -44,19 +41,19 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVWriteQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVWriteQueue;
 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToVMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToVMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue;
 
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMPeerQueue;
-  
+
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = tsDnodeDnodePort;
@@ -91,8 +88,9 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     .pCont   = NULL,
     .contLen = 0
   };
-  
+
   if (pMsg->pCont == NULL) return;
+  if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg);
 
   if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
@@ -151,9 +149,9 @@ void dnodeCleanupClient() {
 }
 
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
-    dDebug("msg:%p is ignored since dnode not running", pMsg);
+    dDebug("msg:%p is ignored since dnode is stopping", pMsg);
     rpcFreeCont(pMsg->pCont);
     return;
   }
@@ -182,6 +180,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcEpSet epSet = {0};
   dnodeGetEpSetForPeer(&epSet);
+
+  assert(tsClientRpc != 0);
   rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
 }
 

src/dnode/src/dnodeShell.c

@@ -15,20 +15,14 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
-#include "tutil.h"
 #include "http.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
 #include "dnodeMRead.h"
 #include "dnodeMWrite.h"
 #include "dnodeShell.h"
+#include "dnodeStep.h"
 
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
@@ -74,6 +68,8 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMReadQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMReadQueue;
 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]   = dnodeSendStartupStep;
+
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
   numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
   if (numOfThreads < 1) {
@@ -142,7 +138,23 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 }
 
+static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  if (strcmp(user, "nettestinternal") == 0) {
+    char pass[32] = {0};
+    taosEncryptPass((uint8_t *)user, strlen(user), pass);
+    *spi = 0;
+    *encrypt = 0;
+    *ckey = 0;
+    memcpy(secret, pass, TSDB_KEY_LEN);
+    dTrace("nettest user is authorized");
+    return 0;
+  }
+
+  return -1;
+}
+
 static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0;  
   int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
   if (code != TSDB_CODE_APP_NOT_READY) return code;
 
@@ -220,4 +232,4 @@ SStatisInfo dnodeGetStatisInfo() {
   }
 
   return info;
-}
+}