Merge branch 'develop' into feature/TD-1413

2020-11-18 10:17:18 +08:00 · 2020-11-18 10:17:18 +08:00 · de1a75f64d
parent bc2f0d9d13 df74e517c3
commit de1a75f64d
361 changed files with 26019 additions and 11936 deletions
--- a/.gitmodules
+++ b/.gitmodules
@ -4,3 +4,9 @@
 [submodule "src/connector/grafanaplugin"]
 	path = src/connector/grafanaplugin
 	url = https://github.com/taosdata/grafanaplugin
+[submodule "tests/examples/rust"]
+	path = tests/examples/rust
+	url = https://github.com/songtianyi/tdengine-rust-bindings.git
+[submodule "src/connector/hivemq-tdengine-extension"]
+	path = src/connector/hivemq-tdengine-extension
+	url = https://github.com/huskar-t/hivemq-tdengine-extension.git
--- a/128
+++ b/128
@ -27,6 +27,7 @@ pipeline {
            cd debug
            cmake .. > /dev/null
            make > /dev/null
+            make install > /dev/null
            cd ${WKC}/tests
            #./test-all.sh smoke
            ./test-all.sh pytest
@ -79,7 +80,20 @@ pipeline {
            cmake .. > /dev/null
            make > /dev/null
            cd ${WKC}/tests/pytest
-            ./crash_gen.sh -a -p -t 4 -s 2000
+            '''
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./crash_gen.sh -a -p -t 4 -s 2000
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./handle_crash_gen_val_log.sh
+                '''
+            }
+            sh '''
            date
            cd ${WKC}/tests
            ./test-all.sh b2
@ -124,14 +138,33 @@ pipeline {
            sh'''
            cd ${WORKSPACE}
            git checkout develop
-            cd tests/gotest
-            bash batchtest.sh
-            cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
-            mvn clean package assembly:single >/dev/null 
-            java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
-            cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
-            python3 PythonChecker.py
            '''
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/gotest
+                bash batchtest.sh
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+                python3 PythonChecker.py
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
+                mvn clean package assembly:single >/dev/null 
+                java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
+                dotnet run
+                '''
+            }
+          
         }
       }

@ -139,5 +172,82 @@ pipeline {
    }

  }
-  
+  post {             
+        success {
+            emailext (
+                subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                body: '''<!DOCTYPE html>
+                <html>
+                <head>
+                <meta charset="UTF-8">
+                </head>
+                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
+                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
+                        <tr>
+                            <td><br />
+                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
+                                <hr size="2" width="100%" align="center" /></td>
+                        </tr>
+                        <tr>
+                            <td>
+                                <ul>
+                                <div style="font-size:18px">
+                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建编号：${BUILD_NUMBER}</li>
+                                    <li>触发用户：${CAUSE}</li>
+                                    <li>变更概要：${CHANGES}</li>
+                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
+                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
+                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                </div>
+                                </ul>
+                            </td>
+                        </tr>
+                    </table></font>
+                </body>
+                </html>''',
+                to: "yqliu@taosdata.com,pxiao@taosdata.com",
+                from: "support@taosdata.com"
+            )
+        }
+        failure {
+            emailext (
+                subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                body: '''<!DOCTYPE html>
+                <html>
+                <head>
+                <meta charset="UTF-8">
+                </head>
+                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
+                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
+                        <tr>
+                            <td><br />
+                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
+                                <hr size="2" width="100%" align="center" /></td>
+                        </tr>
+                        <tr>
+                            <td>
+                                <ul>
+                                <div style="font-size:18px">
+                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建编号：${BUILD_NUMBER}</li>
+                                    <li>触发用户：${CAUSE}</li>
+                                    <li>变更概要：${CHANGES}</li>
+                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
+                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
+                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                </div>
+                                </ul>
+                            </td>
+                        </tr>
+                    </table></font>
+                </body>
+                </html>''',
+                to: "yqliu@taosdata.com,pxiao@taosdata.com",
+                from: "support@taosdata.com"
+            )
+        }
+    }
 }
--- a/cmake/install.inc
+++ b/cmake/install.inc
@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go     DESTINATION connector)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
+  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
  INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
  INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
--- a/cmake/version.inc
+++ b/cmake/version.inc
@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
  SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.6.0")
+  SET(TD_VER_NUMBER "2.0.7.0")
 ENDIF ()

 IF (DEFINED VERCOMPATIBLE)
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。
 - dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。
 - logDir：日志文件目录，客户端和服务器的运行日志文件将写入该目录。默认值：/var/log/taos。
+- tempDir：临时文件目录，客户端和服务器的临时文件（主要是查询时用于保存中间结果的问题）将写入该目录。 默认值：Linux下为 /tmp/，Windows下为环境变量 tmp 或 temp 指向的目录。
 - arbitrator：系统中裁决器的end point, 缺省值为空。
 - role：dnode的可选角色。0-any; 既可作为mnode，也可分配vnode；1-mgmt;只能作为mnode，不能分配vnode；2-dnode;不能作为mnode，只能分配vnode
 - debugFlag：运行日志开关。131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志）。默认值：131或135（不同模块有不同的默认值）。
--- a/documentation20/webdocs/markdowndocs/Documentation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md
@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
 - [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录
 - [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入
 - [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入
- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将MQTT数据直接写入
+- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入
+- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入

 ## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries)

--- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md
@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的
 * __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。
 * __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 
 * __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
-* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
+* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
 * __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。

 采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
--- a/documentation20/webdocs/markdowndocs/TAOS
+++ b/documentation20/webdocs/markdowndocs/TAOS
@ -90,7 +90,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
    ```mysql
    ALTER DATABASE db_name REPLICA 2;
    ```
-    REPLICA参数是指修改数据库副本数，取值范围[1, 3]。在集群中使用，副本数必须小于dnode的数目。
+    REPLICA参数是指修改数据库副本数，取值范围[1, 3]。在集群中使用，副本数必须小于或等于dnode的数目。

    ```mysql
    ALTER DATABASE db_name KEEP 365;
@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数

 - **PERCENTILE**
    ```mysql
-    SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause];
+    SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
    ```
    功能说明：统计表中某列的值百分比分位数。  
    返回结果数据类型： 双精度浮点数Double。  
@ -1016,9 +1016,9 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER
 ```

 ## TAOS SQL 边界限制
- 数据库名最大长度为33
- 表名最大长度为193，每行数据最大长度16k个字符
- 列名最大长度为65，最多允许1024列，最少需要2列，第一列必须是时间戳
+- 数据库名最大长度为32
+- 表名最大长度为192，每行数据最大长度16k个字符
+- 列名最大长度为64，最多允许1024列，最少需要2列，第一列必须是时间戳
 - 标签最多允许128个，可以0个，标签总长度不超过16k个字符
 - SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M
 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制
--- a/documentation20/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation20/webdocs/markdowndocs/administrator-ch.md
@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景
 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
 ```

-示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。 
+示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 

 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。

@ -253,7 +253,7 @@ ALTER USER <user_name> PASS <'password'>;
 修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角

 ```
-ALTER USER <user_name> PRIVILEDGE <super|write|read>;
+ALTER USER <user_name> PRIVILEGE <super|write|read>;
 ```

 修改用户权限为：super/write/read，不需要添加单引号
--- a/documentation20/webdocs/markdowndocs/architecture-ch.md
+++ b/documentation20/webdocs/markdowndocs/architecture-ch.md
@ -4,16 +4,99 @@
 ### 物联网典型场景
 在典型的物联网、车联网、运维监测场景中，往往有多种不同类型的数据采集设备，采集一个到多个不同的物理量。而同一种采集设备类型，往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总，然后进行计算和分析。对于同一类设备，其采集的数据都是很规则的。以智能电表为例，假设每个智能电表采集电流、电压、相位三个量，其采集的数据类似如下的表格：

-| Device ID |  Time Stamp   | current | voltage | phase |     location     | groupId |
-| :-------: | :-----------: | :-----: | :-----: | :---: | :--------------: | :-----: |
-|   d1001   | 1538548685000 |  10.3   |   219   | 0.31  | Beijing.Chaoyang |    2    |
-|   d1002   | 1538548684000 |  10.2   |   220   | 0.23  | Beijing.Chaoyang |    3    |
-|   d1003   | 1538548686500 |  11.5   |   221   | 0.35  | Beijing.Haidian  |    3    |
-|   d1004   | 1538548685500 |  13.4   |   223   | 0.29  | Beijing.Haidian  |    2    |
-|   d1001   | 1538548695000 |  12.6   |   218   | 0.33  | Beijing.Chaoyang |    2    |
-|   d1004   | 1538548696600 |  11.8   |   221   | 0.28  | Beijing.Haidian  |    2    |
-|   d1002   | 1538548696650 |  10.3   |   218   | 0.25  | Beijing.Chaoyang |    3    |
-|   d1001   | 1538548696800 |  12.3   |   221   | 0.31  | Beijing.Chaoyang |    2    |
+<figure><table>
+<thead><tr>
+    <th style="text-align:center;">设备ID</th>
+    <th style="text-align:center;">时间戳</th>
+    <th style="text-align:center;" colspan="3">采集量</th>
+    <th style="text-align:center;" colspan="2">标签</th>
+    </tr>
+
+<tr>
+<th style="text-align:center;">Device ID</th>
+<th style="text-align:center;">Time Stamp</th>
+<th style="text-align:center;">current</th>
+<th style="text-align:center;">voltage</th>
+<th style="text-align:center;">phase</th>
+<th style="text-align:center;">location</th>
+<th style="text-align:center;">groupId</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style="text-align:center;">d1001</td>
+<td style="text-align:center;">1538548685000</td>
+<td style="text-align:center;">10.3</td>
+<td style="text-align:center;">219</td>
+<td style="text-align:center;">0.31</td>
+<td style="text-align:center;">Beijing.Chaoyang</td>
+<td style="text-align:center;">2</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1002</td>
+<td style="text-align:center;">1538548684000</td>
+<td style="text-align:center;">10.2</td>
+<td style="text-align:center;">220</td>
+<td style="text-align:center;">0.23</td>
+<td style="text-align:center;">Beijing.Chaoyang</td>
+<td style="text-align:center;">3</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1003</td>
+<td style="text-align:center;">1538548686500</td>
+<td style="text-align:center;">11.5</td>
+<td style="text-align:center;">221</td>
+<td style="text-align:center;">0.35</td>
+<td style="text-align:center;">Beijing.Haidian</td>
+<td style="text-align:center;">3</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1004</td>
+<td style="text-align:center;">1538548685500</td>
+<td style="text-align:center;">13.4</td>
+<td style="text-align:center;">223</td>
+<td style="text-align:center;">0.29</td>
+<td style="text-align:center;">Beijing.Haidian</td>
+<td style="text-align:center;">2</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1001</td>
+<td style="text-align:center;">1538548695000</td>
+<td style="text-align:center;">12.6</td>
+<td style="text-align:center;">218</td>
+<td style="text-align:center;">0.33</td>
+<td style="text-align:center;">Beijing.Chaoyang</td>
+<td style="text-align:center;">2</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1004</td>
+<td style="text-align:center;">1538548696600</td>
+<td style="text-align:center;">11.8</td>
+<td style="text-align:center;">221</td>
+<td style="text-align:center;">0.28</td>
+<td style="text-align:center;">Beijing.Haidian</td>
+<td style="text-align:center;">2</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1002</td>
+<td style="text-align:center;">1538548696650</td>
+<td style="text-align:center;">10.3</td>
+<td style="text-align:center;">218</td>
+<td style="text-align:center;">0.25</td>
+<td style="text-align:center;">Beijing.Chaoyang</td>
+<td style="text-align:center;">3</td>
+</tr>
+<tr>
+<td style="text-align:center;">d1001</td>
+<td style="text-align:center;">1538548696800</td>
+<td style="text-align:center;">12.3</td>
+<td style="text-align:center;">221</td>
+<td style="text-align:center;">0.31</td>
+<td style="text-align:center;">Beijing.Chaoyang</td>
+<td style="text-align:center;">2</td>
+</tr>
+</tbody>
+</table></figure>

 <center> 表1：智能电表数据示例</center>

@ -221,7 +304,7 @@ TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），

 TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。

-每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当一半以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有一半内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。
+每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当三分之一以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有三分之一内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。

 ### 持久化存储
 TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。
--- a/documentation20/webdocs/markdowndocs/connector-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-ch.md
@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间
 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式
 - httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131

+## CSharp Connector
+
+在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
+
+#### 安装TDengine客户端
+
+C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。
+
+安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件
+
+- TDengineDriver.cs 调用taos.dll文件的Native C方法
+- TDengineTest.cs 参考程序示例
+
+在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件
+
+#### 使用方法
+
+- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
+- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法
+- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中
+
+#### 注意事项
+
+- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。
+- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。
+
+#### 第三方驱动
+
+Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考
+
+```
+//接口下载
+https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos   
+//用法说明    
+https://www.taosdata.com/blog/2020/11/02/1901.html                    
+```
+

 ## Go Connector

--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@ -38,9 +38,9 @@

 6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的

-7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
+7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 

-8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)

 9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
   检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
--- a/documentation20/webdocs/markdowndocs/insert-ch.md
+++ b/documentation20/webdocs/markdowndocs/insert-ch.md
@ -1,6 +1,6 @@
 # 高效写入数据

-TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。
+TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。

 ## SQL写入

@ -218,7 +218,15 @@ use telegraf;
 select * from cpu;
 ```

-## EMQ X Broker直接写入

-MQTT是一流行的物联网数据传输协议，[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
+
+## EMQ Broker 直接写入
+
+[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+## HiveMQ Broker 直接写入
+
+[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。

--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@ -20,6 +20,9 @@
 # data file's directory
 # dataDir                   /var/lib/taos

+# temporary file's directory
+# tempDir                   /tmp/
+
 # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only   
 # arbitrator                arbitrator_hostname:6042     

@ -256,3 +259,5 @@
 # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
 # maxBinaryDisplayWidth 30

+# enable/disable telemetry reporting
+# telemetryReporting    1
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/build/bin/taosdemo                ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taos                    ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_path}/driver 
@ -58,7 +59,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin        ${pkg_dir}${install_home_pat
 cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/nodejs               ${pkg_dir}${install_home_path}/connector
-cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.*  ${pkg_dir}${install_home_path}/connector
+cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.*  ${pkg_dir}${install_home_path}/connector ||:

 cp -r ${compile_dir}/../packaging/deb/DEBIAN        ${pkg_dir}/
 chmod 755 ${pkg_dir}/DEBIAN/*
--- a/packaging/release.sh
+++ b/packaging/release.sh
@ -156,9 +156,15 @@ build_time=$(date +"%F %R")

 # get commint id from git
 gitinfo=$(git rev-parse --verify HEAD)
-enterprise_dir="${top_dir}/../enterprise"
-cd ${enterprise_dir}
-gitinfoOfInternal=$(git rev-parse --verify HEAD)
+
+if [[ "$verMode" == "cluster" ]]; then
+  enterprise_dir="${top_dir}/../enterprise"
+  cd ${enterprise_dir}
+  gitinfoOfInternal=$(git rev-parse --verify HEAD)
+else
+  gitinfoOfInternal=NULL
+fi
+  
 cd ${curr_dir}

 # 2. cmake executable file
@ -193,23 +199,35 @@ cd ${curr_dir}
 # 3. Call the corresponding script for packaging
 if [ "$osType" != "Darwin" ]; then
  if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
-    echo "====do deb package for the ubuntu system===="
-    output_dir="${top_dir}/debs"
-    if [ -d ${output_dir} ]; then
-      ${csudo} rm -rf ${output_dir}
+    ret='0'    
+    command -v dpkg >/dev/null 2>&1 || { ret='1'; }
+    if [ "$ret" -eq 0 ]; then  
+      echo "====do deb package for the ubuntu system===="
+      output_dir="${top_dir}/debs"
+      if [ -d ${output_dir} ]; then
+        ${csudo} rm -rf ${output_dir}
+      fi
+      ${csudo} mkdir -p ${output_dir}
+      cd ${script_dir}/deb
+      ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+    else
+      echo "==========dpkg command not exist, so not release deb package!!!"
    fi
-    ${csudo} mkdir -p ${output_dir}
-    cd ${script_dir}/deb
-    ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}

-    echo "====do rpm package for the centos system===="
-    output_dir="${top_dir}/rpms"
-    if [ -d ${output_dir} ]; then
-      ${csudo} rm -rf ${output_dir}
+    ret='0'    
+    command -v rpmbuild >/dev/null 2>&1 || { ret='1'; }
+    if [ "$ret" -eq 0 ]; then  
+      echo "====do rpm package for the centos system===="
+      output_dir="${top_dir}/rpms"
+      if [ -d ${output_dir} ]; then
+        ${csudo} rm -rf ${output_dir}
+      fi
+      ${csudo} mkdir -p ${output_dir}
+      cd ${script_dir}/rpm
+      ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+    else
+      echo "==========rpmbuild command not exist, so not release rpm package!!!"
    fi
-    ${csudo} mkdir -p ${output_dir}
-    cd ${script_dir}/rpm
-    ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
  fi
 	
  echo "====do tar.gz package for all systems===="
--- a/packaging/rpm/tdengine.spec
+++ b/packaging/rpm/tdengine.spec
@ -2,7 +2,7 @@
 %define cfg_install_dir  /etc/taos
 %define __strip /bin/true

-Name:		TDengine
+Name:		tdengine
 Version:	%{_version}
 Release:	3%{?dist}
 Summary:	tdengine from taosdata
@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/scri
 cp %{_compiledir}/build/bin/taos                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosdemo                %{buildroot}%{homepath}/bin
+cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taoserror.h            %{buildroot}%{homepath}/include
@ -65,7 +66,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn
 cp -r %{_compiledir}/../src/connector/python        %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/go            %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/nodejs        %{buildroot}%{homepath}/connector
-cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.*  %{buildroot}%{homepath}/connector
+cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.*  %{buildroot}%{homepath}/connector ||:
 cp -r %{_compiledir}/../tests/examples/*            %{buildroot}%{homepath}/examples

 #Scripts executed before installation
@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then
    ${csudo} rm -f ${bin_link_dir}/taos       || :
    ${csudo} rm -f ${bin_link_dir}/taosd      || :
    ${csudo} rm -f ${bin_link_dir}/taosdemo   || :
+    #${csudo} rm -f ${bin_link_dir}/taosdump   || :
    ${csudo} rm -f ${cfg_link_dir}/*          || :
    ${csudo} rm -f ${inc_link_dir}/taos.h     || :
    ${csudo} rm -f ${inc_link_dir}/taoserror.h     || :
--- a/packaging/tools/get_client.sh
+++ b/packaging/tools/get_client.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+log_dir=$1
+result_file=$2
+
+if [ ! -n "$1" ];then
+  echo "Pleas input the director of taosdlog."
+  echo "usage: ./get_client.sh <taosdlog directory> <result file>"
+  exit 1
+else
+  log_dir=$1
+fi
+
+if [ ! -n "$2" ];then
+  result_file=clientInfo.txt
+else
+  result_file=$2
+fi
+
+grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file}
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@ -278,11 +278,11 @@ function install_service_on_sysvinit() {

    # Install taosd service
    if ((${os_type}==1)); then
-    ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d
-    ${csudo} cp    ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+    ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d
+    ${csudo} cp    ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
    elif ((${os_type}==2)); then
-    ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d
-    ${csudo} cp    ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+    ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d
+    ${csudo} cp    ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
    fi
    
    #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"
--- a/packaging/tools/makeclient.sh
+++ b/packaging/tools/makeclient.sh
@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then
    strip ${build_dir}/bin/taos
    bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
  else  
-    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
  fi
  lib_files="${build_dir}/lib/libtaos.so.${version}"
 else
@ -110,7 +110,7 @@ mkdir -p ${install_dir}/connector

 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
  if [ "$osType" != "Darwin" ]; then
-    cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+    cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
  fi
  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
  cp -r ${connector_dir}/python        ${install_dir}/connector/
--- a/packaging/tools/makeclient_power.sh
+++ b/packaging/tools/makeclient_power.sh
@ -76,8 +76,10 @@ if [ "$osType" != "Darwin" ]; then
  else 
    cp ${build_dir}/bin/taos          ${install_dir}/bin/power
    cp ${script_dir}/remove_power.sh  ${install_dir}/bin
-    cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo   
+    cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo 
+    cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
    cp ${script_dir}/set_core.sh      ${install_dir}/bin
+    cp ${script_dir}/get_client.sh    ${install_dir}/bin
  fi
 else
  cp ${bin_files} ${install_dir}/bin
@ -135,7 +137,7 @@ mkdir -p ${install_dir}/connector

 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
  if [ "$osType" != "Darwin" ]; then
-    cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+    cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
  fi
  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
  cp -r ${connector_dir}/python  ${install_dir}/connector/
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
  strip ${build_dir}/bin/taos
  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
 fi

 lib_files="${build_dir}/lib/libtaos.so.${version}"
@ -124,7 +124,7 @@ cp ${lib_files} ${install_dir}/driver
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-  cp ${build_dir}/lib/*.jar            ${install_dir}/connector
+  cp ${build_dir}/lib/*.jar            ${install_dir}/connector ||:
  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
  cp -r ${connector_dir}/python        ${install_dir}/connector/
  cp -r ${connector_dir}/go            ${install_dir}/connector
--- a/packaging/tools/makepkg_power.sh
+++ b/packaging/tools/makepkg_power.sh
@ -77,8 +77,10 @@ else
  cp ${build_dir}/bin/taosd         ${install_dir}/bin/powerd
  cp ${script_dir}/remove_power.sh  ${install_dir}/bin
  cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo   
+  cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
  cp ${build_dir}/bin/tarbitrator   ${install_dir}/bin
  cp ${script_dir}/set_core.sh      ${install_dir}/bin
+  cp ${script_dir}/get_client.sh    ${install_dir}/bin
 fi
 chmod a+x ${install_dir}/bin/* || :

@ -156,7 +158,7 @@ cp ${lib_files} ${install_dir}/driver
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-  cp ${build_dir}/lib/*.jar      ${install_dir}/connector
+  cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
  cp -r ${connector_dir}/python  ${install_dir}/connector/
  cp -r ${connector_dir}/go      ${install_dir}/connector
--- a/packaging/tools/post.sh
+++ b/packaging/tools/post.sh
@ -81,8 +81,10 @@ function install_lib() {
    ${csudo} ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
    ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
    
-    ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1           || :
-    ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so   || :
+    if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
+      ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1           || :
+      ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so   || :
+    fi
 }

 function install_bin() {
@ -121,8 +123,11 @@ function install_config() {
    echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}"
    echo
    echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
-    read firstEp
-    while true; do
+    #read firstEp
+    if exec < /dev/tty; then
+    	read firstEp;
+	fi
+	while true; do
        if [ ! -z "$firstEp" ]; then
            # check the format of the firstEp
            #if [[ $firstEp == $FQDN_PATTERN ]]; then
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: '2.0.5.1'
+version: '2.0.7.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@ -72,7 +72,7 @@ parts:
      - usr/bin/taosd
      - usr/bin/taos
      - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.5.1
+      - usr/lib/libtaos.so.2.0.6.0
      - usr/lib/libtaos.so.1
      - usr/lib/libtaos.so

--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@ -490,7 +490,7 @@ static bool balanceMontiorDropping() {

    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
-      if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; 
+      if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; 
      if (mnodeGetDnodesNum() <= 1) continue;

      mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
@ -571,8 +571,8 @@ static void balanceCheckDnodeAccess() {
      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
        pDnode->status = TAOS_DN_STATUS_OFFLINE;
        pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
-        mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
-              pDnode->lastAccess);
+        mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
+              pDnode->lastAccess, pDnode->status);
        balanceSetVgroupOffline(pDnode);
      }
    }
--- a/src/client/inc/tscLocalMerge.h
+++ b/src/client/inc/tscLocalMerge.h
@ -62,11 +62,11 @@ typedef struct SLocalReducer {
  bool                   hasUnprocessedRow;
  tOrderDescriptor *     pDesc;
  SColumnModel *         resColModel;
+  SColumnModel*          finalModel;
  tExtMemBuffer **       pExtMemBuffer;      // disk-based buffer
  SFillInfo*             pFillInfo;          // interpolation support structure
-  char *                 pFinalRes;          // result data after interpo
-  tFilePage *            discardData;
-  SResultInfo *          pResInfo;
+  char*                  pFinalRes;          // result data after interpo
+  tFilePage*             discardData;
  bool                   discard;
  int32_t                offset;             // limit offset value
  bool                   orderPrjOnSTable;   // projection query on stable
@ -75,7 +75,8 @@ typedef struct SLocalReducer {
 typedef struct SRetrieveSupport {
  tExtMemBuffer **  pExtMemBuffer;     // for build loser tree
  tOrderDescriptor *pOrderDescriptor;
-  SColumnModel *    pFinalColModel;    // colModel for final result
+  SColumnModel*     pFinalColModel;    // colModel for final result
+  SColumnModel*     pFFColModel;
  int32_t           subqueryIndex;     // index of current vnode in vnode list
  SSqlObj *         pParentSql;
  tFilePage *       localBuffer;       // temp buffer, there is a buffer for each vnode to
@ -97,7 +98,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
 * create local reducer to launch the second-stage reduce process at client site
 */
 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                           SColumnModel *finalModel, SSqlObj* pSql);
+                           SColumnModel *finalModel, SColumnModel *pFFModel, SSqlObj* pSql);

 void tscDestroyLocalReducer(SSqlObj *pSql);

--- a/src/client/inc/tscSubquery.h
+++ b/src/client/inc/tscSubquery.h
@ -23,7 +23,7 @@ extern "C" {
 #include "tscUtil.h"
 #include "tsclient.h"

-void tscFetchDatablockFromSubquery(SSqlObj* pSql);
+void tscFetchDatablockForSubquery(SSqlObj* pSql);

 void tscSetupOutputColumnIndex(SSqlObj* pSql);
 void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
@ -41,6 +41,8 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql);
 void tscBuildResFromSubqueries(SSqlObj *pSql);
 TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult);

+char *getArithemicInputSrc(void *param, const char *name, int32_t colId);
+
 #ifdef __cplusplus
 }
 #endif
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@ -75,6 +75,7 @@ typedef struct SJoinSupporter {
  SArray*         exprList;
  SFieldInfo      fieldsInfo;
  STagCond        tagCond;
+  SSqlGroupbyExpr groupInfo;       // group by info
  struct STSBuf*  pTSBuf;          // the TSBuf struct that holds the compressed timestamp array
  FILE*           f;               // temporary file in order to create TSBuf
  char            path[PATH_MAX];  // temporary file path, todo dynamic allocate memory
@ -86,8 +87,8 @@ typedef struct SJoinSupporter {
 } SJoinSupporter;

 typedef struct SVgroupTableInfo {
-  SCMVgroupInfo vgInfo;
-  SArray*       itemList;   //SArray<STableIdInfo>
+  SVgroupInfo vgInfo;
+  SArray*     itemList;   //SArray<STableIdInfo>
 } SVgroupTableInfo;

 static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
@ -124,6 +125,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t
 */
 bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo);
 bool tscIsTWAQuery(SQueryInfo* pQueryInfo);
+bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);

 bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex);
 bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
@ -157,7 +159,7 @@ SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t ind
 TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);

 void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
-void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
+void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);

 int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
 void    tscFieldInfoClear(SFieldInfo* pFieldInfo);
@ -166,15 +168,15 @@ static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQue

 int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2);

-void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex);
+void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);

 int32_t   tscGetResRowLength(SArray* pExprList);

 SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-    int16_t size, int16_t interSize, bool isTagCol);
+    int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);

 SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                           int16_t size, int16_t interSize, bool isTagCol);
+                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);

 SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
                           int16_t size);
@ -225,8 +227,9 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo);

 void tscClearSubqueryInfo(SSqlCmd* pCmd);
 void tscFreeVgroupTableInfo(SArray* pVgroupTables);
-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables);
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables);
 void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);

 int  tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
 int  tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
@ -237,7 +240,7 @@ void tscDoQuery(SSqlObj* pSql);

 SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
 void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src);
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
 /**
 * The create object function must be successful expect for the out of memory issue.
 *
@ -265,6 +268,7 @@ void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub
 void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);

 int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);

 void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);

--- a/src/client/inc/tschemautil.h
+++ b/src/client/inc/tschemautil.h
@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
 * @param colId
 * @return
 */
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);

 /**
 * check if the schema is valid or not, including following aspects:
@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema();
 */
 STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);

-//todo tags value as well as the table id structure needs refactor
-char *tsGetTagsValue(STableMeta *pMeta);
-
 #ifdef __cplusplus
 }
 #endif
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@ -30,6 +30,7 @@ extern "C" {
 #include "tsqlfunction.h"
 #include "tutil.h"
 #include "tcache.h"
+#include "tref.h"

 #include "qExecutor.h"
 #include "qSqlparser.h"
@ -89,12 +90,12 @@ typedef struct STableComInfo {
  int32_t rowSize;
 } STableComInfo;

-typedef struct SCMCorVgroupInfo {
-  int32_t    version;
-  int8_t     inUse;
-  int8_t     numOfEps;
-  SEpAddr1   epAddr[TSDB_MAX_REPLICA];
-} SCMCorVgroupInfo;
+typedef struct SCorVgroupInfo {
+  int32_t  version;
+  int8_t   inUse;
+  int8_t   numOfEps;
+  SEpAddr1 epAddr[TSDB_MAX_REPLICA];
+} SCorVgroupInfo;

 typedef struct STableMeta {
  STableComInfo  tableInfo;
@ -102,8 +103,8 @@ typedef struct STableMeta {
  int16_t        sversion;
  int16_t        tversion;
  char           sTableId[TSDB_TABLE_FNAME_LEN];
-  SCMVgroupInfo  vgroupInfo;
-  SCMCorVgroupInfo  corVgroupInfo;
+  SVgroupInfo    vgroupInfo;
+  SCorVgroupInfo corVgroupInfo;
  STableId       id;
  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
 } STableMeta;
@ -127,7 +128,7 @@ typedef struct STableMetaInfo {
 typedef struct SSqlExpr {
  char      aliasName[TSDB_COL_NAME_LEN];  // as aliasName
  SColIndex colInfo;
-  int64_t   uid;            // refactor use the pointer
+  uint64_t  uid;            // refactor use the pointer
  int16_t   functionId;     // function id in aAgg array
  int16_t   resType;        // return value type
  int16_t   resBytes;       // length of return value
@ -135,6 +136,7 @@ typedef struct SSqlExpr {
  int16_t   numOfParams;    // argument value of each function
  tVariant  param[3];       // parameters are not more than 3
  int32_t   offset;         // sub result column value of arithmetic expression.
+  int16_t   resColId;          // result column id
 } SSqlExpr;

 typedef struct SColumnIndex {
@ -250,6 +252,7 @@ typedef struct SQueryInfo {
  int64_t          clauseLimit;   // limit for current sub clause
  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
  int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
+  int16_t          resColumnId;   // result column id
 } SQueryInfo;

 typedef struct {
@ -329,6 +332,7 @@ typedef struct STscObj {
  char               writeAuth : 1;
  char               superAuth : 1;
  uint32_t           connId;
+  uint64_t           rid;      // ref ID returned by taosAddRef
  struct SSqlObj *   pHb;
  struct SSqlObj *   sqlList;
  struct SSqlStream *streamList;
@ -338,16 +342,16 @@ typedef struct STscObj {
 } STscObj;

 typedef struct SSubqueryState {
-  int32_t          numOfRemain;         // the number of remain unfinished subquery
-  int32_t          numOfSub;            // the number of total sub-queries
-  uint64_t         numOfRetrievedRows;  // total number of points in this query
+  int32_t  numOfRemain;         // the number of remain unfinished subquery
+  int32_t  numOfSub;            // the number of total sub-queries
+  uint64_t numOfRetrievedRows;  // total number of points in this query
 } SSubqueryState;

 typedef struct SSqlObj {
  void            *signature;
  pthread_t        owner;        // owner of sql object, by which it is executed
  STscObj         *pTscObj;
-  void            *pRpcCtx;
+  int64_t          rpcRid;
  void            (*fp)();
  void            (*fetchFp)();
  void            *param;
@ -430,14 +434,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
 */
 void tscFreeSqlResult(SSqlObj *pSql);

-/**
- * only free part of resources allocated during query.
- * TODO remove it later
- * Note: this function is multi-thread safe.
- * @param pObj
- */
-void tscPartiallyFreeSqlObj(SSqlObj *pSql);
-
 /**
 * free sql object, release allocated resource
 * @param pObj
@ -446,7 +442,7 @@ void tscFreeSqlObj(SSqlObj *pSql);
 void tscFreeRegisteredSqlObj(void *pSql);
 void tscFreeTableMetaHelper(void *pTableMeta);

-void tscCloseTscObj(STscObj *pObj);
+void tscCloseTscObj(void *pObj);

 // todo move to taos? or create a new file: taos_internal.h
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
@ -468,17 +464,16 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo,  const char* s

 int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);

-static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
+static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex, int32_t offset) {
  SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pFieldInfo->internalField, columnIndex);
-  assert(pInfo->pSqlExpr != NULL);

-  int32_t type = pInfo->pSqlExpr->resType;
-  int32_t bytes = pInfo->pSqlExpr->resBytes;
+  int32_t type = pInfo->field.type;
+  int32_t bytes = pInfo->field.bytes;

-  char* pData = pRes->data + (int32_t)(pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row);
+  char* pData = pRes->data + (int32_t)(offset * pRes->numOfRows + bytes * pRes->row);

  // user defined constant value output columns
-  if (TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) {
+  if (pInfo->pSqlExpr != NULL && TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) {
    if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
      pData = pInfo->pSqlExpr->param[1].pz;
      pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen;
@ -516,13 +511,14 @@ extern void *    tscQhandle;
 extern int       tscKeepConn[];
 extern int       tsInsertHeadSize;
 extern int       tscNumOfThreads;
+extern int       tscRefId;
  
 extern SRpcCorEpSet tscMgmtEpSet;

 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

-int32_t tscCompareTidTags(const void* p1, const void* p2);
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
+int16_t getNewResColId(SQueryInfo* pQueryInfo);

 #ifdef __cplusplus
 }
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
  }

  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscFetchDatablockFromSubquery(pSql);
+    tscFetchDatablockForSubquery(pSql);
  } else {
    tscProcessSql(pSql);
  }
@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) {
  
  // handle the sub queries of join query
  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscFetchDatablockFromSubquery(pSql);
+    tscFetchDatablockForSubquery(pSql);
  } else if (pRes->completed) {
    if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) {
      if (hasMoreVnodesToTry(pSql)) {  // sequentially retrieve data from remain vnodes.
@ -351,7 +351,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
    SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);

    if (pSup->pSqlExpr != NULL) {
-      tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
+      tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0);
    } else {
 //      todo add
    }
@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
  SSqlRes *pRes = &pSql->res;
  pRes->code = code;

-  const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
+  SSqlObj *sub = (SSqlObj*) res;
+  const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
  if (code != TSDB_CODE_SUCCESS) {
    tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
    goto _error;
@ -427,8 +428,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
      } else {
        assert(code == TSDB_CODE_SUCCESS);      
      }
-     
-      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL);
+
+      // param already freed by other routine and pSql in tscCache when ctrl + c 
+      if (atomic_load_ptr(&pSql->param) == NULL) {
+        return;
+      }
+      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));

      SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
      SSqlObj *         pParObj = trs->pParentSql;
@ -437,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
      assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
          pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);

+      // tscProcessSql can add error into async res
+      tscProcessSql(pSql);
+      return;
+    } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
+      tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
+      STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+      code = tscGetTableMeta(pSql, pTableMetaInfo);
+      if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+        return;
+      } else {
+        assert(code == TSDB_CODE_SUCCESS);
+      }
+
+      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
      // tscProcessSql can add error into async res
      tscProcessSql(pSql);
      return;
@ -461,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
          tscResetSqlCmdObj(pCmd, false);

          code = tsParseSql(pSql, true);
-
          if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
            return;
          } else if (code != TSDB_CODE_SUCCESS) {
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@ -49,82 +49,6 @@ typedef struct SCreateBuilder {
 } SCreateBuilder; 
 static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);

-static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) {
-  char buf[512] = {0};
-
-  int32_t len = 0;
-  int32_t MAX_BOOL_TYPE_LENGTH = 5;  // max(strlen("true"), strlen("false"));
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-      return length;
-    case TSDB_DATA_TYPE_NCHAR:
-      return length;
-    case TSDB_DATA_TYPE_DOUBLE: {
-      double dv = 0;
-      dv = GET_DOUBLE_VAL(pData);
-      len = sprintf(buf, "%lf", dv);
-      if (strncasecmp("nan", buf, 3) == 0) {
-        len = 4;
-      }
-    } break;
-    case TSDB_DATA_TYPE_FLOAT: {
-      float fv = 0;
-      fv = GET_FLOAT_VAL(pData);
-      len = sprintf(buf, "%f", fv);
-      if (strncasecmp("nan", buf, 3) == 0) {
-        len = 4;
-      }
-    } break;
-    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_BIGINT:
-      len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
-      break;
-    case TSDB_DATA_TYPE_BOOL:
-      len = MAX_BOOL_TYPE_LENGTH;
-      break;
-    default:
-      len = sprintf(buf, "%d", *(int32_t *)pData);
-      break;
-  };
-  return len;
-}
-
-/*
- * we need to convert all data into string, so we need to sprintf all kinds of
- * non-string data into string, and record its length to get the right
- * maximum length. The length may be less or greater than its original binary length:
- * For example:
- * length((short) 1) == 1, less than sizeof(short)
- * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t)
- */
-static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) {
-  STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta;
-
-  if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE ||
-      pMeta->tableType == TSDB_STREAM_TABLE) {
-    return 0;
-  }
-
-  char *   pTagValue = tsGetTagsValue(pMeta);
-  SSchema *pTagsSchema = tscGetTableTagSchema(pMeta);
-
-  int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type);
-
-  pTagValue += pTagsSchema[0].bytes;
-  int32_t numOfTags = tscGetNumOfTags(pMeta);
-  
-  for (int32_t i = 1; i < numOfTags; ++i) {
-    int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type);
-    if (len < tLen) {
-      len = tLen;
-    }
-
-    pTagValue += pTagsSchema[i].bytes;
-  }
-
-  return len;
-}
-
 static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
  SSqlRes *pRes = &pSql->res;

@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
    return 0;
  }

-  // the following is handle display tags value for meters created according to metric
-  char *pTagValue = tsGetTagsValue(pMeta);
+  // the following is handle display tags for table created according to super table
  for (int32_t i = numOfRows; i < totalNumOfRows; ++i) {
    // field name
    TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
    char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
    const char *src = "TAG";
    STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
-
-    pTagValue += pSchema[i].bytes;
  }

  return 0;
@ -241,7 +162,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
  
  SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
-      (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
+      (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false);
  
  rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);

@ -251,7 +172,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
  
  pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE),
-      typeColLength, false);
+      -1000, typeColLength, false);
  
  rowLen += typeColLength + VARSTR_HEADER_SIZE;

@ -261,7 +182,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
  
  pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
-      sizeof(int32_t), false);
+      -1000, sizeof(int32_t), false);
  
  rowLen += sizeof(int32_t);

@ -271,7 +192,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
  
  pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE),
-      noteColLength, false);
+      -1000, noteColLength, false);
  
  rowLen += noteColLength + VARSTR_HEADER_SIZE;
  return rowLen;
@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
  const int32_t TYPE_COLUMN_LENGTH = 16;
  const int32_t NOTE_COLUMN_MIN_LENGTH = 8;

-  int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql);
-  if (noteFieldLen == 0) {
-    noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
-  }
+  int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql);
+//  if (noteFieldLen == 0) {
+//    noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
+//  }

  int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
  tscFieldInfoUpdateOffset(pQueryInfo);
@ -486,8 +407,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const
  } 

  SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
-  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
-      f.bytes, f.bytes - VARSTR_HEADER_SIZE, false);
+  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false);

  rowLen += f.bytes; 

@ -501,7 +421,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const

  pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
  pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, 
-      (int16_t)(ddlLen + VARSTR_HEADER_SIZE), ddlLen, false);
+      (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false);

  rowLen += ddlLen + VARSTR_HEADER_SIZE;

@ -698,7 +618,11 @@ static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName,
  for (int32_t i = 0; i < numOfRows; ++i) {
    uint8_t type = pSchema[i].type;
    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+      int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+      if (type == TSDB_DATA_TYPE_NCHAR) {
+        bytes =  bytes/TSDB_NCHAR_SIZE;
+      }
+      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName, bytes);
    } else {
      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName); 
    }
@ -721,7 +645,11 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName,
  for (int32_t i = 0; i < numOfRows; ++i) {
    uint8_t type = pSchema[i].type;
    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+      int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+      if (type == TSDB_DATA_TYPE_NCHAR) {
+        bytes =  bytes/TSDB_NCHAR_SIZE;
+      }
+      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes);
    } else {
      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); 
    }
@ -731,7 +659,11 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName,
  for (int32_t i = numOfRows; i < totalRows; i++) {
    uint8_t type = pSchema[i].type;
    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+      int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+      if (type == TSDB_DATA_TYPE_NCHAR) {
+        bytes =  bytes/TSDB_NCHAR_SIZE;
+      }
+      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes);
    } else {
      snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); 
    }
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@ -13,14 +13,15 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "tscLocalMerge.h"
+#include "tscSubquery.h"
 #include "os.h"
+#include "qAst.h"
 #include "tlosertree.h"
+#include "tscLog.h"
 #include "tscUtil.h"
 #include "tschemautil.h"
 #include "tsclient.h"
-#include "tutil.h"
-#include "tscLog.h"
-#include "tscLocalMerge.h"

 typedef struct SCompareParam {
  SLocalDataSource **pLocalData;
@ -29,6 +30,8 @@ typedef struct SCompareParam {
  int32_t            groupOrderType;
 } SCompareParam;

+static void doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize);
+
 int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
  int32_t pLeftIdx = *(int32_t *)pLeft;
  int32_t pRightIdx = *(int32_t *)pRight;
@ -97,14 +100,14 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
      pCtx->param[2].i64Key = pQueryInfo->order.order;
      pCtx->param[2].nType  = TSDB_DATA_TYPE_BIGINT;
      pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
+    } else if (functionId == TSDB_FUNC_APERCT) {
+      pCtx->param[0].i64Key = pExpr->param[0].i64Key;
+      pCtx->param[0].nType  = pExpr->param[0].nType;
    }

-    SResultInfo *pResInfo = &pReducer->pResInfo[i];
-    pResInfo->bufLen = pExpr->interBytes;
-    pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen);
-
-    pCtx->resultInfo = &pReducer->pResInfo[i];
-    pCtx->resultInfo->superTableQ = true;
+    pCtx->interBufBytes = pExpr->interBytes;
+    pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo));
+    pCtx->stableQuery = true;
  }

  int16_t          n = 0;
@ -132,28 +135,41 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
 }

 static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) {
-  int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
+  int32_t numOfCols = (int32_t)tscNumOfFields(pQueryInfo);
  int32_t offset = 0;
  
  SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
  for(int32_t i = 0; i < numOfCols; ++i) {
-    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-    
-    pFillCol[i].col.bytes  = pExpr->resBytes;
-    pFillCol[i].col.type   = (int8_t)pExpr->resType;
-    pFillCol[i].col.colId  = pExpr->colInfo.colId;
-    pFillCol[i].flag       = pExpr->colInfo.flag;
-    pFillCol[i].col.offset = offset;
-    pFillCol[i].functionId = pExpr->functionId;
-    pFillCol[i].fillVal.i  = pQueryInfo->fillVal[i];
-    offset += pExpr->resBytes;
+    SInternalField* pIField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
+
+    if (pIField->pArithExprInfo == NULL) {
+      SSqlExpr* pExpr = pIField->pSqlExpr;
+
+      pFillCol[i].col.bytes  = pExpr->resBytes;
+      pFillCol[i].col.type   = (int8_t)pExpr->resType;
+      pFillCol[i].col.colId  = pExpr->colInfo.colId;
+      pFillCol[i].flag       = pExpr->colInfo.flag;
+      pFillCol[i].col.offset = offset;
+      pFillCol[i].functionId = pExpr->functionId;
+      pFillCol[i].fillVal.i  = pQueryInfo->fillVal[i];
+    } else {
+      pFillCol[i].col.bytes  = pIField->field.bytes;
+      pFillCol[i].col.type   = (int8_t)pIField->field.type;
+      pFillCol[i].col.colId  = -100;
+      pFillCol[i].flag       = TSDB_COL_NORMAL;
+      pFillCol[i].col.offset = offset;
+      pFillCol[i].functionId = -1;
+      pFillCol[i].fillVal.i  = pQueryInfo->fillVal[i];
+    }
+
+    offset += pFillCol[i].col.bytes;
  }
  
  return pFillCol;
 }

 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                           SColumnModel *finalmodel, SSqlObj* pSql) {
+                           SColumnModel *finalmodel, SColumnModel *pFFModel, SSqlObj* pSql) {
  SSqlCmd* pCmd = &pSql->cmd;
  SSqlRes* pRes = &pSql->res;
  
@ -227,7 +243,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
      if (ds == NULL) {
        tscError("%p failed to create merge structure", pSql);
        pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-        taosTFree(pReducer);
+        tfree(pReducer);
        return;
      }
      
@ -254,7 +270,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
      
      if (ds->filePage.num == 0) {  // no data in this flush, the index does not increase
        tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx);
-        taosTFree(ds);
+        tfree(ds);
        continue;
      }
      
@ -264,7 +280,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
  
  // no data actually, no need to merge result.
  if (idx == 0) {
-    taosTFree(pReducer);
+    tfree(pReducer);
    return;
  }

@ -272,7 +288,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd

  SCompareParam *param = malloc(sizeof(SCompareParam));
  if (param == NULL) {
-    taosTFree(pReducer);
+    tfree(pReducer);
    return;
  }

@ -286,8 +302,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd

  pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
  if (pReducer->pLoserTree == NULL || pRes->code != 0) {
-    taosTFree(param);
-    taosTFree(pReducer);
+    tfree(param);
+    tfree(pReducer);
    return;
  }

@ -330,22 +346,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd

  if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
      /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
-    taosTFree(pReducer->pTempBuffer);
-    taosTFree(pReducer->discardData);
-    taosTFree(pReducer->pResultBuf);
-    taosTFree(pReducer->pFinalRes);
-    taosTFree(pReducer->prevRowOfInput);
-    taosTFree(pReducer->pLoserTree);
-    taosTFree(param);
-    taosTFree(pReducer);
+    tfree(pReducer->pTempBuffer);
+    tfree(pReducer->discardData);
+    tfree(pReducer->pResultBuf);
+    tfree(pReducer->pFinalRes);
+    tfree(pReducer->prevRowOfInput);
+    tfree(pReducer->pLoserTree);
+    tfree(param);
+    tfree(pReducer);
    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
    return;
  }
  
-  size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
-  
  pReducer->pTempBuffer->num = 0;
-  pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));

  tscCreateResPointerInfo(pRes, pQueryInfo);
  tscInitSqlContext(pCmd, pReducer, pDesc);
@ -373,8 +386,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
    SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
    pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
-                                           4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
-                                           tinfo.precision, pQueryInfo->fillType, pFillCol);
+                                           4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
+                                           tinfo.precision, pQueryInfo->fillType, pFillCol, pSql);
  }
 }

@ -489,47 +502,41 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
      tscDebug("%p waiting for delete procedure, status: %d", pSql, status);
    }

-    pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
+    pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo);

    if (pLocalReducer->pCtx != NULL) {
-      for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+      int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+      for (int32_t i = 0; i < numOfExprs; ++i) {
        SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];

        tVariantDestroy(&pCtx->tag);
+        tfree(pCtx->resultInfo);
+
        if (pCtx->tagInfo.pTagCtxList != NULL) {
-          taosTFree(pCtx->tagInfo.pTagCtxList);
+          tfree(pCtx->tagInfo.pTagCtxList);
        }
      }

-      taosTFree(pLocalReducer->pCtx);
+      tfree(pLocalReducer->pCtx);
    }

-    taosTFree(pLocalReducer->prevRowOfInput);
+    tfree(pLocalReducer->prevRowOfInput);

-    taosTFree(pLocalReducer->pTempBuffer);
-    taosTFree(pLocalReducer->pResultBuf);
-
-    if (pLocalReducer->pResInfo != NULL) {
-      size_t num = tscSqlExprNumOfExprs(pQueryInfo);
-      for (int32_t i = 0; i < num; ++i) {
-        taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
-      }
-
-      taosTFree(pLocalReducer->pResInfo);
-    }
+    tfree(pLocalReducer->pTempBuffer);
+    tfree(pLocalReducer->pResultBuf);

    if (pLocalReducer->pLoserTree) {
-      taosTFree(pLocalReducer->pLoserTree->param);
-      taosTFree(pLocalReducer->pLoserTree);
+      tfree(pLocalReducer->pLoserTree->param);
+      tfree(pLocalReducer->pLoserTree);
    }

-    taosTFree(pLocalReducer->pFinalRes);
-    taosTFree(pLocalReducer->discardData);
+    tfree(pLocalReducer->pFinalRes);
+    tfree(pLocalReducer->discardData);

    tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel,
                              pLocalReducer->numOfVnode);
    for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) {
-      taosTFree(pLocalReducer->pLocalDataSrc[i]);
+      tfree(pLocalReducer->pLocalDataSrc[i]);
    }

    pLocalReducer->numOfBuffer = 0;
@ -563,7 +570,8 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
  if (numOfGroupByCols > 0) {

    if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
-      int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
+      int32_t numOfInternalOutput = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+      int32_t startCols = numOfInternalOutput - pQueryInfo->groupbyExpr.numOfGroupCols;

      // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns
      for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
@ -596,7 +604,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
  }

  *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
-  taosTFree(orderColIndexList);
+  tfree(orderColIndexList);

  if (*pOrderDesc == NULL) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -682,6 +690,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr

    pSchema[i].bytes = pExpr->resBytes;
    pSchema[i].type = (int8_t)pExpr->resType;
+    tstrncpy(pSchema[i].name, pExpr->aliasName, tListLen(pSchema[i].name));
+
    rlen += pExpr->resBytes;
  }

@ -707,7 +717,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr

  if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    taosTFree(pSchema);
+    tfree(pSchema);
    return pRes->code;
  }

@ -744,8 +754,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
  }
  
  *pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity);
-  taosTFree(pSchema);

+   tfree(pSchema);
  return TSDB_CODE_SUCCESS;
 }

@ -764,7 +774,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
    pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
  }

-  taosTFree(pMemBuffer);
+  tfree(pMemBuffer);
 }

 /**
@ -924,7 +934,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
  }

  while (1) {
-    int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
+    int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);

    if (pQueryInfo->limit.offset < newRows) {
      newRows -= pQueryInfo->limit.offset;
@ -953,7 +963,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
        }

        // all output in current group are completed
-        int32_t totalRemainRows = (int32_t)getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
+        int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
        if (totalRemainRows <= 0) {
          break;
        }
@ -974,10 +984,11 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
      savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo);
    }

+    int32_t offset = 0;
    for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
      TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-      int16_t     offset = getColumnModelOffset(pLocalReducer->resColModel, i);
      memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows));
+      offset += pField->bytes;
    }

    pRes->numOfRowsGroup += pRes->numOfRows;
@ -986,10 +997,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO

  pBeforeFillData->num = 0;
  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    taosTFree(pResPages[i]);
+    tfree(pResPages[i]);
  }
  
-  taosTFree(pResPages);
+  tfree(pResPages);
 }

 static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
@ -1072,7 +1083,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
      continue;
    }

-    SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
+    SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
    if (maxOutput < pResInfo->numOfRes) {
      maxOutput = pResInfo->numOfRes;
    }
@ -1230,6 +1241,10 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur

  tColModelCompact(pModel, pResBuf, pModel->capacity);

+  if (tscIsSecondStageQuery(pQueryInfo)) {
+    doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize);
+  }
+
 #ifdef _DEBUG_VIEW
  printf("final result before interpo:\n");
 //  tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
@ -1253,10 +1268,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
  return true;
 }

-void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {  // reset output buffer to the beginning
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    pLocalReducer->pCtx[i].aOutputBuf =
-        pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity;
+void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning
+  size_t t = tscSqlExprNumOfExprs(pQueryInfo);
+  for (int32_t i = 0; i < t; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity;
  }

  memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage));
@ -1301,7 +1317,7 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
    int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));

    // the first column must be the timestamp column
-    int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+    int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
    if (rows > 0) {  // do fill gap
      doFillResult(pSql, pLocalReducer, false);
    }
@ -1330,7 +1346,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
      ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) {
      int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey;

-      int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+      int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
      if (rows > 0) {
        doFillResult(pSql, pLocalReducer, true);
      }
@ -1501,8 +1517,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
          if (pLocalReducer->discard && sameGroup) {
            pLocalReducer->hasUnprocessedRow = false;
            tmpBuffer->num = 0;
-          } else {
-            // current row does not belongs to the previous group, so it is not be handled yet.
+          } else { // current row does not belongs to the previous group, so it is not be handled yet.
            pLocalReducer->hasUnprocessedRow = true;
          }

@ -1596,3 +1611,44 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
  pRes->pLocalReducer->pResultBuf->num = numOfRes;
  pRes->data = pRes->pLocalReducer->pResultBuf->data;
 }
+
+void doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
+  char* pbuf = calloc(1, pOutput->num * rowSize);
+
+  size_t size = tscNumOfFields(pQueryInfo);
+  SArithmeticSupport arithSup = {0};
+
+  // todo refactor
+  arithSup.offset     = 0;
+  arithSup.numOfCols  = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+  arithSup.exprList   = pQueryInfo->exprList;
+  arithSup.data       = calloc(arithSup.numOfCols, POINTER_BYTES);
+
+  for(int32_t k = 0; k < arithSup.numOfCols; ++k) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
+    arithSup.data[k] = (pOutput->data + pOutput->num* pExpr->offset);
+  }
+
+  int32_t offset = 0;
+
+  for (int i = 0; i < size; ++i) {
+    SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
+    
+    // calculate the result from several other columns
+    if (pSup->pArithExprInfo != NULL) {
+      arithSup.pArithExpr = pSup->pArithExprInfo;
+      tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc);
+    } else {
+      SSqlExpr* pExpr = pSup->pSqlExpr;
+      memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num);
+    }
+
+    offset += pSup->field.bytes;
+  }
+
+  assert(finalRowSize <= rowSize);
+  memcpy(pOutput->data, pbuf, pOutput->num * finalRowSize);
+
+  tfree(pbuf);
+  tfree(arithSup.data);
+}
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
  }

  int32_t code = TSDB_CODE_TSC_INVALID_SQL;
-  char *  tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
+  char *  tmpTokenBuf = calloc(1, 16*1024);  // used for deleting Escape character: \\, \', \"
  if (NULL == tmpTokenBuf) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }
@ -1148,6 +1148,10 @@ int tsParseInsertSql(SSqlObj *pSql) {

      index = 0;
      sToken = tStrGetToken(str, &index, false, 0, NULL);
+      if (sToken.type != TK_STRING && sToken.type != TK_ID) {
+        code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
+        goto _error;
+      }
      str += index;
      if (sToken.n == 0) {
        code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
@ -1309,7 +1313,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
  if ((!pCmd->parseFinished) && (!initial)) {
    tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql);
  }
-  
+
  ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
  if (TSDB_CODE_SUCCESS != ret) {
    return ret;
@ -1406,7 +1410,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
    assert(taos_errno(pSql) == code);

    taos_free_result(pSql);
-    taosTFree(pSupporter);
+    tfree(pSupporter);
    fclose(fp);

    pParentSql->res.code = code;
@ -1445,7 +1449,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {

  char *tokenBuf = calloc(1, 4096);

-  while ((readLen = taosGetline(&line, &n, fp)) != -1) {
+  while ((readLen = tgetline(&line, &n, fp)) != -1) {
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }
@ -1470,7 +1474,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
    }
  }

-  taosTFree(tokenBuf);
+  tfree(tokenBuf);
  free(line);

  if (count > 0) {
@ -1483,7 +1487,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {

  } else {
    taos_free_result(pSql);
-    taosTFree(pSupporter);
+    tfree(pSupporter);
    fclose(fp);

    pParentSql->fp = pParentSql->fetchFp;
@ -1513,7 +1517,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
    pSql->res.code = TAOS_SYSTEM_ERROR(errno);
    tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));

-    taosTFree(pSupporter)
+    tfree(pSupporter)
    tscQueueAsyncRes(pSql);

    return;
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
 }

 int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
-  SCMHeartBeatMsg *pHeartbeat = pMsg;
+  SHeartBeatMsg *pHeartbeat = pMsg;
  int allocedQueriesNum = pHeartbeat->numOfQueries;
  int allocedStreamsNum = pHeartbeat->numOfStreams;
  
@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
  }

  int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
-                   sizeof(SCMHeartBeatMsg);
+                   sizeof(SHeartBeatMsg);
  pHeartbeat->connId = htonl(pObj->connId);
  pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
  pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex)
 }

 // TODO for large number of columns, employ the binary search method
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
  STableComInfo tinfo = tscGetTableInfo(pTableMeta);

  for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
@ -130,17 +130,7 @@ SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
  return NULL;
 }

-struct SSchema tscGetTbnameColumnSchema() {
-  struct SSchema s = {
-      .colId = TSDB_TBNAME_COLUMN_INDEX,
-      .type  = TSDB_DATA_TYPE_BINARY,
-      .bytes = TSDB_TABLE_NAME_LEN
-  };
-  
-  strcpy(s.name, TSQL_TBNAME_L);
-  return s;
-}
-static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) {
+static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
  corVgroupInfo->version = 0;
  corVgroupInfo->inUse = 0;
  corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
@ -166,7 +156,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
  pTableMeta->id.tid = pTableMetaMsg->tid;
  pTableMeta->id.uid = pTableMetaMsg->uid;

-  SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
+  SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
  pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
  pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;

@ -197,28 +187,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
  return pTableMeta;
 }

-/**
- * the TableMeta data format in memory is as follows:
- *
- * +--------------------+
- * |STableMeta Body data|  sizeof(STableMeta)
- * +--------------------+
- * |Schema data         |  numOfTotalColumns * sizeof(SSchema)
- * +--------------------+
- * |Tags data           |  tag_col_1.bytes + tag_col_2.bytes + ....
- * +--------------------+
- *
- * @param pTableMeta
- * @return
- */
-char* tsGetTagsValue(STableMeta* pTableMeta) {
-  int32_t offset = 0;
-//  int32_t  numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags;
-//  uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema);
-
-  return ((char*)pTableMeta + offset);
-}
-
 // todo refactor
 UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
  for (int32_t i = 0; i < num; ++i) {
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) {
  return initial * (2<<(count - 2));
 }

-static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
+static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
  assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);

  SRpcEpSet* pEpSet = &pSql->epSet;
@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
  tscMgmtEpSet.epSet = *pEpSet;
  taosCorEndWrite(&tscMgmtEpSet.version);
 }
-static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
+static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
  if (pVgroupInfo == NULL) { return;}
  taosCorBeginRead(&pVgroupInfo->version);
  int8_t inUse = pVgroupInfo->inUse;
@ -117,14 +117,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
  SSqlCmd *pCmd = &pObj->cmd;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
-  SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
+  SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;

  taosCorBeginWrite(&pVgroupInfo->version);
  tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
  pVgroupInfo->inUse = pEpSet->inUse;
  pVgroupInfo->numOfEps = pEpSet->numOfEps;
  for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
-    taosTFree(pVgroupInfo->epAddr[i].fqdn);
+    tfree(pVgroupInfo->epAddr[i].fqdn);
    pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
    pVgroupInfo->epAddr[i].port = pEpSet->port[i];
  }
@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
  if (pObj == NULL) return;

  if (pObj != pObj->signature) {
-    tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
+    tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
    return;
  }

@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
  SSqlRes *pRes = &pSql->res;

  if (code == 0) {
-    SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
-    SRpcEpSet *      epSet = &pRsp->epSet;
+    SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp;
+    SRpcEpSet *    epSet = &pRsp->epSet;
    if (epSet->numOfEps > 0) {
      tscEpSetHtons(epSet);
      tscUpdateMgmtEpSet(epSet);
-    } 
+    }

    pSql->pTscObj->connId = htonl(pRsp->connId);

@ -175,44 +175,44 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
      if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
    }
  } else {
-    tscDebug("heartbeat failed, code:%s", tstrerror(code));
+    tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
  }

  if (pObj->pHb != NULL) {
    int32_t waitingDuring = tsShellActivityTimer * 500;
-    tscDebug("%p start heartbeat in %dms", pSql, waitingDuring);
+    tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);

-    taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
+    taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
  } else {
    tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
  }
 }

 void tscProcessActivityTimer(void *handle, void *tmrId) {
-  STscObj *pObj = (STscObj *)handle;
-  if (pObj == NULL || pObj->signature != pObj) {
-    return;
-  }
+  int64_t rid = (int64_t) handle;
+  STscObj *pObj = taosAcquireRef(tscRefId, rid);
+  if (pObj == NULL) return; 

  SSqlObj* pHB = pObj->pHb;
-  if (pObj->pTimer != tmrId || pHB == NULL) {
-    return;
-  }

  void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
  if (p == NULL) {
    tscWarn("%p HB object has been released already", pHB);
+    taosReleaseRef(tscRefId, pObj->rid);
    return;
  }

  assert(*pHB->self == pHB);

+  pHB->retry = 0;
  int32_t code = tscProcessSql(pHB);
  taosCacheRelease(tscObjCache, (void**) &p, false);

  if (code != TSDB_CODE_SUCCESS) {
    tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
  }
+
+  taosReleaseRef(tscRefId, rid);
 }

 int tscSendMsgToServer(SSqlObj *pSql) {
@ -237,15 +237,11 @@ int tscSendMsgToServer(SSqlObj *pSql) {
      .pCont   = pMsg,
      .contLen = pSql->cmd.payloadLen,
      .ahandle = pSql,
-      .handle  = &pSql->pRpcCtx,
+      .handle  = NULL,
      .code    = 0
  };

-  // NOTE: the rpc context should be acquired before sending data to server.
-  // Otherwise, the pSql object may have been released already during the response function, which is
-  // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
-  // cause crash.
-  rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
+  rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid);
  return TSDB_CODE_SUCCESS;
 }

@ -265,7 +261,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
  SSqlCmd *pCmd = &pSql->cmd;

  assert(*pSql->self == pSql);
-  pSql->pRpcCtx = NULL;
+  pSql->rpcRid = -1;

  if (pObj->signature != pObj) {
    tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
@ -361,7 +357,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
        memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen);
      }
    } else {
-      pRes->pRsp = NULL;
+      tfree(pRes->pRsp);
    }

    /*
@ -549,11 +545,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
  SQueryInfo *         pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);

  int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-  
-  size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+  size_t  numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
  int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs);
-  
-  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
+
+  int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0;
+
+  int32_t tableSerialize = 0;
+  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  if (pTableMetaInfo->pVgroupTables != NULL) {
+    size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+
+    int32_t totalTables = 0;
+    for (int32_t i = 0; i < numOfGroups; ++i) {
+      SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i);
+      totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList);
+    }
+
+    tableSerialize = totalTables * sizeof(STableIdInfo);
+  }
+
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize +
+         tableSerialize + 4096;
 }

 static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
@ -563,7 +576,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
    
-    SCMVgroupInfo* pVgroupInfo = NULL;
+    SVgroupInfo* pVgroupInfo = NULL;
    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
      int32_t index = pTableMetaInfo->vgroupIndex;
      assert(index >= 0);
@ -685,7 +698,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  pQueryMsg->queryType      = htonl(pQueryInfo->type);
  
  size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
-  pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);
+  pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number

  // set column list ids
  size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
@ -747,12 +760,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
      return TSDB_CODE_TSC_INVALID_SQL;
    }

+    assert(pExpr->resColId < 0);
+
    pSqlFuncExpr->colInfo.colId    = htons(pExpr->colInfo.colId);
    pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
    pSqlFuncExpr->colInfo.flag     = htons(pExpr->colInfo.flag);

    pSqlFuncExpr->functionId  = htons(pExpr->functionId);
    pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams);
+    pSqlFuncExpr->resColId    = htons(pExpr->resColId);
    pMsg += sizeof(SSqlFuncMsg);

    for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
@ -770,7 +786,73 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

    pSqlFuncExpr = (SSqlFuncMsg *)pMsg;
  }
-  
+
+  if(tscIsSecondStageQuery(pQueryInfo)) {
+    size_t output = tscNumOfFields(pQueryInfo);
+    pQueryMsg->secondStageOutput = htonl((int32_t) output);
+
+    SSqlFuncMsg *pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+
+    for (int32_t i = 0; i < output; ++i) {
+      SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i);
+      SSqlExpr *pExpr = pField->pSqlExpr;
+      if (pExpr != NULL) {
+        if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) {
+          tscError("%p table schema is not matched with parsed sql", pSql);
+          return TSDB_CODE_TSC_INVALID_SQL;
+        }
+
+        pSqlFuncExpr1->colInfo.colId    = htons(pExpr->colInfo.colId);
+        pSqlFuncExpr1->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
+        pSqlFuncExpr1->colInfo.flag     = htons(pExpr->colInfo.flag);
+
+        pSqlFuncExpr1->functionId  = htons(pExpr->functionId);
+        pSqlFuncExpr1->numOfParams = htons(pExpr->numOfParams);
+        pMsg += sizeof(SSqlFuncMsg);
+
+        for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
+          // todo add log
+          pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
+          pSqlFuncExpr1->arg[j].argBytes = htons(pExpr->param[j].nLen);
+
+          if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) {
+            memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen);
+            pMsg += pExpr->param[j].nLen;
+          } else {
+            pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key);
+          }
+        }
+
+        pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+      } else {
+        assert(pField->pArithExprInfo != NULL);
+        SExprInfo* pExprInfo = pField->pArithExprInfo;
+
+        pSqlFuncExpr1->colInfo.colId    = htons(pExprInfo->base.colInfo.colId);
+        pSqlFuncExpr1->functionId  = htons(pExprInfo->base.functionId);
+        pSqlFuncExpr1->numOfParams = htons(pExprInfo->base.numOfParams);
+        pMsg += sizeof(SSqlFuncMsg);
+
+        for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
+          // todo add log
+          pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExprInfo->base.arg[j].argType);
+          pSqlFuncExpr1->arg[j].argBytes = htons(pExprInfo->base.arg[j].argBytes);
+
+          if (pExprInfo->base.arg[j].argType == TSDB_DATA_TYPE_BINARY) {
+            memcpy(pMsg, pExprInfo->base.arg[j].argValue.pz, pExprInfo->base.arg[j].argBytes);
+            pMsg += pExprInfo->base.arg[j].argBytes;
+          } else {
+            pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExprInfo->base.arg[j].argValue.i64);
+          }
+        }
+
+        pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+      }
+    }
+  } else {
+    pQueryMsg->secondStageOutput = 0;
+  }
+
  // serialize the table info (sid, uid, tags)
  pMsg = doSerializeTableInfo(pQueryMsg, pSql, pMsg);
  
@ -797,7 +879,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  }

  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-    for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+    for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) {
      *((int64_t *)pMsg) = htobe64(pQueryInfo->fillVal[i]);
      pMsg += sizeof(pQueryInfo->fillVal[0]);
    }
@ -857,42 +939,25 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

  // compressed ts block
  pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload));
-  int32_t tsLen = 0;
-  int32_t numOfBlocks = 0;

  if (pQueryInfo->tsBuf != NULL) {
-    int32_t vnodeId = htonl(pQueryMsg->head.vgId);
-    STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId);
-    assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL);  // this query should not be sent
-
-    // todo refactor
-    if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
-      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
-      tscError("%p: fseek failed: %s", pSql, tstrerror(code));
+    // note: here used the index instead of actual vnode id.
+    int32_t vnodeIndex = pTableMetaInfo->vgroupIndex;
+    int32_t code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks);
+    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }

-    size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
-    if (s != pBlockInfo->compLen) {
-      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
-      tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
-      return code;
-    }
+    pMsg += pQueryMsg->tsLen;

-    pMsg += pBlockInfo->compLen;
-    tsLen = pBlockInfo->compLen;
-    numOfBlocks = pBlockInfo->numOfBlocks;
-  }
-
-  pQueryMsg->tsLen = htonl(tsLen);
-  pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks);
-  if (pQueryInfo->tsBuf != NULL) {
    pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
+    pQueryMsg->tsLen   = htonl(pQueryMsg->tsLen);
+    pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
  }

  int32_t msgLen = (int32_t)(pMsg - pCmd->payload);

-  tscDebug("%p msg built success,len:%d bytes", pSql, msgLen);
+  tscDebug("%p msg built success, len:%d bytes", pSql, msgLen);
  pCmd->payloadLen = msgLen;
  pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;
  
@ -904,10 +969,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateDbMsg);
+  pCmd->payloadLen = sizeof(SCreateDbMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;

-  SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload;
+  SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;

  assert(pCmd->numOfClause == 1);
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
@ -918,13 +983,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateDnodeMsg);
+  pCmd->payloadLen = sizeof(SCreateDnodeMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
+  SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload;
  strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
  
  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
@ -934,13 +999,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateAcctMsg);
+  pCmd->payloadLen = sizeof(SCreateAcctMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload;
+  SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload;

  SStrToken *pName = &pInfo->pDCLInfo->user.user;
  SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd;
@ -979,14 +1044,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateUserMsg);
+  pCmd->payloadLen = sizeof(SCreateUserMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload;
+  SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload;

  SUserInfo *pUser = &pInfo->pDCLInfo->user;
  strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n);
@ -1011,21 +1076,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCfgDnodeMsg);
+  pCmd->payloadLen = sizeof(SCfgDnodeMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE;
  return TSDB_CODE_SUCCESS;
 }

 int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropDbMsg);
+  pCmd->payloadLen = sizeof(SDropDbMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
+  SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload;

  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
@ -1055,13 +1120,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropDnodeMsg);
+  pCmd->payloadLen = sizeof(SDropDnodeMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
+  SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
@ -1071,7 +1136,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropUserMsg);
+  pCmd->payloadLen = sizeof(SDropUserMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@ -1079,7 +1144,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+  SDropUserMsg *  pDropMsg = (SDropUserMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));

@ -1088,7 +1153,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropUserMsg);
+  pCmd->payloadLen = sizeof(SDropUserMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@ -1096,7 +1161,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+  SDropUserMsg *  pDropMsg = (SDropUserMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));

@ -1105,14 +1170,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMUseDbMsg);
+  pCmd->payloadLen = sizeof(SUseDbMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload;
+  SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  strcpy(pUseDbMsg->db, pTableMetaInfo->name);
  pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB;
@ -1124,14 +1189,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  STscObj *pObj = pSql->pTscObj;
  SSqlCmd *pCmd = &pSql->cmd;
  pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW;
-  pCmd->payloadLen = sizeof(SCMShowMsg) + 100;
+  pCmd->payloadLen = sizeof(SShowMsg) + 100;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload;
+  SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;

  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  size_t nameLen = strlen(pTableMetaInfo->name);
@ -1158,13 +1223,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    pShowMsg->payloadLen = htons(pEpAddr->n);
  }

-  pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen;
+  pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen;
  return TSDB_CODE_SUCCESS;
 }

 int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMKillQueryMsg);
+  pCmd->payloadLen = sizeof(SKillQueryMsg);

  switch (pCmd->command) {
    case TSDB_SQL_KILL_QUERY:
@ -1276,8 +1341,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) {
  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
-         TSDB_EXTRA_PAYLOAD_SIZE;
+  return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE;
 }

 int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@ -1296,7 +1360,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }
  
-  SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
+  SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload;
  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);

  strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
@ -1345,10 +1409,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {

 int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMAlterDbMsg);
+  pCmd->payloadLen = sizeof(SAlterDbMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;

-  SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
+  SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));

@ -1473,14 +1537,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  STscObj *pObj = pSql->pTscObj;
  SSqlCmd *pCmd = &pSql->cmd;
  pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT;
-  pCmd->payloadLen = sizeof(SCMConnectMsg);
+  pCmd->payloadLen = sizeof(SConnectMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
+  SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload;

  // TODO refactor full_name
  char *db;  // ugly code to move the space
@ -1502,11 +1566,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

-  SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
+  STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload;
  strcpy(pInfoMsg->tableId, pTableMetaInfo->name);
  pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0);

-  char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
+  char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);

  size_t len = htonl(pCmd->tagData.dataLen);
  if (pSql->cmd.autoCreated) {
@ -1525,7 +1589,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 /**
 *  multi table meta req pkg format:
- *  | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
+ *  | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
 *      no used         4B
 **/
 int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@ -1543,16 +1607,16 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize);
  memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN);  // server don't need the db

-  SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
+  SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
  pInfoMsg->numOfTables = htonl((int32_t)pCmd->count);

  if (pCmd->payloadLen > 0) {
    memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen);
  }

-  taosTFree(tmpData);
+  tfree(tmpData);

-  pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg);
+  pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;

  assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize);
@ -1597,12 +1661,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  
  char* pMsg = pCmd->payload;
  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  
-  SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg;
+
+  SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg;
  pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables);
-  pMsg += sizeof(SCMSTableVgroupMsg);
-  
-  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+  pMsg += sizeof(SSTableVgroupMsg);
+
+  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
    STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
    size_t size = sizeof(pTableMetaInfo->name);
    tstrncpy(pMsg, pTableMetaInfo->name, size);
@ -1635,14 +1699,17 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    numOfStreams++;
  }

-  int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100;
+  int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100;
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
    pthread_mutex_unlock(&pObj->mutex);
-    tscError("%p failed to malloc for heartbeat msg", pSql);
+    tscError("%p failed to create heartbeat msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
+  // TODO the expired hb and client can not be identified by server till now.
+  SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload;
+  tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer));
+
  pHeartbeat->numOfQueries = numOfQueries;
  pHeartbeat->numOfStreams = numOfStreams;

@ -1716,8 +1783,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {

  pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
      strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
-  
-  // todo handle out of memory case
+
  if (pTableMetaInfo->pTableMeta == NULL) {
    free(pTableMeta);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -1731,7 +1797,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {

 /**
 *  multi table meta rsp pkg format:
- *  | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
+ *  | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
 *  |...... 1B        1B            4B
 **/
 int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
@ -1748,9 +1814,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {

  rsp++;

-  SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp;
+  SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp;
  totalNum = htonl(pInfo->numOfTables);
-  rsp += sizeof(SCMMultiTableInfoMsg);
+  rsp += sizeof(SMultiTableInfoMsg);

  for (i = 0; i < totalNum; i++) {
    SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
@ -1842,10 +1908,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
  SSqlRes* pRes = &pSql->res;
  
  // NOTE: the order of several table must be preserved.
-  SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
+  SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp;
  pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables);
-  char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg);
-  
+  char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg);
+
  // master sqlObj locates in param
  SSqlObj* parent = pSql->param;
  assert(parent != NULL);
@ -1857,18 +1923,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
    SVgroupsMsg *  pVgroupMsg = (SVgroupsMsg *) pMsg;
    pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);

-    size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
+    size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);

-    size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
+    size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
    pInfo->vgroupList = calloc(1, vgroupsz);
    assert(pInfo->vgroupList != NULL);

    pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
    for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
      //just init, no need to lock
-      SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
+      SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];

-      SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
+      SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
      pVgroups->vgId = htonl(vmsg->vgId);
      pVgroups->numOfEps = vmsg->numOfEps;

@ -1890,10 +1956,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
 * current process do not use the cache at all
 */
 int tscProcessShowRsp(SSqlObj *pSql) {
-  STableMetaMsg * pMetaMsg;
-  SCMShowRsp *pShow;
-  SSchema *    pSchema;
-  char         key[20];
+  STableMetaMsg *pMetaMsg;
+  SShowRsp *     pShow;
+  SSchema *      pSchema;
+  char           key[20];

  SSqlRes *pRes = &pSql->res;
  SSqlCmd *pCmd = &pSql->cmd;
@ -1902,7 +1968,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {

  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

-  pShow = (SCMShowRsp *)pRes->pRsp;
+  pShow = (SShowRsp *)pRes->pRsp;
  pShow->qhandle = htobe64(pShow->qhandle);
  pRes->qhandle = pShow->qhandle;

@ -1949,13 +2015,13 @@ int tscProcessShowRsp(SSqlObj *pSql) {
    SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f);
    
    pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index,
-                     pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false);
+                     pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pQueryInfo), pTableSchema[i].bytes, false);
  }
  
  pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
  tscFieldInfoUpdateOffset(pQueryInfo);
  
-  taosTFree(pTableMeta);
+  tfree(pTableMeta);
  return 0;
 }

@ -1980,7 +2046,7 @@ static void createHBObj(STscObj* pObj) {

  pSql->cmd.command = pQueryInfo->command;
  if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) {
-    taosTFree(pSql);
+    tfree(pSql);
    return;
  }

@ -1995,11 +2061,12 @@ static void createHBObj(STscObj* pObj) {
 }

 int tscProcessConnectRsp(SSqlObj *pSql) {
-  char temp[TSDB_TABLE_FNAME_LEN * 2];
  STscObj *pObj = pSql->pTscObj;
  SSqlRes *pRes = &pSql->res;

-  SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
+  char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
+
+  SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp;
  tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId));  // copy acctId from response
  int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);

@ -2017,7 +2084,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
  pObj->connId = htonl(pConnect->connId);

  createHBObj(pObj);
-  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
+
+  //launch a timer to send heartbeat to maintain the connection and send status to mnode
+  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);

  return 0;
 }
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@ -28,7 +28,6 @@
 #include "tutil.h"
 #include "ttimer.h"
 #include "tscProfile.h"
-#include "ttimer.h"

 static bool validImpl(const char* str, size_t maxsize) {
  if (str == NULL) {
@ -161,6 +160,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
  registerSqlObj(pSql);
  tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);

+  pObj->rid = taosAddRef(tscRefId, pObj);
  return pSql;
 }

@ -278,9 +278,9 @@ void taos_close(TAOS *taos) {

  SSqlObj* pHb = pObj->pHb;
  if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
-    if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode
-      rpcCancelRequest(pHb->pRpcCtx);
-      pHb->pRpcCtx = NULL;
+    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
+      rpcCancelRequest(pHb->rpcRid);
+      pHb->rpcRid = -1;
    }

    tscDebug("%p HB is freed", pHb);
@ -296,7 +296,8 @@ void taos_close(TAOS *taos) {
  }

  tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
-  tscCloseTscObj(pObj);
+
+  taosRemoveRef(tscRefId, pObj->rid);
 }

 void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
@ -480,7 +481,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {

  assert(0);
  for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
+    tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0);
  }

  *rows = pRes->tsrow;
@ -564,7 +565,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
    pRes->rspType = 0;

    pSql->subState.numOfSub = 0;
-    taosTFree(pSql->pSubs);
+    tfree(pSql->pSubs);

    assert(pSql->fp == NULL);

@ -746,9 +747,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
    assert(pSubObj->self == (SSqlObj**) p);

    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    if (pSubObj->pRpcCtx != NULL) {
-      rpcCancelRequest(pSubObj->pRpcCtx);
-      pSubObj->pRpcCtx = NULL;
+    if (pSubObj->rpcRid > 0) {
+      rpcCancelRequest(pSubObj->rpcRid);
+      pSubObj->rpcRid = -1;
    }

    tscQueueAsyncRes(pSubObj);
@ -773,7 +774,7 @@ void taos_stop_query(TAOS_RES *res) {
  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);

  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    assert(pSql->pRpcCtx == NULL);
+    assert(pSql->rpcRid <= 0);
    tscKillSTableQuery(pSql);
  } else {
    if (pSql->cmd.command < TSDB_SQL_LOCAL) {
@ -782,9 +783,9 @@ void taos_stop_query(TAOS_RES *res) {
       * reset and freed in the processMsgFromServer function, and causes the invalid
       * write problem for rpcCancelRequest.
       */
-      if (pSql->pRpcCtx != NULL) {
-        rpcCancelRequest(pSql->pRpcCtx);
-        pSql->pRpcCtx = NULL;
+      if (pSql->rpcRid > 0) {
+        rpcCancelRequest(pSql->rpcRid);
+        pSql->rpcRid = -1;
      }

      tscQueueAsyncRes(pSql);
@ -892,7 +893,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
  if (sqlLen > tsMaxSQLStringLen) {
    tscError("%p sql too long", pSql);
    pRes->code = TSDB_CODE_TSC_INVALID_SQL;
-    taosTFree(pSql);
+    tfree(pSql);
    return pRes->code;
  }

@ -901,7 +902,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
    tscError("%p failed to malloc sql string buffer", pSql);
    tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
-    taosTFree(pSql);
+    tfree(pSql);
    return pRes->code;
  }

--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf

    taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
    tscFreeSqlResult(pSql);
-    taosTFree(pSql->pSubs);
+    tfree(pSql->pSubs);
    pSql->subState.numOfSub = 0;
    pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
    tscSetNextLaunchTimer(pStream, pSql);
@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) {
    pStream->pSql = NULL;

    taos_free_result(pSql);
-    taosTFree(pStream);
+    tfree(pStream);
  }
 }
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@ -32,11 +32,26 @@ typedef struct SInsertSupporter {
 static void freeJoinSubqueryObj(SSqlObj* pSql);
 static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);

-static bool tsCompare(int32_t order, int64_t left, int64_t right) {
+static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
+  if (left == right) {
+    return 0;
+  }
+
  if (order == TSDB_ORDER_ASC) {
-    return left < right;
+    return left < right? -1:1;
  } else {
-    return left > right;
+    return left > right? -1:1;
+  }
+}
+
+static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) {
+  while (tsBufNextPos(pTSBuf)) {
+    STSElem el1 = tsBufGetElem(pTSBuf);
+
+    int32_t res = tVariantCompare(el1.tag, tag1);
+    if (res != 0) { // it is a record with new tag
+      return;
+    }
  }
 }

@ -51,10 +66,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ

  SLimitVal* pLimit = &pQueryInfo->limit;
  int32_t    order = pQueryInfo->order.order;
-  
+
  SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0);
  SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0);
-  
+
  pSubQueryInfo1->tsBuf = output1;
  pSubQueryInfo2->tsBuf = output2;

@ -88,59 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
  int64_t numOfInput1 = 1;
  int64_t numOfInput2 = 1;

-  while (1) {
-    STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
-    STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+  while(1) {
+    STSElem elem = tsBufGetElem(pSupporter1->pTSBuf);

-#ifdef _DEBUG_VIEW
-    tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
-#endif
+    // no data in pSupporter1 anymore, jump out of loop
+    if (!tsBufIsValidElem(&elem)) {
+      break;
+    }

-    int32_t res = tVariantCompare(elem1.tag, elem2.tag);
-    if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) {
-      if (!tsBufNextPos(pSupporter1->pTSBuf)) {
-        break;
-      }
+    // find the data in supporter2 with the same tag value
+    STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag);

-      numOfInput1++;
-    } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) {
-      if (!tsBufNextPos(pSupporter2->pTSBuf)) {
-        break;
-      }
+    /**
+     * there are elements in pSupporter2 with the same tag, continue
+     */
+    tVariant tag1 = {0};
+    tVariantAssign(&tag1, elem.tag);

-      numOfInput2++;
-    } else {
-      /*
-       * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
-       * final results which is acquired after the secondry merge of in the client.
-       */
-      if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
-        if (win->skey > elem1.ts) {
-          win->skey = elem1.ts;
+    if (tsBufIsValidElem(&e2)) {
+      while (1) {
+        STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
+        STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+
+        // data with current are exhausted
+        if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) {
+          break;
        }
-  
-        if (win->ekey < elem1.ts) {
-          win->ekey = elem1.ts;
+
+        if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag
+          skipRemainValue(pSupporter1->pTSBuf, &tag1);
+          break;
        }
-        
-        tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
-        tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));

-      } else {
-        pLimit->offset -= 1;
+        /*
+         * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
+         * final results which is acquired after the secondary merge of in the client.
+         */
+        int32_t re = tsCompare(order, elem1.ts, elem2.ts);
+        if (re < 0) {
+          tsBufNextPos(pSupporter1->pTSBuf);
+          numOfInput1++;
+        } else if (re > 0) {
+          tsBufNextPos(pSupporter2->pTSBuf);
+          numOfInput2++;
+        } else {
+          if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
+            if (win->skey > elem1.ts) {
+              win->skey = elem1.ts;
+            }
+
+            if (win->ekey < elem1.ts) {
+              win->ekey = elem1.ts;
+            }
+
+            tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
+            tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
+          } else {
+            pLimit->offset -= 1;//offset apply to projection?
+          }
+
+          tsBufNextPos(pSupporter1->pTSBuf);
+          numOfInput1++;
+
+          tsBufNextPos(pSupporter2->pTSBuf);
+          numOfInput2++;
+        }
      }
-
-      if (!tsBufNextPos(pSupporter1->pTSBuf)) {
-        break;
-      }
-
-      numOfInput1++;
-
-      if (!tsBufNextPos(pSupporter2->pTSBuf)) {
-        break;
-      }
-
-      numOfInput2++;
+    } else {  // no data in pSupporter2, ignore current data in pSupporter2
+      skipRemainValue(pSupporter1->pTSBuf, &tag1);
    }
  }

@ -162,8 +192,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ

  TSKEY et = taosGetTimestampUs();
  tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks "
-           "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal,
-           output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st);
+           "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us",
+           pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey,
+           tsBufGetNumOfGroup(output1), et - st);

  return output1->numOfTotal;
 }
@ -224,7 +255,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
    pSupporter->pVgroupTables = NULL;
  }

-  taosTFree(pSupporter->pIdTagList);
+  tfree(pSupporter->pIdTagList);
  tscTagCondRelease(&pSupporter->tagCond);
  free(pSupporter);
 }
@ -248,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
  return false;
 }

+static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+  int32_t  num = 0;
+  int32_t* list = NULL;
+  tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+  // The virtual node, of which all tables are disqualified after the timestamp intersection,
+  // is removed to avoid next stage query.
+  // TODO: If tables from some vnodes are not qualified for next stage query, discard them.
+  for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) {
+    SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k);
+
+    bool found = false;
+    for (int32_t f = 0; f < num; ++f) {
+      if (p->vgInfo.vgId == list[f]) {
+        found = true;
+        break;
+      }
+    }
+
+    if (!found) {
+      tscRemoveVgroupTableGroup(pVgroupTables, k);
+    } else {
+      k++;
+    }
+  }
+
+  assert(taosArrayGetSize(pVgroupTables) > 0);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+  tfree(list);
+}
+
+static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+  int32_t  num = 0;
+  int32_t* list = NULL;
+  tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+  size_t numOfGroups = taosArrayGetSize(pVgroupTables);
+
+  SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo));
+
+  SVgroupTableInfo info;
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t vnodeId = list[i];
+
+    for (int32_t j = 0; j < numOfGroups; ++j) {
+      SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j);
+      if (p1->vgInfo.vgId == vnodeId) {
+        tscVgroupTableCopy(&info, p1);
+        break;
+      }
+    }
+
+    taosArrayPush(pNew, &info);
+  }
+
+  tfree(list);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+  return pNew;
+}
+
 /*
 * launch secondary stage query to fetch the result that contains timestamp in set
 */
@ -320,26 +413,27 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
    pQueryInfo->colList = pSupporter->colList;
    pQueryInfo->exprList = pSupporter->exprList;
    pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
+    pQueryInfo->groupbyExpr = pSupporter->groupInfo;
+
+    assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
+  
+    tscFieldInfoUpdateOffset(pQueryInfo);
+  
+    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+    pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;

    pSupporter->exprList = NULL;
    pSupporter->colList = NULL;
-    memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
-  
-    SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
-    assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
-  
-    tscFieldInfoUpdateOffset(pNewQueryInfo);
-  
-    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
-    pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;
    pSupporter->pVgroupTables = NULL;
+    memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
+    memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr));

    /*
     * When handling the projection query, the offset value will be modified for table-table join, which is changed
     * during the timestamp intersection.
     */
    pSupporter->limit = pQueryInfo->limit;
-    pNewQueryInfo->limit = pSupporter->limit;
+    pQueryInfo->limit = pSupporter->limit;

    SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
    SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0);
@ -354,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {

      tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
      tscPrintSelectClause(pNew, 0);
-      tscFieldInfoUpdateOffset(pNewQueryInfo);
+      tscFieldInfoUpdateOffset(pQueryInfo);

      pExpr = tscSqlExprGet(pQueryInfo, 0);
    }
@ -369,39 +463,21 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
      pExpr->numOfParams = 1;
    }

-    int32_t num = 0;
-    int32_t *list = NULL;
-    tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list);
-
-    if (pTableMetaInfo->pVgroupTables != NULL) {
-      for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) {
-        SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k);
-
-        bool found = false;
-        for(int32_t f = 0; f < num; ++f) {
-          if (p->vgInfo.vgId == list[f]) {
-            found = true;
-            break;
-          }
-        }
-
-        if (!found) {
-          tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k);
-        } else {
-          k++;
-        }
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+      assert(pTableMetaInfo->pVgroupTables != NULL);
+      if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+        SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables);
+        tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
+        pTableMetaInfo->pVgroupTables = p;
+      } else {
+        filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables);
      }
-
-      assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0);
-      TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
    }

-    taosTFree(list);
-
-    size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
+    size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
    tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s",
-             pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList),
-             numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
+             pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList),
+             numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
  }
  
  //prepare the subqueries object failed, abort
@ -447,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
  assert(pSqlObj->subState.numOfRemain > 0);

  if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
-    tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
+    tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
    freeJoinSubqueryObj(pSqlObj);
  }
 }
@ -460,18 +536,36 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {

 }

-int32_t tscCompareTidTags(const void* p1, const void* p2) {
-  const STidTags* t1 = (const STidTags*) varDataVal(p1);
-  const STidTags* t2 = (const STidTags*) varDataVal(p2);
+int32_t tidTagsCompar(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) (p1);
+  const STidTags* t2 = (const STidTags*) (p2);
  
  if (t1->vgId != t2->vgId) {
    return (t1->vgId > t2->vgId) ? 1 : -1;
  }

-  if (t1->tid != t2->tid) {
-    return (t1->tid > t2->tid) ? 1 : -1;
+  tstr* tag1 = (tstr*) t1->tag;
+  tstr* tag2 = (tstr*) t2->tag;
+
+  if (tag1->len != tag2->len) {
+    return (tag1->len > tag2->len)? 1: -1;
  }
-  return 0;
+
+  return strncmp(tag1->data, tag2->data, tag1->len);
+}
+
+int32_t tagValCompar(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) varDataVal(p1);
+  const STidTags* t2 = (const STidTags*) varDataVal(p2);
+
+  tstr* tag1 = (tstr*) t1->tag;
+  tstr* tag2 = (tstr*) t2->tag;
+
+  if (tag1->len != tag2->len) {
+    return (tag1->len > tag2->len)? 1: -1;
+  }
+
+  return memcmp(tag1->data, tag2->data, tag1->len);
 }

 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@ -489,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
      SVgroupTableInfo info = {{0}};
      for (int32_t m = 0; m < pvg->numOfVgroups; ++m) {
        if (tt->vgId == pvg->vgroups[m].vgId) {
-          tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
+          tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
          break;
        }
      }
@ -497,17 +591,29 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr

      vgTables = taosArrayInit(4, sizeof(STableIdInfo));
      info.itemList = vgTables;
+
+      if (taosArrayGetSize(result) > 0) {
+        SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1);
+        tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList));
+      }
+
      taosArrayPush(result, &info);
    }

-    tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId)
    STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN};
    taosArrayPush(vgTables, &item);
+
+    tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId);
    prev = tt;
  }

  pTableMetaInfo->pVgroupTables = result;
  pTableMetaInfo->vgroupIndex = 0;
+
+  if (taosArrayGetSize(result) > 0) {
+    SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1);
+    tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList));
+  }
 }

 static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@ -582,22 +688,24 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq
 }

 static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
-  tscDebug("%p all subqueries retrieve <tid, tags> complete, do tags match", pParentSql);
-
  SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
  SJoinSupporter* p2 = pParentSql->pSubs[1]->param;

-  qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
-  qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
+  tscDebug("%p all subquery retrieve <tid, tags> complete, do tags match, %d, %d", pParentSql, p1->num, p2->num);
+
+  // sort according to the tag value
+  qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar);
+  qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar);

  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);

-  SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+  SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);

  // int16_t for padding
-  *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t));
-  *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t));
+  int32_t size = p1->tagSize - sizeof(int16_t);
+  *s1 = taosArrayInit(p1->num, size);
+  *s2 = taosArrayInit(p2->num, size);

  if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) {
    return TSDB_CODE_QRY_DUP_JOIN_KEY;
@ -625,6 +733,27 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
    }
  }

+  // reorganize the tid-tag value according to both the vgroup id and tag values
+  // sort according to the tag value
+  size_t t1 = taosArrayGetSize(*s1);
+  size_t t2 = taosArrayGetSize(*s2);
+
+  qsort((*s1)->pData, t1, size, tidTagsCompar);
+  qsort((*s2)->pData, t2, size, tidTagsCompar);
+
+#if 0
+  for(int32_t k = 0; k < t1; ++k) {
+    STidTags* p =  (*s1)->pData + size * k;
+    printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+  }
+
+  for(int32_t k = 0; k < t1; ++k) {
+    STidTags* p =  (*s2)->pData + size * k;
+    printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+  }
+#endif
+
+  tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2);
  return TSDB_CODE_SUCCESS;
 }

@ -724,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
  }

  if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) {  // no results,return.
+    assert(pParentSql->fp != tscJoinQueryCallback);
+
    tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
    freeJoinSubqueryObj(pParentSql);

    // set no result command
    pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+    assert(pParentSql->fp != tscJoinQueryCallback);
+
    (*pParentSql->fp)(pParentSql->param, pParentSql, 0);
  } else {
    // proceed to for ts_comp query
@ -744,10 +877,10 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
    tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);

    SSqlObj* psub1 = pParentSql->pSubs[0];
-    ((SJoinSupporter*)psub1->param)->pVgroupTables =  tscCloneVgroupTableInfo(pTableMetaInfo1->pVgroupTables);
+    ((SJoinSupporter*)psub1->param)->pVgroupTables =  tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables);

    SSqlObj* psub2 = pParentSql->pSubs[1];
-    ((SJoinSupporter*)psub2->param)->pVgroupTables =  tscCloneVgroupTableInfo(pTableMetaInfo2->pVgroupTables);
+    ((SJoinSupporter*)psub2->param)->pVgroupTables =  tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables);

    pParentSql->subState.numOfSub = 2;
    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;
@ -928,6 +1061,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
    }
  }

+  assert(pState->numOfRemain > 0);
  if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
    tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub);
    return;
@ -941,6 +1075,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
  }

  // update the records for each subquery in parent sql object.
+  bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0);
  for (int32_t i = 0; i < pState->numOfSub; ++i) {
    if (pParentSql->pSubs[i] == NULL) {
      tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i);
@ -954,7 +1089,10 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
               pRes1->numOfRows, pRes1->numOfTotal);
      assert(pRes1->row < pRes1->numOfRows);
    } else {
-      pRes1->numOfClauseTotal += pRes1->numOfRows;
+      if (!stableQuery) {
+        pRes1->numOfClauseTotal += pRes1->numOfRows;
+      }
+
      tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i,
               pRes1->numOfRows, pRes1->numOfTotal);
    }
@ -964,7 +1102,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
  tscBuildResFromSubqueries(pParentSql);
 }

-void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
+void tscFetchDatablockForSubquery(SSqlObj* pSql) {
  assert(pSql->subState.numOfSub >= 1);
  
  int32_t numOfFetch = 0;
@ -1026,11 +1164,22 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
  if (numOfFetch <= 0) {
    bool tryNextVnode = false;

-    SSqlObj*    pp = pSql->pSubs[0];
-    SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0);
+    bool orderedPrjQuery = false;
+    for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+      SSqlObj* pSub = pSql->pSubs[i];
+      if (pSub == NULL) {
+        continue;
+      }
+
+      SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0);
+      orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0);
+      if (orderedPrjQuery) {
+        break;
+      }
+    }

    // get the number of subquery that need to retrieve the next vnode.
-    if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) {
+    if (orderedPrjQuery) {
      for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
        SSqlObj* pSub = pSql->pSubs[i];
        if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) {
@ -1134,7 +1283,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
  SSqlCmd* pCmd = &pSql->cmd;
  SSqlRes* pRes = &pSql->res;

-  tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex);

  // the column transfer support struct has been built
  if (pRes->pColumnIndex != NULL) {
@ -1230,21 +1378,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
    return;
  }

-  // wait for the other subqueries response from vnode
-  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
-    return;
+
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+  // In case of consequence query from other vnode, do not wait for other query response here.
+  if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) {
+    if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
+      return;
+    }
  }

  tscSetupOutputColumnIndex(pParentSql);
-  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

  /**
   * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
   * data instead of returning to its invoker
   */
  if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-//    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;  // reset the record value
-
    pSql->fp = joinRetrieveFinalResCallback;  // continue retrieve data
    pSql->cmd.command = TSDB_SQL_FETCH;
    tscProcessSql(pSql);
@ -1313,6 +1463,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
      return TSDB_CODE_TSC_OUT_OF_MEMORY;
    }

+    pSupporter->groupInfo = pNewQueryInfo->groupbyExpr;
+    memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
+
    pNew->cmd.numOfCols = 0;
    pNewQueryInfo->interval.interval = 0;
    pSupporter->limit = pNewQueryInfo->limit;
@ -1333,17 +1486,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
      assert(pTagCond->joinInfo.hasJoin);

      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
-      SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+      SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);

-      // get the tag colId column index
-      int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta);
-      SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
-      for(int32_t i = 0; i < numOfTags; ++i) {
-        if (pSchema[i].colId == tagColId) {
-          colIndex.columnIndex = i;
-          break;
-        }
-      }
+      colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId);

      int16_t bytes = 0;
      int16_t type  = 0;
@ -1424,6 +1569,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {

  tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+
    SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i);
    
    if (pSupporter == NULL) {  // failed to create support struct, abort current query
@ -1478,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
    
    SRetrieveSupport* pSupport = pSub->param;
    
-    taosTFree(pSupport->localBuffer);
-    taosTFree(pSupport);
+    tfree(pSupport->localBuffer);
+    tfree(pSupport);
    
    taos_free_result(pSub);
  }
@ -1496,9 +1642,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
  }
  
  tExtMemBuffer **  pMemoryBuf = NULL;
-  tOrderDescriptor *pDesc = NULL;
-  SColumnModel *    pModel = NULL;
-  
+  tOrderDescriptor *pDesc  = NULL;
+  SColumnModel     *pModel = NULL;
+
  pRes->qhandle = 0x1;  // hack the qhandle check
  
  const uint32_t nBufferSize = (1u << 16);  // 64KB
@ -1520,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
  if (ret != 0) {
    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
    tscQueueAsyncRes(pSql);
-    taosTFree(pMemoryBuf);
+    tfree(pMemoryBuf);
    return ret;
  }
  
@ -1529,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
  tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);

  if (pSql->pSubs == NULL) {
-    taosTFree(pSql->pSubs);
+    tfree(pSql->pSubs);
    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);

@ -1554,19 +1700,19 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
    trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage));
    if (trs->localBuffer == NULL) {
      tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
-      taosTFree(trs);
+      tfree(trs);
      break;
    }
    
    trs->subqueryIndex  = i;
    trs->pParentSql     = pSql;
    trs->pFinalColModel = pModel;
-    
+
    SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL);
    if (pNew == NULL) {
      tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
-      taosTFree(trs->localBuffer);
-      taosTFree(trs);
+      tfree(trs->localBuffer);
+      tfree(trs);
      break;
    }
    
@ -1616,12 +1762,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) {
  }

  tscDebug("%p start to free subquery supp obj:%p", pSql, trsupport);
-//  int32_t  index = trsupport->subqueryIndex;
-//  SSqlObj *pParentSql = trsupport->pParentSql;
-
-//  assert(pSql == pParentSql->pSubs[index]);
-  taosTFree(trsupport->localBuffer);
-  taosTFree(trsupport);
+  tfree(trsupport->localBuffer);
+  tfree(trsupport);
 }

 static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
@ -1645,8 +1787,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
  SSqlObj *pParentSql = trsupport->pParentSql;
  int32_t  subqueryIndex = trsupport->subqueryIndex;

-  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+  STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
+  SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];

  tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);

@ -1810,7 +1952,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
  SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
  tscClearInterpInfo(pPQueryInfo);
  
-  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, pParentSql);
+  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql);
  tscDebug("%p build loser tree completed", pParentSql);
  
  pParentSql->res.precision = pSql->res.precision;
@ -1848,7 +1990,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
  
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo  *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+  SVgroupInfo  *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];

  if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
    trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
@ -1959,7 +2101,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
  assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
  
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
+  SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];

  // stable query killed or other subquery failed, all query stopped
  if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
@ -2021,7 +2163,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
    pParentObj->res.code = pSql->res.code;
  }

-  taosTFree(pSupporter);
+  tfree(pSupporter);

  if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
    return;
@ -2165,7 +2307,8 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
    numOfRes = (int32_t)(MIN(numOfRes, remain));
  }

-  if (numOfRes == 0) {
+  if (numOfRes == 0) {  // no result any more, free all subquery objects
+    freeJoinSubqueryObj(pSql);
    return;
  }

@ -2223,7 +2366,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);

    size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
-    pRes->numOfCols =  (int32_t)numOfExprs;
+    pRes->numOfCols =  (int16_t)numOfExprs;

    pRes->tsrow  = calloc(numOfExprs, POINTER_BYTES);
    pRes->buffer = calloc(numOfExprs, POINTER_BYTES);
@ -2271,7 +2414,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
  }
 }

-static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
+char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
  SArithmeticSupport *pSupport = (SArithmeticSupport *) param;

  int32_t index = -1;
@ -2295,55 +2438,29 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) {

  assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
  if (pRes->row >= pRes->numOfRows) {  // all the results has returned to invoker
-    taosTFree(pRes->tsrow);
+    tfree(pRes->tsrow);
    return pRes->tsrow;
  }

  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);

  size_t size = tscNumOfFields(pQueryInfo);
+  int32_t offset = 0;
+
  for (int i = 0; i < size; ++i) {
-    SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
-    if (pSup->pSqlExpr != NULL) {
-      tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
-    }
+    tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, offset);
+    TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
+
+    offset += pField->bytes;

    // primary key column cannot be null in interval query, no need to check
    if (i == 0 && pQueryInfo->interval.interval > 0) {
      continue;
    }

-    TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
    if (pRes->tsrow[i] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
      transferNcharData(pSql, i, pField);
    }
-
-    // calculate the result from several other columns
-    if (pSup->pArithExprInfo != NULL) {
-      if (pRes->pArithSup == NULL) {
-        pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport));
-      }
-
-      pRes->pArithSup->offset     = 0;
-      pRes->pArithSup->pArithExpr = pSup->pArithExprInfo;
-      pRes->pArithSup->numOfCols  = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
-      pRes->pArithSup->exprList   = pQueryInfo->exprList;
-      pRes->pArithSup->data       = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES);
-
-      if (pRes->buffer[i] == NULL) {
-        TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
-        pRes->buffer[i] = malloc(field->bytes);
-      }
-
-      for(int32_t k = 0; k < pRes->pArithSup->numOfCols; ++k) {
-        SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
-        pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
-      }
-
-      tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup,
-          TSDB_ORDER_ASC, getArithemicInputSrc);
-      pRes->tsrow[i] = (unsigned char*)pRes->buffer[i];
-    }
  }

  pRes->row++;  // index increase one-step
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@ -35,6 +35,7 @@ void *  tscTmr;
 void *  tscQhandle;
 void *  tscCheckDiskUsageTmr;
 int     tsInsertHeadSize;
+int     tscRefId = -1;

 int tscNumOfThreads;

@ -78,7 +79,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon


 void taos_init_imp(void) {
-  char temp[128];
+  char temp[128]  = {0};
  
  errno = TSDB_CODE_SUCCESS;
  srand(taosGetTimestampSec());
@ -102,8 +103,8 @@ void taos_init_imp(void) {

    taosReadGlobalCfg();
    taosCheckGlobalCfg();
-    taosPrintGlobalCfg();

+    rpcInit();
    tscDebug("starting to initialize TAOS client ...");
    tscDebug("Local End Point is:%s", tsLocalEp);
  }
@ -145,29 +146,45 @@ void taos_init_imp(void) {
    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
  }

+  tscRefId = taosOpenRef(200, tscCloseTscObj);
+
+  // in other language APIs, taos_cleanup is not available yet.
+  // So, to make sure taos_cleanup will be invoked to clean up the allocated
+  // resource to suppress the valgrind warning.
+  atexit(taos_cleanup);
  tscDebug("client is initialized successfully");
 }

 void taos_init() { pthread_once(&tscinit, taos_init_imp); }

-void taos_cleanup() {
-  if (tscMetaCache != NULL) {
-    taosCacheCleanup(tscMetaCache);
-    tscMetaCache = NULL;
+// this function may be called by user or system, or by both simultaneously.
+void taos_cleanup(void) {
+  tscDebug("start to cleanup client environment");

-    taosCacheCleanup(tscObjCache);
-    tscObjCache = NULL;
-  }
-  
-  if (tscQhandle != NULL) {
-    taosCleanUpScheduler(tscQhandle);
-    tscQhandle = NULL;
+  void* m = tscMetaCache;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) {
+    taosCacheCleanup(m);
  }

+  m = tscObjCache;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+    taosCacheCleanup(m);
+  }
+
+  m = tscQhandle;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) {
+    taosCleanUpScheduler(m);
+  }
+
+  taosCloseRef(tscRefId);
  taosCleanupKeywordsTable();
  taosCloseLog();
-  
-  taosTmrCleanUp(tscTmr);
+  if (tscEmbedded == 0) rpcCleanup();
+
+  m = tscTmr;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {
+    taosTmrCleanUp(m);
+  }
 }

 static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@ -71,7 +71,8 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
 }

 bool tscQueryTags(SQueryInfo* pQueryInfo) {
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+  int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+  for (int32_t i = 0; i < numOfCols; ++i) {
    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
    int32_t functId = pExpr->functionId;

@ -201,13 +202,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {

 bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-  
  for (int32_t i = 0; i < size; ++i) {
    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
    assert(pExpr != NULL);
-//    if (pExpr == NULL) {
-//      return false;
-//    }

    int32_t functionId = pExpr->functionId;
    if (functionId == TSDB_FUNC_TAG) {
@ -222,6 +219,24 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
  return true;
 }

+bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo) {
+  size_t numOfOutput = tscNumOfFields(pQueryInfo);
+  size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+  if (numOfOutput == numOfExprs) {
+    return false;
+  }
+
+  for(int32_t i = 0; i < numOfOutput; ++i) {
+    SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pArithExprInfo;
+    if (pExprInfo != NULL) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 bool tscIsTWAQuery(SQueryInfo* pQueryInfo) {
  size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
  for (int32_t i = 0; i < numOfExprs; ++i) {
@ -245,7 +260,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
  }

  pQueryInfo->fillType = TSDB_FILL_NONE;
-  taosTFree(pQueryInfo->fillVal);
+  tfree(pQueryInfo->fillVal);
 }

 int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@ -259,7 +274,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {

    // not enough memory
    if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
-      taosTFree(pRes->tsrow);
+      tfree(pRes->tsrow);
      pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
      return pRes->code;
    }
@ -271,24 +286,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
 static void tscDestroyResPointerInfo(SSqlRes* pRes) {
  if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
    for (int i = 0; i < pRes->numOfCols; i++) {
-      taosTFree(pRes->buffer[i]);
+      tfree(pRes->buffer[i]);
    }
    
    pRes->numOfCols = 0;
  }
  
-  taosTFree(pRes->pRsp);
+  tfree(pRes->pRsp);

-  taosTFree(pRes->tsrow);
-  taosTFree(pRes->length);
-  taosTFree(pRes->buffer);
+  tfree(pRes->tsrow);
+  tfree(pRes->length);
+  tfree(pRes->buffer);

-  taosTFree(pRes->pGroupRec);
-  taosTFree(pRes->pColumnIndex);
+  tfree(pRes->pGroupRec);
+  tfree(pRes->pColumnIndex);

  if (pRes->pArithSup != NULL) {
-    taosTFree(pRes->pArithSup->data);
-    taosTFree(pRes->pArithSup);
+    tfree(pRes->pArithSup->data);
+    tfree(pRes->pArithSup);
  }
  
  pRes->data = NULL;  // pRes->data points to the buffer of pRsp, no need to free
@ -305,11 +320,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
    
    freeQueryInfoImpl(pQueryInfo);
    clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache);
-    taosTFree(pQueryInfo);
+    tfree(pQueryInfo);
  }
  
  pCmd->numOfClause = 0;
-  taosTFree(pCmd->pQueryInfo);
+  tfree(pCmd->pQueryInfo);
 }

 void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
@ -338,34 +353,6 @@ void tscFreeSqlResult(SSqlObj* pSql) {
  memset(&pSql->res, 0, sizeof(SSqlRes));
 }

-void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
-  if (pSql == NULL || pSql->signature != pSql) {
-    return;
-  }
-
-  SSqlCmd* pCmd = &pSql->cmd;
-  int32_t cmd = pCmd->command;
-  if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscRemoveFromSqlList(pSql);
-  }
-  
-  // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-//  if (pObj->signature == pObj) {
-    //pthread_mutex_lock(&pObj->mutex);
-    taosTFree(pSql->sqlstr);
-    //pthread_mutex_unlock(&pObj->mutex);
-//  }
-  
-  tscFreeSqlResult(pSql);
-  
-  taosTFree(pSql->pSubs);
-  pSql->subState.numOfSub = 0;
-  pSql->self = 0;
-  
-  tscResetSqlCmdObj(pCmd, false);
-}
-
 static void tscFreeSubobj(SSqlObj* pSql) {
  if (pSql->subState.numOfSub == 0) {
    return;
@ -404,7 +391,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
  if (ref == 0) {
    tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
-    tscCloseTscObj(pTscObj);
+    taosRemoveRef(tscRefId, pTscObj->rid);
  }
 }

@ -415,14 +402,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) {
  assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);

  for(int32_t i = 0; i < numOfEps; ++i) {
-    taosTFree(p->vgroupInfo.epAddr[i].fqdn);
+    tfree(p->vgroupInfo.epAddr[i].fqdn);
  }

  int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
  assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);

  for(int32_t i = 0; i < numOfEps1; ++i) {
-    taosTFree(p->corVgroupInfo.epAddr[i].fqdn);
+    tfree(p->corVgroupInfo.epAddr[i].fqdn);
  }
 }

@ -434,22 +421,32 @@ void tscFreeSqlObj(SSqlObj* pSql) {
  tscDebug("%p start to free sqlObj", pSql);

  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
  tscFreeSubobj(pSql);

-  tscPartiallyFreeSqlObj(pSql);
+  SSqlCmd* pCmd = &pSql->cmd;
+  int32_t cmd = pCmd->command;
+  if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
+    tscRemoveFromSqlList(pSql);
+  }

  pSql->signature = NULL;
  pSql->fp = NULL;
-  
-  SSqlCmd* pCmd = &pSql->cmd;
+  tfree(pSql->sqlstr);
+
+  tfree(pSql->pSubs);
+  pSql->subState.numOfSub = 0;
+  pSql->self = 0;
+
+  tscFreeSqlResult(pSql);
+  tscResetSqlCmdObj(pCmd, false);

  memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
-  taosTFree(pCmd->payload);
+  tfree(pCmd->payload);
  pCmd->allocSize = 0;
  
-  taosTFree(pSql->sqlstr);
  tsem_destroy(&pSql->rspSem);
-
  free(pSql);
 }

@ -458,15 +455,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
    return;
  }

-  taosTFree(pDataBlock->pData);
-  taosTFree(pDataBlock->params);
+  tfree(pDataBlock->pData);
+  tfree(pDataBlock->params);

  // free the refcount for metermeta
  if (pDataBlock->pTableMeta != NULL) {
    taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
  }

-  taosTFree(pDataBlock);
+  tfree(pDataBlock);
 }

 SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
@ -741,7 +738,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {

        taosHashCleanup(pVnodeDataBlockHashList);
        tscDestroyBlockArrayList(pVnodeDataBlockList);
-        taosTFree(dataBuf->pData);
+        tfree(dataBuf->pData);

        return TSDB_CODE_TSC_OUT_OF_MEMORY;
      }
@ -786,8 +783,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
 }

 // TODO: all subqueries should be freed correctly before close this connection.
-void tscCloseTscObj(STscObj* pObj) {
-  assert(pObj != NULL);
+void tscCloseTscObj(void *param) {
+  STscObj *pObj = param;

  pObj->signature = NULL;
  taosTmrStopA(&(pObj->pTimer));
@ -801,7 +798,7 @@ void tscCloseTscObj(STscObj* pObj) {
  pthread_mutex_destroy(&pObj->mutex);

  tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
-  taosTFree(pObj);
+  tfree(pObj);
 }

 bool tscIsInsertData(char* sqlstr) {
@ -876,28 +873,11 @@ void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) {
  
  SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
  pExpr->offset = 0;
-  
-  for (int32_t i = 1; i < numOfExprs; ++i) {
-    SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1);
-    SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i);
-  
-    p->offset = prev->offset + prev->resBytes;
-  }
-}

-void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) {
-  if (tscSqlExprNumOfExprs(pQueryInfo) == 0) {
-    return;
-  }
-  
-  SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  pExpr->offset = 0;
-  
-  size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
  for (int32_t i = 1; i < numOfExprs; ++i) {
    SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1);
    SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i);
-    
+
    p->offset = prev->offset + prev->resBytes;
  }
 }
@ -965,18 +945,26 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
    
    if (pInfo->pArithExprInfo != NULL) {
      tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
-      taosTFree(pInfo->pArithExprInfo);
+
+      SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base;
+      for(int32_t j = 0; j < pFuncMsg->numOfParams; ++j) {
+        if (pFuncMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) {
+          tfree(pFuncMsg->arg[j].argValue.pz);
+        }
+      }
+
+      tfree(pInfo->pArithExprInfo);
    }
  }
  
  taosArrayDestroy(pFieldInfo->internalField);
-  taosTFree(pFieldInfo->final);
+  tfree(pFieldInfo->final);

  memset(pFieldInfo, 0, sizeof(SFieldInfo));
 }

 static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-    int16_t size, int16_t interSize, int32_t colType) {
+    int16_t size, int16_t resColId, int16_t interSize, int32_t colType) {
  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex);
  
  SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr));
@ -1009,8 +997,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol

  pExpr->resType       = type;
  pExpr->resBytes      = size;
+  pExpr->resColId      = resColId;
  pExpr->interBytes    = interSize;
-  
+
  if (pTableMetaInfo->pTableMeta) {
    pExpr->uid = pTableMetaInfo->pTableMeta->id.uid;
  }
@ -1019,20 +1008,20 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
 }

 SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                           int16_t size, int16_t interSize, bool isTagCol) {
+                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
  int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList);
  if (index == num) {
-    return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+    return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
  }
  
-  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
  taosArrayInsert(pQueryInfo->exprList, index, &pExpr);
  return pExpr;
 }

 SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-    int16_t size, int16_t interSize, bool isTagCol) {
-  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+    int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
+  SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
  taosArrayPush(pQueryInfo->exprList, &pExpr);
  return pExpr;
 }
@ -1060,16 +1049,14 @@ size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
  return taosArrayGetSize(pQueryInfo->exprList);
 }

-void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) {
-  if (pExpr == NULL || argument == NULL || bytes == 0) {
-    return;
-  }
+void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) {
+  assert (pExpr != NULL || argument != NULL || bytes != 0);

  // set parameter value
  // transfer to tVariant from byte data/no ascii data
  tVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type);
-
  pExpr->numOfParams += 1;
+
  assert(pExpr->numOfParams <= 3);
 }

@ -1086,7 +1073,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) {
    tVariantDestroy(&pExpr->param[i]);
  }
  
-  taosTFree(pExpr);
+  tfree(pExpr);
  
  return NULL;
 }
@ -1186,11 +1173,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
 static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) {
  for(int32_t i = 0; i < numOfFilters; ++i) {
    if (pFilterInfo[i].filterstr) {
-      taosTFree(pFilterInfo[i].pz);
+      tfree(pFilterInfo[i].pz);
    }
  }
  
-  taosTFree(pFilterInfo);
+  tfree(pFilterInfo);
 }

 SColumn* tscColumnClone(const SColumn* src) {
@ -1256,8 +1243,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
 *
 */
 static int32_t validateQuoteToken(SStrToken* pToken) {
-  strdequote(pToken->z);
-  pToken->n = (uint32_t)strtrim(pToken->z);
+  tscDequoteAndTrimToken(pToken);

  int32_t k = tSQLGetToken(pToken->z, &pToken->type);

@ -1272,8 +1258,6 @@ static int32_t validateQuoteToken(SStrToken* pToken) {
 }

 void tscDequoteAndTrimToken(SStrToken* pToken) {
-  assert(pToken->type == TK_STRING);
-
  uint32_t first = 0, last = pToken->n;

  // trim leading spaces
@ -1385,7 +1369,8 @@ int32_t tscValidateName(SStrToken* pToken) {
    } else {
      pStr[firstPartLen] = TS_PATH_DELIMITER[0];
      memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n);
-      pStr[firstPartLen + sizeof(TS_PATH_DELIMITER[0]) + pToken->n] = 0;
+      uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1));
+      memset(pToken->z + pToken->n - offset, ' ', offset);
    }
    pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0]));
    pToken->z = pStr;
@ -1478,7 +1463,7 @@ void tscTagCondRelease(STagCond* pTagCond) {
    size_t s = taosArrayGetSize(pTagCond->pCond);
    for (int32_t i = 0; i < s; ++i) {
      SCond* p = taosArrayGet(pTagCond->pCond, i);
-      taosTFree(p->cond);
+      tfree(p->cond);
    }
  
    taosArrayDestroy(pTagCond->pCond);
@ -1624,6 +1609,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
  pQueryInfo->exprList   = taosArrayInit(4, POINTER_BYTES);
  pQueryInfo->colList    = taosArrayInit(4, POINTER_BYTES);
  pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX;
+  pQueryInfo->resColumnId= -1000;
 }

 int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
@ -1665,11 +1651,12 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
  if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
    taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
    pQueryInfo->groupbyExpr.columnInfo = NULL;
+    pQueryInfo->groupbyExpr.numOfGroupCols = 0;
  }
  
  pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);

-  taosTFree(pQueryInfo->fillVal);
+  tfree(pQueryInfo->fillVal);
 }

 void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@ -1689,7 +1676,7 @@ void tscFreeVgroupTableInfo(SArray* pVgroupTables) {
    SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);

    for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
-      taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
+      tfree(pInfo->vgInfo.epAddr[j].fqdn);
    }

    taosArrayDestroy(pInfo->itemList);
@ -1706,14 +1693,25 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) {

  SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index);
  for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
-    taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
+    tfree(pInfo->vgInfo.epAddr[j].fqdn);
  }

  taosArrayDestroy(pInfo->itemList);
  taosArrayRemove(pVgroupTable, index);
 }

-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
+  memset(info, 0, sizeof(SVgroupTableInfo));
+
+  info->vgInfo = pInfo->vgInfo;
+  for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+    info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
+  }
+
+  info->itemList = taosArrayClone(pInfo->itemList);
+}
+
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) {
  if (pVgroupTables == NULL) {
    return NULL;
  }
@ -1724,14 +1722,8 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
  SVgroupTableInfo info;
  for (size_t i = 0; i < num; i++) {
    SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
-    memset(&info, 0, sizeof(SVgroupTableInfo));
+    tscVgroupTableCopy(&info, pInfo);

-    info.vgInfo = pInfo->vgInfo;
-    for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
-      info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
-    }
-
-    info.itemList = taosArrayClone(pInfo->itemList);
    taosArrayPush(pa, &info);
  }

@ -1739,7 +1731,7 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
 }

 void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
-  tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
+  tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables);
  
  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@ -1749,7 +1741,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
    free(pTableMetaInfo);
  }
  
-  taosTFree(pQueryInfo->pTableMetaInfo);
+  tfree(pQueryInfo->pTableMetaInfo);
 }

 STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
@ -1779,6 +1771,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
    pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
  }

+  // TODO handle malloc failure
  pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
  if (pTableMetaInfo->tagColList == NULL) {
    return NULL;
@ -1788,7 +1781,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
    tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
  }

-  pTableMetaInfo->pVgroupTables = tscCloneVgroupTableInfo(pVgroupTables);
+  pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables);
  
  pQueryInfo->numOfTables += 1;
  return pTableMetaInfo;
@ -2155,6 +2148,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
  }
 }

+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) {
+  int32_t numOfTags = tscGetNumOfTags(pTableMeta);
+
+  SSchema* pSchema = tscGetTableTagSchema(pTableMeta);
+  for(int32_t i = 0; i < numOfTags; ++i) {
+    if (pSchema[i].colId == colId) {
+      return i;
+    }
+  }
+
+  // can not reach here
+  assert(0);
+  return INT16_MIN;
+}
+
 bool tscIsUpdateQuery(SSqlObj* pSql) {
  if (pSql == NULL || pSql->signature != pSql) {
    terrno = TSDB_CODE_TSC_DISCONNECTED;
@ -2332,7 +2340,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
  
  pRes->numOfTotal = num;
  
-  taosTFree(pSql->pSubs);
+  tfree(pSql->pSubs);
  pSql->subState.numOfSub = 0;
  pSql->fp = fp;

@ -2424,7 +2432,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
    return NULL;
  }

-  size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
+  size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups;
  SVgroupsInfo* pNew = calloc(1, size);
  if (pNew == NULL) {
    return NULL;
@ -2433,9 +2441,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
  pNew->numOfVgroups = vgroupList->numOfVgroups;

  for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
-    SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i];
+    SVgroupInfo* pNewVInfo = &pNew->vgroups[i];

-    SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i];
+    SVgroupInfo* pvInfo = &vgroupList->vgroups[i];
    pNewVInfo->vgId = pvInfo->vgId;
    pNewVInfo->numOfEps = pvInfo->numOfEps;

@ -2454,21 +2462,22 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
  }

  for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
-    SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
+    SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];

    for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
-      taosTFree(pVgroupInfo->epAddr[j].fqdn);
+      tfree(pVgroupInfo->epAddr[j].fqdn);
    }
  }

-  taosTFree(vgroupList);
+  tfree(vgroupList);
  return NULL;
 }

-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) {
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
  dst->vgId = src->vgId;
  dst->numOfEps = src->numOfEps;
  for(int32_t i = 0; i < dst->numOfEps; ++i) {
+    tfree(dst->epAddr[i].fqdn);
    dst->epAddr[i].port = src->epAddr[i].port;
    dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
  }
--- a/src/common/inc/tdataformat.h
+++ b/src/common/inc/tdataformat.h
@ -80,7 +80,7 @@ typedef struct {
 #define schemaFLen(s) ((s)->flen)
 #define schemaVLen(s) ((s)->vlen)
 #define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s) taosTFree((s))
+#define tdFreeSchema(s) tfree((s))

 STSchema *tdDupSchema(STSchema *pSchema);
 int       tdEncodeSchema(void **buf, STSchema *pSchema);
@ -119,6 +119,33 @@ void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
 int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);

+// ----------------- Semantic timestamp key definition
+typedef uint64_t TKEY;
+
+#define TKEY_INVALID UINT64_MAX
+#define TKEY_NULL TKEY_INVALID
+#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
+#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
+#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
+
+#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
+#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
+#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
+#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
+#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
+
+static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
+  TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
+  TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
+
+  if (key1 < key2) {
+    return -1;
+  } else if (key1 > key2) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
 // ----------------- Data row structure

 /* A data row, the format is like below:
@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 * +----------+----------+---------------------------------+---------------------------------+
 * |   len    | sversion |           First part            |             Second part         |
 * +----------+----------+---------------------------------+---------------------------------+
+ * 
+ * NOTE: timestamp in this row structure is TKEY instead of TSKEY
 */
 typedef void *SDataRow;

@ -137,11 +166,13 @@ typedef void *SDataRow;
 #define dataRowLen(r) (*(uint16_t *)(r))
 #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
+#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
+#define dataRowKey(r) tdGetKey(dataRowTKey(r))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
 #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
 #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
+#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))

 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
  char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));

-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-      memcpy(ptr, value, varDataTLen(value));
-      dataRowLen(row) += varDataTLen(value);
-      break;
-    default:
+  if (IS_VAR_DATA_TYPE(type)) {
+    *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+    memcpy(ptr, value, varDataTLen(value));
+    dataRowLen(row) += varDataTLen(value);
+  } else {
+    if (offset == 0) {
+      ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
+      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
+      memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
+    } else {
      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-      break;
+    }
  }

  return 0;
@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i

 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-    default:
-      return POINTER_SHIFT(row, offset);
+  if (IS_VAR_DATA_TYPE(type)) {
+    return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
+  } else {
+    return POINTER_SHIFT(row, offset);
  }
 }

@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }

 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
 void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
 void dataColSetOffset(SDataCol *pCol, int nEle);

 bool isNEleNull(SDataCol *pCol, int nEle);
@ -204,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);

 // Get the data pointer from a column-wised data
 static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
-  switch (pCol->type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
-      break;
-
-    default:
-      return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
-      break;
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+  } else {
+    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
  }
 }

 static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
  ASSERT(rows > 0);

-  switch (pDataCol->type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
-      break;
-    default:
-      return TYPE_BYTES[pDataCol->type] * rows;
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
+  } else {
+    return TYPE_BYTES[pDataCol->type] * rows;
  }
 }

@ -243,9 +265,14 @@ typedef struct {
 } SDataCols;

 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
-#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
-#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
-#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
+#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
+#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
+#define dataColsTKeyLast(pCols) \
+  (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsKeyLast(pCols) \
+  (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))

 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
@ -253,10 +280,7 @@ int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 void       tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
-void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
-                              int limit2, int tRows);

 // ----------------- K-V data row structure
 /*
@ -284,7 +308,7 @@ typedef struct {
 #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
 #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
 #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
-#define kvRowFree(r) taosTFree(r)
+#define kvRowFree(r) tfree(r)
 #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))

 SKVRow tdKVRowDup(SKVRow row);
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@ -44,6 +44,7 @@ extern int32_t  tsMaxShellConns;
 extern int32_t  tsShellActivityTimer;
 extern uint32_t tsMaxTmrCtrl;
 extern float    tsNumOfThreadsPerCore;
+extern int32_t  tsNumOfCommitThreads;
 extern float    tsRatioOfQueryThreads;  // todo remove it
 extern int8_t   tsDaylight;
 extern char     tsTimezone[];
@ -51,6 +52,7 @@ extern char     tsLocale[];
 extern char     tsCharset[];            // default encode string
 extern int32_t  tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
+extern char     tsTempDir[];

 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer for each data node during query processing
@ -87,6 +89,7 @@ extern int16_t tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
 extern int32_t tsQuorum;
+extern int32_t tsUpdate;

 // balance
 extern int32_t tsEnableBalance;
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@ -35,6 +35,6 @@ bool tscValidateTableNameLength(size_t len);

 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);

-// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
+SSchema tscGetTbnameColumnSchema();

 #endif  // TDENGINE_NAME_H
--- a/src/common/src/tdataformat.c
+++ b/src/common/src/tdataformat.c
@ -18,6 +18,9 @@
 #include "tcoding.h"
 #include "wchar.h"

+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows);
+
 /**
 * Duplicate the schema and return a new object
 */
@ -94,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {

 void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
  if (pBuilder) {
-    taosTFree(pBuilder->columns);
+    tfree(pBuilder->columns);
  }
 }

@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
  pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;

  pDataCol->len = 0;
-  if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
    pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
    pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
    pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
  }
 }

+// value from timestamp should be TKEY here instead of TSKEY
 void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
  ASSERT(pCol != NULL && value != NULL);

-  switch (pCol->type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      // set offset
-      pCol->dataOff[numOfRows] = pCol->len;
-      // Copy data
-      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
-      // Update the length
-      pCol->len += varDataTLen(value);
-      break;
-    default:
-      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-      memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
-      pCol->len += pCol->bytes;
-      break;
-  }
-}
-
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
-  int pointsLeft = numOfRows - pointsToPop;
-
-  ASSERT(pointsLeft > 0);
-
-  if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
-    ASSERT(pCol->len > 0);
-    VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
-    pCol->len = pCol->len - toffset;
-    ASSERT(pCol->len > 0);
-    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
-    dataColSetOffset(pCol, pointsLeft);
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    // set offset
+    pCol->dataOff[numOfRows] = pCol->len;
+    // Copy data
+    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+    // Update the length
+    pCol->len += varDataTLen(value);
  } else {
    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-    pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
-    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
+    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
+    pCol->len += pCol->bytes;
  }
 }

 bool isNEleNull(SDataCol *pCol, int nEle) {
-  switch (pCol->type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      for (int i = 0; i < nEle; i++) {
-        if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
-      }
-      return true;
-    default:
-      for (int i = 0; i < nEle; i++) {
-        if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
-      }
-      return true;
+  for (int i = 0; i < nEle; i++) {
+    if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
  }
+  return true;
 }

 void dataColSetNullAt(SDataCol *pCol, int index) {
@ -367,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {

 void tdFreeDataCols(SDataCols *pCols) {
  if (pCols) {
-    taosTFree(pCols->buf);
-    taosTFree(pCols->cols);
+    tfree(pCols->buf);
+    tfree(pCols->cols);
    free(pCols);
  }
 }
@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
    pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
    pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));

-    if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+    if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
      ASSERT(pDataCols->cols[i].dataOff != NULL);
      pRet->cols[i].dataOff =
          (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
      pRet->cols[i].len = pDataCols->cols[i].len;
      if (pDataCols->cols[i].len > 0) {
        memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
-        if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+        if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
          memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
        }
      }
@ -420,58 +392,54 @@ void tdResetDataCols(SDataCols *pCols) {
 }

 void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
-  ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));

  int rcol = 0;
  int dcol = 0;

-  while (dcol < pCols->numOfCols) {
-    SDataCol *pDataCol = &(pCols->cols[dcol]);
-    if (rcol >= schemaNCols(pSchema)) {
-      dataColSetNullAt(pDataCol, pCols->numOfRows);
-      dcol++;
-      continue;
+  if (dataRowDeleted(row)) {
+    for (; dcol < pCols->numOfCols; dcol++) {
+      SDataCol *pDataCol = &(pCols->cols[dcol]);
+      if (dcol == 0) {
+        dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
+      } else {
+        dataColSetNullAt(pDataCol, pCols->numOfRows);
+      }
    }
+  } else {
+    while (dcol < pCols->numOfCols) {
+      SDataCol *pDataCol = &(pCols->cols[dcol]);
+      if (rcol >= schemaNCols(pSchema)) {
+        dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dcol++;
+        continue;
+      }

-    STColumn *pRowCol = schemaColAt(pSchema, rcol);
-    if (pRowCol->colId == pDataCol->colId) {
-      void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
-      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
-      dcol++;
-      rcol++;
-    } else if (pRowCol->colId < pDataCol->colId) {
-      rcol++;
-    } else {
-      dataColSetNullAt(pDataCol, pCols->numOfRows);
-      dcol++;
+      STColumn *pRowCol = schemaColAt(pSchema, rcol);
+      if (pRowCol->colId == pDataCol->colId) {
+        void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
+        dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+        dcol++;
+        rcol++;
+      } else if (pRowCol->colId < pDataCol->colId) {
+        rcol++;
+      } else {
+        dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dcol++;
+      }
    }
  }
  pCols->numOfRows++;
 }

-// Pop pointsToPop points from the SDataCols
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
-  int pointsLeft = pCols->numOfRows - pointsToPop;
-  if (pointsLeft <= 0) {
-    tdResetDataCols(pCols);
-    return;
-  }
-
-  for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
-    SDataCol *pCol = pCols->cols + iCol;
-    dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
-  }
-  pCols->numOfRows = pointsLeft;
-}
-
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
-  ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
  ASSERT(target->numOfCols == source->numOfCols);

  SDataCols *pTarget = NULL;

  if (dataColsKeyLast(target) < dataColsKeyFirst(source)) {  // No overlap
+    ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
    for (int i = 0; i < rowsToMerge; i++) {
      for (int j = 0; j < source->numOfCols; j++) {
        if (source->cols[j].len > 0) {
@ -499,17 +467,23 @@ _err:
  return -1;
 }

-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
+// src2 data has more priority than src1
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows) {
  tdResetDataCols(target);
  ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);

  while (target->numOfRows < tRows) {
    if (*iter1 >= limit1 && *iter2 >= limit2) break;

-    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
-    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
+    TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
+    TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);

-    if (key1 <= key2) {
+    ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+
+    if (key1 < key2) {
      for (int i = 0; i < src1->numOfCols; i++) {
        ASSERT(target->cols[i].type == src1->cols[i].type);
        if (src1->cols[i].len > 0) {
@ -520,19 +494,23 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi

      target->numOfRows++;
      (*iter1)++;
-      if (key1 == key2) (*iter2)++;
-    } else {
-      for (int i = 0; i < src2->numOfCols; i++) {
-        ASSERT(target->cols[i].type == src2->cols[i].type);
-        if (src2->cols[i].len > 0) {
-          dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
-                           target->maxPoints);
+    } else if (key1 >= key2) {
+      if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
+        for (int i = 0; i < src2->numOfCols; i++) {
+          ASSERT(target->cols[i].type == src2->cols[i].type);
+          if (src2->cols[i].len > 0) {
+            dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
+                             target->maxPoints);
+          }
        }
+        target->numOfRows++;
      }

-      target->numOfRows++;
      (*iter2)++;
+      if (key1 == key2) (*iter1)++;
    }
+
+    ASSERT(target->numOfRows <= target->maxPoints);
  }
 }

@ -691,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) {
 }

 void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) {
-  taosTFree(pBuilder->pColIdx);
-  taosTFree(pBuilder->buf);
+  tfree(pBuilder->pColIdx);
+  tfree(pBuilder->buf);
 }

 void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@ -51,6 +51,7 @@ int32_t tsMaxShellConns  = 5000;
 int32_t tsMaxConnections = 5000;
 int32_t tsShellActivityTimer  = 3;  // second
 float   tsNumOfThreadsPerCore = 1.0f;
+int32_t tsNumOfCommitThreads = 1;
 float   tsRatioOfQueryThreads = 0.5f;
 int8_t  tsDaylight       = 0;
 char    tsTimezone[TSDB_TIMEZONE_LEN] = {0};
@ -58,6 +59,7 @@ char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int32_t tsEnableCoreFile = 0;
 int32_t tsMaxBinaryDisplayWidth = 30;
+char    tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";

 /*
 * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -119,6 +121,7 @@ int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int32_t tsUpdate        = TSDB_DEFAULT_DB_UPDATE_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@ -222,6 +225,8 @@ int32_t (*monitorStartSystemFp)() = NULL;
 void (*monitorStopSystemFp)() = NULL;
 void (*monitorExecuteSQLFp)(char *sql) = NULL;

+char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
+
 static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;

 void taosSetAllDebugFlag() {
@ -461,6 +466,16 @@ static void doInitGlobalConfig(void) {
  cfg.unitType = TAOS_CFG_UTYPE_NONE;
  taosInitConfigOption(cfg);

+  cfg.option = "numOfCommitThreads";
+  cfg.ptr = &tsNumOfCommitThreads;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 1;
+  cfg.maxValue = 100;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
  cfg.option = "ratioOfQueryThreads";
  cfg.ptr = &tsRatioOfQueryThreads;
  cfg.valType = TAOS_CFG_VTYPE_FLOAT;
@ -824,6 +839,16 @@ static void doInitGlobalConfig(void) {
  cfg.unitType = TAOS_CFG_UTYPE_NONE;
  taosInitConfigOption(cfg);

+  cfg.option = "update";
+  cfg.ptr = &tsUpdate;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_UPDATE;
+  cfg.maxValue = TSDB_MAX_DB_UPDATE;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
  cfg.option = "mqttHostName";
  cfg.ptr = tsMqttHostName;
  cfg.valType = TAOS_CFG_VTYPE_STRING;
@ -1349,13 +1374,23 @@ static void doInitGlobalConfig(void) {
  cfg.ptrLength = 0;
  cfg.unitType = TAOS_CFG_UTYPE_NONE;
  taosInitConfigOption(cfg);
+
+  cfg.option = "tempDir";
+  cfg.ptr = tsTempDir;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = tListLen(tsTempDir);
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
 }

 void taosInitGlobalCfg() {
  pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
 }

-bool taosCheckGlobalCfg() {
+int32_t taosCheckGlobalCfg() {
  char fqdn[TSDB_FQDN_LEN];
  uint16_t port;

@ -1415,7 +1450,9 @@ bool taosCheckGlobalCfg() {
  tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
  tsHttpPort = tsServerPort + TSDB_PORT_HTTP;

-  return true;
+  taosPrintGlobalCfg();
+
+  return 0;
 }

 int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@ -188,3 +188,14 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
    pToken->z = r;
  }
 }
+
+SSchema tscGetTbnameColumnSchema() {
+  struct SSchema s = {
+      .colId = TSDB_TBNAME_COLUMN_INDEX,
+      .type  = TSDB_DATA_TYPE_BINARY,
+      .bytes = TSDB_TABLE_NAME_LEN
+  };
+
+  strcpy(s.name, TSQL_TBNAME_L);
+  return s;
+}
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@ -355,32 +355,6 @@ bool isValidDataType(int32_t type) {
  return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
 }

-//bool isNull(const char *val, int32_t type) {
-//  switch (type) {
-//    case TSDB_DATA_TYPE_BOOL:
-//      return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
-//    case TSDB_DATA_TYPE_TINYINT:
-//      return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
-//    case TSDB_DATA_TYPE_SMALLINT:
-//      return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
-//    case TSDB_DATA_TYPE_INT:
-//      return *(uint32_t *)val == TSDB_DATA_INT_NULL;
-//    case TSDB_DATA_TYPE_BIGINT:
-//    case TSDB_DATA_TYPE_TIMESTAMP:
-//      return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
-//    case TSDB_DATA_TYPE_FLOAT:
-//      return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
-//    case TSDB_DATA_TYPE_DOUBLE:
-//      return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
-//    case TSDB_DATA_TYPE_NCHAR:
-//      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
-//    case TSDB_DATA_TYPE_BINARY:
-//      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
-//    default:
-//      return false;
-//  };
-//}
-
 void setVardataNull(char* val, int32_t type) {
  if (type == TSDB_DATA_TYPE_BINARY) {
    varDataSetLen(val, sizeof(int8_t));
@ -433,14 +407,10 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
        *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
      }
      break;
-    case TSDB_DATA_TYPE_NCHAR: // todo : without length?
-      for (int32_t i = 0; i < numOfElems; ++i) {
-        *(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
-      }
-      break;
+    case TSDB_DATA_TYPE_NCHAR:
    case TSDB_DATA_TYPE_BINARY:
      for (int32_t i = 0; i < numOfElems; ++i) {
-        *(uint8_t *)(val + i * bytes) = TSDB_DATA_BINARY_NULL;
+        setVardataNull(val + i * bytes, type);
      }
      break;
    default: {
--- a/src/common/src/tvariant.c
+++ b/src/common/src/tvariant.c
@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
      break;
    }
    case TSDB_DATA_TYPE_BINARY: {  // todo refactor, extract a method
-      pVar->pz = calloc(len, sizeof(char));
+      pVar->pz = calloc(len + 1, sizeof(char));
      memcpy(pVar->pz, pz, len);
      pVar->nLen = (int32_t)len;
      break;
@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) {
  if (pVar == NULL) return;
  
  if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
-    taosTFree(pVar->pz);
+    tfree(pVar->pz);
    pVar->nLen = 0;
  }

--- a/src/connector/C#/TDengineDriver.cs
+++ b/src/connector/C#/TDengineDriver.cs
@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace TDengineDriver
+{
+  enum TDengineDataType {
+    TSDB_DATA_TYPE_NULL = 0,     // 1 bytes
+    TSDB_DATA_TYPE_BOOL = 1,     // 1 bytes
+    TSDB_DATA_TYPE_TINYINT = 2,  // 1 bytes
+    TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes
+    TSDB_DATA_TYPE_INT = 4,      // 4 bytes
+    TSDB_DATA_TYPE_BIGINT = 5,   // 8 bytes
+    TSDB_DATA_TYPE_FLOAT = 6,    // 4 bytes
+    TSDB_DATA_TYPE_DOUBLE = 7,   // 8 bytes
+    TSDB_DATA_TYPE_BINARY = 8,   // string
+    TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes
+    TSDB_DATA_TYPE_NCHAR = 10    // unicode string
+  }
+
+  enum TDengineInitOption
+  {
+    TSDB_OPTION_LOCALE = 0,
+    TSDB_OPTION_CHARSET = 1,
+    TSDB_OPTION_TIMEZONE = 2,
+    TDDB_OPTION_CONFIGDIR = 3,
+    TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4
+  }
+
+  class TDengineMeta
+  {
+    public string name;
+    public short size;
+    public byte type;
+    public string TypeName()
+    {
+      switch ((TDengineDataType)type)
+      {
+        case TDengineDataType.TSDB_DATA_TYPE_BOOL:
+          return "BOOLEAN";
+        case TDengineDataType.TSDB_DATA_TYPE_TINYINT:
+          return "BYTE";
+        case TDengineDataType.TSDB_DATA_TYPE_SMALLINT:
+          return "SHORT";
+        case TDengineDataType.TSDB_DATA_TYPE_INT:
+          return "INT";
+        case TDengineDataType.TSDB_DATA_TYPE_BIGINT:
+          return "LONG";
+        case TDengineDataType.TSDB_DATA_TYPE_FLOAT:
+          return "FLOAT";
+        case TDengineDataType.TSDB_DATA_TYPE_DOUBLE:
+          return "DOUBLE";
+        case TDengineDataType.TSDB_DATA_TYPE_BINARY:
+          return "STRING";
+        case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP:
+          return "TIMESTAMP";
+        case TDengineDataType.TSDB_DATA_TYPE_NCHAR:
+          return "NCHAR";
+        default:
+          return "undefine";
+      }
+    }
+  }
+
+  class TDengine
+  {
+    public const int TSDB_CODE_SUCCESS = 0;
+
+    [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)]
+    static extern public void Init();
+
+    [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)]
+    static extern public void Cleanup();
+
+    [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)]
+    static extern public void Options(int option, string value);
+
+    [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)]
+    static extern public IntPtr Connect(string ip, string user, string password, string db, short port);
+
+    [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)]
+    static extern private IntPtr taos_errstr(IntPtr res);
+    static public string Error(IntPtr res)
+    {
+      IntPtr errPtr = taos_errstr(res);
+      return Marshal.PtrToStringAnsi(errPtr);
+    }
+
+    [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)]
+    static extern public int ErrorNo(IntPtr res);
+
+    [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)]
+    static extern public IntPtr Query(IntPtr conn, string sqlstr);
+
+    [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)]
+    static extern public int AffectRows(IntPtr res);
+
+    [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)]
+    static extern public int FieldCount(IntPtr res);
+
+    [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)]
+    static extern private IntPtr taos_fetch_fields(IntPtr res);
+    static public List<TDengineMeta> FetchFields(IntPtr res)
+    {
+      const int fieldSize = 68;
+
+      List<TDengineMeta> metas = new List<TDengineMeta>();
+      if (res == IntPtr.Zero)
+      {
+        return metas;
+      }
+
+      int fieldCount = FieldCount(res);
+      IntPtr fieldsPtr = taos_fetch_fields(res);
+  
+      for (int i = 0; i < fieldCount; ++i)
+      {
+        int offset = i * fieldSize;
+        
+        TDengineMeta meta = new TDengineMeta();
+        meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset);
+        meta.type = Marshal.ReadByte(fieldsPtr + offset + 65);
+        meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66);
+        metas.Add(meta);
+      }
+
+      return metas;
+    }
+
+    [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)]
+    static extern public IntPtr FetchRows(IntPtr res);
+
+    [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)]
+    static extern public IntPtr FreeResult(IntPtr res);
+
+    [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)]
+    static extern public int Close(IntPtr taos);
+  }
+}
--- a/src/connector/go
+++ b/src/connector/go
@ -1 +1 @@
-Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b
+Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
--- a/src/connector/hivemq-tdengine-extension
+++ b/src/connector/hivemq-tdengine-extension
@ -0,0 +1 @@
+Subproject commit b62a26ecc164a310104df57691691b237e091c89
--- a/src/connector/jdbc/deploy-pom.xml
+++ b/src/connector/jdbc/deploy-pom.xml
@ -5,14 +5,13 @@

  <groupId>com.taosdata.jdbc</groupId>
  <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.6</version>
+  <version>2.0.10</version>
  <packaging>jar</packaging>

  <name>JDBCDriver</name>
  <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
  <description>TDengine JDBC Driver</description>

-
  <licenses>
    <license>
      <name>GNU AFFERO GENERAL PUBLIC LICENSE Version 3</name>
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
@ -52,7 +52,6 @@ public class TSDBConnection implements Connection {

    public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException {
        this.dbMetaData = meta;
-
        connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST),
                Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")),
                info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER),
@ -197,12 +196,14 @@ public class TSDBConnection implements Connection {
    }

    public SQLWarning getWarnings() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        //todo: implement getWarnings according to the warning messages returned from TDengine
+        return null;
+//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
    }

    public void clearWarnings() throws SQLException {
        // left blank to support HikariCP connection
-        //todo: implement getWarnings according to the warning messages returned from TDengine
+        //todo: implement clearWarnings according to the warning messages returned from TDengine
    }

    public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
@ -96,7 +96,7 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData {
    }

    public int getDriverMajorVersion() {
-        return 0;
+        return 2;
    }

    public int getDriverMinorVersion() {
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
@ -14,13 +14,9 @@
 *****************************************************************************/
 package com.taosdata.jdbc;

-
 import java.io.*;
-
 import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
+import java.util.*;
 import java.util.logging.Logger;

 /**
@ -44,76 +40,53 @@ import java.util.logging.Logger;
 */
 public class TSDBDriver implements java.sql.Driver {

-
    @Deprecated
    private static final String URL_PREFIX1 = "jdbc:TSDB://";

    private static final String URL_PREFIX = "jdbc:TAOS://";

-    /**
-     * Key used to retrieve the database value from the properties instance passed
-     * to the driver.
-     */
-    public static final String PROPERTY_KEY_DBNAME = "dbname";
-
    /**
     * Key used to retrieve the host value from the properties instance passed to
     * the driver.
     */
    public static final String PROPERTY_KEY_HOST = "host";
-    /**
-     * Key used to retrieve the password value from the properties instance passed
-     * to the driver.
-     */
-    public static final String PROPERTY_KEY_PASSWORD = "password";
-
    /**
     * Key used to retrieve the port number value from the properties instance
     * passed to the driver.
     */
    public static final String PROPERTY_KEY_PORT = "port";
-
+    /**
+     * Key used to retrieve the database value from the properties instance passed
+     * to the driver.
+     */
+    public static final String PROPERTY_KEY_DBNAME = "dbname";
    /**
     * Key used to retrieve the user value from the properties instance passed to
     * the driver.
     */
    public static final String PROPERTY_KEY_USER = "user";
-
-
+    /**
+     * Key used to retrieve the password value from the properties instance passed
+     * to the driver.
+     */
+    public static final String PROPERTY_KEY_PASSWORD = "password";
    /**
     * Key for the configuration file directory of TSDB client in properties instance
     */
    public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir";
-
    /**
     * Key for the timezone used by the TSDB client in properties instance
     */
    public static final String PROPERTY_KEY_TIME_ZONE = "timezone";
-
    /**
     * Key for the locale used by the TSDB client in properties instance
     */
    public static final String PROPERTY_KEY_LOCALE = "locale";
-
-
    /**
     * Key for the char encoding used by the TSDB client in properties instance
     */
    public static final String PROPERTY_KEY_CHARSET = "charset";

-    public static final String PROPERTY_KEY_PROTOCOL = "protocol";
-
-
-    /**
-     * Index for port coming out of parseHostPortPair().
-     */
-    public final static int PORT_NUMBER_INDEX = 1;
-
-    /**
-     * Index for host coming out of parseHostPortPair().
-     */
-    public final static int HOST_NAME_INDEX = 0;
-
    private TSDBDatabaseMetaData dbMetaData = null;

    static {
@ -169,9 +142,11 @@ public class TSDBDriver implements java.sql.Driver {
    }

    public Connection connect(String url, Properties info) throws SQLException {
-        if (url == null) {
+        if (url == null)
            throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!"));
-        }
+
+        if (!acceptsURL(url))
+            return null;

        Properties props = null;
        if ((props = parseURL(url, info)) == null) {
@ -179,7 +154,10 @@ public class TSDBDriver implements java.sql.Driver {
        }

        //load taos.cfg start
-        if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) {
+        if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null ||
+                info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && (
+                info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null ||
+                        info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) {
            File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
            File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0];
            List<String> endpoints = loadConfigEndpoints(cfgFile);
@ -190,7 +168,9 @@ public class TSDBDriver implements java.sql.Driver {
        }

        try {
-            TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET),
+            TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR),
+                    (String) props.get(PROPERTY_KEY_LOCALE),
+                    (String) props.get(PROPERTY_KEY_CHARSET),
                    (String) props.get(PROPERTY_KEY_TIME_ZONE));
            Connection newConn = new TSDBConnection(props, this.dbMetaData);
            return newConn;
@ -208,43 +188,15 @@ public class TSDBDriver implements java.sql.Driver {
    }

    /**
-     * Parses hostPortPair in the form of [host][:port] into an array, with the
-     * element of index HOST_NAME_INDEX being the host (or null if not specified),
-     * and the element of index PORT_NUMBER_INDEX being the port (or null if not
-     * specified).
-     *
-     * @param hostPortPair host and port in form of of [host][:port]
-     * @return array containing host and port as Strings
-     * @throws SQLException if a parse error occurs
+     * @param url the URL of the database
+     * @return <code>true</code> if this driver understands the given URL;
+     * <code>false</code> otherwise
+     * @throws SQLException if a database access error occurs or the url is {@code null}
     */
-    protected static String[] parseHostPortPair(String hostPortPair) throws SQLException {
-        String[] splitValues = new String[2];
-
-        int portIndex = hostPortPair.indexOf(":");
-
-        String hostname = null;
-
-        if (portIndex != -1) {
-            if ((portIndex + 1) < hostPortPair.length()) {
-                String portAsString = hostPortPair.substring(portIndex + 1);
-                hostname = hostPortPair.substring(0, portIndex);
-
-                splitValues[HOST_NAME_INDEX] = hostname;
-
-                splitValues[PORT_NUMBER_INDEX] = portAsString;
-            } else {
-                throw new SQLException(TSDBConstants.WrapErrMsg("port is not proper!"));
-            }
-        } else {
-            splitValues[HOST_NAME_INDEX] = hostPortPair;
-            splitValues[PORT_NUMBER_INDEX] = null;
-        }
-
-        return splitValues;
-    }
-
    public boolean acceptsURL(String url) throws SQLException {
-        return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX);
+        if (url == null)
+            throw new SQLException(TSDBConstants.WrapErrMsg("url is null"));
+        return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1));
    }

    public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@ -252,15 +204,17 @@ public class TSDBDriver implements java.sql.Driver {
            info = new Properties();
        }

-        if ((url != null) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1))) {
+        if (acceptsURL(url)) {
            info = parseURL(url, info);
        }

        DriverPropertyInfo hostProp = new DriverPropertyInfo(PROPERTY_KEY_HOST, info.getProperty(PROPERTY_KEY_HOST));
-        hostProp.required = true;
+        hostProp.required = false;
+        hostProp.description = "Hostname";

        DriverPropertyInfo portProp = new DriverPropertyInfo(PROPERTY_KEY_PORT, info.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
        portProp.required = false;
+        portProp.description = "Port";

        DriverPropertyInfo dbProp = new DriverPropertyInfo(PROPERTY_KEY_DBNAME, info.getProperty(PROPERTY_KEY_DBNAME));
        dbProp.required = false;
@ -268,9 +222,11 @@ public class TSDBDriver implements java.sql.Driver {

        DriverPropertyInfo userProp = new DriverPropertyInfo(PROPERTY_KEY_USER, info.getProperty(PROPERTY_KEY_USER));
        userProp.required = true;
+        userProp.description = "User";

        DriverPropertyInfo passwordProp = new DriverPropertyInfo(PROPERTY_KEY_PASSWORD, info.getProperty(PROPERTY_KEY_PASSWORD));
        passwordProp.required = true;
+        passwordProp.description = "Password";

        DriverPropertyInfo[] propertyInfo = new DriverPropertyInfo[5];
        propertyInfo[0] = hostProp;
@ -283,20 +239,68 @@ public class TSDBDriver implements java.sql.Driver {
    }

    /**
-     * example: jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password
+     * example: jdbc:TAOS://127.0.0.1:0/db?user=root&password=your_password
     */
-    public Properties parseURL(String url, Properties defaults) throws java.sql.SQLException {
+    public Properties parseURL(String url, Properties defaults) {
        Properties urlProps = (defaults != null) ? defaults : new Properties();
-        if (url == null) {
+        if (url == null || url.length() <= 0 || url.trim().length() <= 0)
            return null;
-        }
-
-        if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) {
+        if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1))
            return null;
-        }

+        // parse properties
        String urlForMeta = url;
+        int beginningOfSlashes = url.indexOf("//");
+        int index = url.indexOf("?");
+        if (index != -1) {
+            String paramString = url.substring(index + 1, url.length());
+            url = url.substring(0, index);
+            StringTokenizer queryParams = new StringTokenizer(paramString, "&");
+            while (queryParams.hasMoreElements()) {
+                String parameterValuePair = queryParams.nextToken();
+                int indexOfEqual = parameterValuePair.indexOf("=");
+                String parameter = null;
+                String value = null;
+                if (indexOfEqual != -1) {
+                    parameter = parameterValuePair.substring(0, indexOfEqual);
+                    if (indexOfEqual + 1 < parameterValuePair.length()) {
+                        value = parameterValuePair.substring(indexOfEqual + 1);
+                    }
+                }
+                if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) {
+                    urlProps.setProperty(parameter, value);
+                }
+            }
+        }
+        // parse Product Name
+        String dbProductName = url.substring(0, beginningOfSlashes);
+        dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1);
+        dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
+        // parse dbname
+        url = url.substring(beginningOfSlashes + 2);
+        int indexOfSlash = url.indexOf("/");
+        if (indexOfSlash != -1) {
+            if (indexOfSlash + 1 < url.length()) {
+                urlProps.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, url.substring(indexOfSlash + 1));
+            }
+            url = url.substring(0, indexOfSlash);
+        }
+        // parse port
+        int indexOfColon = url.indexOf(":");
+        if (indexOfColon != -1) {
+            if (indexOfColon + 1 < url.length()) {
+                urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PORT, url.substring(indexOfColon + 1));
+            }
+            url = url.substring(0, indexOfColon);
+        }
+        if (url != null && url.length() > 0 && url.trim().length() > 0) {
+            urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url);
+        }

+        this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER));
+
+        /*
+        String urlForMeta = url;
        String dbProductName = url.substring(url.indexOf(":") + 1);
        dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
        int beginningOfSlashes = url.indexOf("//");
@ -345,11 +349,11 @@ public class TSDBDriver implements java.sql.Driver {

        user = urlProps.getProperty(PROPERTY_KEY_USER).toString();
        this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user);
-
+*/
        return urlProps;
    }

-    public void setPropertyValue(Properties property, String[] keyValuePair) {
+    private void setPropertyValue(Properties property, String[] keyValuePair) {
        switch (keyValuePair[0].toLowerCase()) {
            case PROPERTY_KEY_USER:
                property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]);
@ -372,13 +376,12 @@ public class TSDBDriver implements java.sql.Driver {
        }
    }

-
    public int getMajorVersion() {
-        return 1;
+        return 2;
    }

    public int getMinorVersion() {
-        return 1;
+        return 0;
    }

    public boolean jdbcCompliant() {
@ -389,33 +392,4 @@ public class TSDBDriver implements java.sql.Driver {
        return null;
    }

-    /**
-     * Returns the host property
-     *
-     * @param props the java.util.Properties instance to retrieve the hostname from.
-     * @return the host
-     */
-    public String host(Properties props) {
-        return props.getProperty(PROPERTY_KEY_HOST, "localhost");
-    }
-
-    /**
-     * Returns the port number property
-     *
-     * @param props the properties to get the port number from
-     * @return the port number
-     */
-    public int port(Properties props) {
-        return Integer.parseInt(props.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
-    }
-
-    /**
-     * Returns the database property from <code>props</code>
-     *
-     * @param props the Properties to look for the database property.
-     * @return the database name.
-     */
-    public String database(Properties props) {
-        return props.getProperty(PROPERTY_KEY_DBNAME);
-    }
 }
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
@ -111,8 +111,8 @@ public class TSDBJNIConnector {
     * @throws SQLException
     */
    public long executeQuery(String sql) throws SQLException {
-    	// close previous result set if the user forgets to invoke the 
-    	// free method to close previous result set.
+        // close previous result set if the user forgets to invoke the
+        // free method to close previous result set.
        if (!this.isResultsetClosed) {
            freeResultSet(taosResultSetPointer);
        }
@ -122,23 +122,23 @@ public class TSDBJNIConnector {
            pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
        } catch (Exception e) {
            e.printStackTrace();
-            this.freeResultSet(pSql);
+            this.freeResultSetImp(this.taos, pSql);
            throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
        }
-        
+
        int code = this.getErrCode(pSql);
        if (code != 0) {
            affectedRows = -1;
            String msg = this.getErrMsg(pSql);
-            
-            this.freeResultSet(pSql);
+
+            this.freeResultSetImp(this.taos, pSql);
            throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code);
        }

        // Try retrieving result set for the executed SQL using the current connection pointer. 
        taosResultSetPointer = this.getResultSetImp(this.taos, pSql);
        isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER);
-        
+
        return pSql;
    }

@ -171,11 +171,11 @@ public class TSDBJNIConnector {
    }

    private native long getResultSetImp(long connection, long pSql);
-    
+
    public boolean isUpdateQuery(long pSql) {
-    	return isUpdateQueryImp(this.taos, pSql) == 1? true:false;
+        return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false;
    }
-    
+
    private native long isUpdateQueryImp(long connection, long pSql);

    /**
@ -191,7 +191,7 @@ public class TSDBJNIConnector {
            res = this.freeResultSetImp(this.taos, result);
            taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
        }
-        
+
        isResultsetClosed = true;
        return res;
    }
@ -274,7 +274,7 @@ public class TSDBJNIConnector {
     * Consume a subscription
     */
    long consume(long subscription) {
-       return this.consumeImp(subscription);
+        return this.consumeImp(subscription);
    }

    private native long consumeImp(long subscription);
--- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
@ -1,47 +1,210 @@
 package com.taosdata.jdbc;

+import org.junit.BeforeClass;
 import org.junit.Test;

-import java.sql.SQLException;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.*;
 import java.util.Properties;

-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;

 public class TSDBDriverTest {

-    @Test
-    public void urlParserTest() throws SQLException {
-        TSDBDriver driver = new TSDBDriver();
-        String url = "jdbc:TSDB://127.0.0.1:0/db";
+    private static final String[] validURLs = {
+            "jdbc:TAOS://localhost:0",
+            "jdbc:TAOS://localhost",
+            "jdbc:TAOS://localhost:6030/test",
+            "jdbc:TAOS://localhost:6030",
+            "jdbc:TAOS://localhost:6030/",
+            "jdbc:TSDB://localhost:6030",
+            "jdbc:TSDB://localhost:6030/",
+            "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata",
+            "jdbc:TAOS://:",
+            "jdbc:TAOS://:/",
+            "jdbc:TAOS://:/test",
+            "jdbc:TAOS://localhost:0/?user=root&password=taosdata"
+    };
+    private static boolean islibLoaded = false;
+    private static boolean isTaosdActived;

-        Properties properties = new Properties(); 
-        driver.parseURL(url, properties);
-        assertEquals(properties.get("host"), "127.0.0.1");
-        assertEquals(properties.get("port"), "0");
-        assertEquals(properties.get("dbname"), "db");                
-        assertEquals(properties.get("user"), "root");   
-        assertEquals(properties.get("password"), "your_password");
+    private Connection conn;

-        url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8";
-        properties = new Properties(); 
-        driver.parseURL(url, properties);
-        assertEquals(properties.get("host"), "127.0.0.1");
-        assertEquals(properties.get("port"), "0");
-        assertEquals(properties.get("dbname"), "log");        
-        assertEquals(properties.get("charset"), "UTF-8"); 
-        
-        url = "jdbc:TSDB://127.0.0.1:0/";
-        properties = new Properties(); 
-        driver.parseURL(url, properties);
-        assertEquals(properties.get("host"), "127.0.0.1");
-        assertEquals(properties.get("port"), "0"); 
-        assertEquals(properties.get("dbname"), null);
-        
-        url = "jdbc:TSDB://127.0.0.1:0/db";
-        properties = new Properties(); 
-        driver.parseURL(url, properties);
-        assertEquals(properties.get("host"), "127.0.0.1");
-        assertEquals(properties.get("port"), "0"); 
-        assertEquals(properties.get("dbname"), "db");
+    @BeforeClass
+    public static void before() {
+        String osName = System.getProperty("os.name").toLowerCase();
+        if (!osName.equals("linux"))
+            return;
+        // try to load taos lib
+        try {
+            System.loadLibrary("taos");
+            islibLoaded = true;
+        } catch (UnsatisfiedLinkError error) {
+            System.out.println("load tdengine lib failed.");
+            error.printStackTrace();
+        }
+        // check taosd is activated
+        try {
+            String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""};
+            Process exec = Runtime.getRuntime().exec(cmd);
+            BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream()));
+            int lineCnt = 0;
+            while (reader.readLine() != null) {
+                lineCnt++;
+            }
+            if (lineCnt > 0)
+                isTaosdActived = true;
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            Class.forName("com.taosdata.jdbc.TSDBDriver");
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        }
    }
+
+    @Test
+    public void testConnectWithJdbcURL() {
+        final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+        try {
+            if (islibLoaded && isTaosdActived) {
+                conn = DriverManager.getConnection(url);
+                assertNotNull("failure - connection should not be null", conn);
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+            fail("failure - should not throw Exception");
+        }
+    }
+
+    @Test
+    public void testConnectWithProperties() {
+        final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+        Properties connProps = new Properties();
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+        try {
+            if (islibLoaded && isTaosdActived) {
+                conn = DriverManager.getConnection(jdbcUrl, connProps);
+                assertNotNull("failure - connection should not be null", conn);
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+            fail("failure - should not throw Exception");
+        }
+    }
+
+    @Test
+    public void testConnectWithConfigFile() {
+        String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata";
+        Properties connProps = new Properties();
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+        connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+        try {
+            if (islibLoaded && isTaosdActived) {
+                conn = DriverManager.getConnection(jdbcUrl, connProps);
+                assertNotNull("failure - connection should not be null", conn);
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+            fail("failure - should not throw Exception");
+        }
+    }
+
+    @Test(expected = SQLException.class)
+    public void testAcceptsURL() throws SQLException {
+        Driver driver = new TSDBDriver();
+        for (String url : validURLs) {
+            assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url));
+        }
+        driver.acceptsURL(null);
+        fail("acceptsURL throws exception when parameter is null");
+    }
+
+    @Test
+    public void testParseURL() {
+        TSDBDriver driver = new TSDBDriver();
+
+        String url = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata&charset=UTF-8";
+        Properties config = new Properties();
+        Properties actual = driver.parseURL(url, config);
+        assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.get("host"));
+        assertEquals("failure - port should be 0", "0", actual.get("port"));
+        assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+        assertEquals("failure - user should be root", "root", actual.get("user"));
+        assertEquals("failure - password should be taosdata", "taosdata", actual.get("password"));
+        assertEquals("failure - charset should be UTF-8", "UTF-8", actual.get("charset"));
+
+        url = "jdbc:TAOS://127.0.0.1:0";
+        config = new Properties();
+        actual = driver.parseURL(url, config);
+        assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+        assertEquals("failure - port should be 0", "0", actual.get("port"));
+        assertNull("failure - dbname should be null", actual.get("dbname"));
+
+        url = "jdbc:TAOS://127.0.0.1:0/db";
+        config = new Properties();
+        actual = driver.parseURL(url, config);
+        assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+        assertEquals("failure - port should be 0", "0", actual.get("port"));
+        assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+
+        url = "jdbc:TAOS://:/?";
+        config = new Properties();
+        config.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+        config.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
+        actual = driver.parseURL(url, config);
+        assertEquals("failure - user should be root", "root", actual.getProperty("user"));
+        assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password"));
+        assertNull("failure - host should be null", actual.getProperty("host"));
+        assertNull("failure - port should be null", actual.getProperty("port"));
+        assertNull("failure - dbname should be null", actual.getProperty("dbname"));
+    }
+
+    @Test
+    public void testGetPropertyInfo() throws SQLException {
+        Driver driver = new TSDBDriver();
+        final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+        Properties connProps = new Properties();
+        DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps);
+        for (DriverPropertyInfo info : propertyInfo) {
+            if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST))
+                assertEquals("failure - host should be localhost", "localhost", info.value);
+            if (info.name.equals(TSDBDriver.PROPERTY_KEY_PORT))
+                assertEquals("failure - port should be 6030", "6030", info.value);
+            if (info.name.equals(TSDBDriver.PROPERTY_KEY_DBNAME))
+                assertEquals("failure - dbname should be test", "log", info.value);
+            if (info.name.equals(TSDBDriver.PROPERTY_KEY_USER))
+                assertEquals("failure - user should be root", "root", info.value);
+            if (info.name.equals(TSDBDriver.PROPERTY_KEY_PASSWORD))
+                assertEquals("failure - password should be root", "taosdata", info.value);
+        }
+    }
+
+    @Test
+    public void testGetMajorVersion() {
+        assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion());
+    }
+
+    @Test
+    public void testGetMinorVersion() {
+        assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion());
+    }
+
+    @Test
+    public void testJdbcCompliant() {
+        assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant());
+    }
+
+    @Test
+    public void testGetParentLogger() throws SQLFeatureNotSupportedException {
+        assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger());
+    }
+
 }
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@ -39,16 +39,16 @@
 #define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ  ", cqDebugFlag, __VA_ARGS__); }}

 typedef struct {
-  int      vgId;
+  int32_t  vgId;
  char     user[TSDB_USER_LEN];
  char     pass[TSDB_PASSWORD_LEN];
  char     db[TSDB_DB_NAME_LEN];
  FCqWrite cqWrite;
  void    *ahandle;
-  int      num;      // number of continuous streams
+  int32_t  num;      // number of continuous streams
  struct SCqObj *pHead;
  void    *dbConn;
-  int      master;
+  int32_t  master;
  void    *tmrCtrl;
  pthread_mutex_t mutex;
 } SCqContext;
@ -57,7 +57,7 @@ typedef struct SCqObj {
  tmr_h          tmrId;
  uint64_t       uid;
  int32_t        tid;      // table ID
-  int            rowSize;  // bytes of a row
+  int32_t        rowSize;  // bytes of a row
  char *         sqlStr;   // SQL string
  STSchema *     pSchema;  // pointer to schema array
  void *         pStream;
@ -115,7 +115,7 @@ void cqClose(void *handle) {
    SCqObj *pTemp = pObj;
    pObj = pObj->next;
    tdFreeSchema(pTemp->pSchema);
-    taosTFree(pTemp->sqlStr);
+    tfree(pTemp->sqlStr);
    free(pTemp);
  } 
  
@ -175,7 +175,7 @@ void cqStop(void *handle) {
  pthread_mutex_unlock(&pContext->mutex);
 }

-void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) {
+void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
  SCqContext *pContext = handle;

  SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@ -237,7 +237,7 @@ void cqDrop(void *handle) {
  pthread_mutex_unlock(&pContext->mutex);
 }

-static void doCreateStream(void *param, TAOS_RES *result, int code) {
+static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
  SCqObj* pObj = (SCqObj*)param;
  SCqContext* pContext = pObj->pContext;
  SSqlObj* pSql = (SSqlObj*)result;
@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {

  cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);

-  int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
+  int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
  char *buffer = calloc(size, 1);

  SWalHead   *pHead = (SWalHead *)buffer;
@ -334,7 +334,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
  pHead->version = 0;

  // write into vnode write queue
-  pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ);
+  pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
  free(buffer);
 }

--- a/src/cq/test/cqtest.c
+++ b/src/cq/test/cqtest.c
@ -24,7 +24,7 @@
 int64_t  ver = 0;
 void    *pCq = NULL;

-int writeToQueue(void *pVnode, void *data, int type) {
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { 
  return 0;
 }

--- a/src/dnode/inc/dnodeCfg.h
+++ b/src/dnode/inc/dnodeCfg.h
@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_CFG_H
+#define TDENGINE_DNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dnodeInitCfg();
+void    dnodeCleanupCfg();
+void    dnodeUpdateCfg(SDnodeCfg *cfg);
+int32_t dnodeGetDnodeId();
+void    dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeEps.h
+++ b/src/dnode/inc/dnodeEps.h
@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_EP_H
+#define TDENGINE_DNODE_EP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitEps();
+void    dnodeCleanupEps();
+void    dnodeUpdateEps(SDnodeEps *eps);
+void    dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
+bool    dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeMInfos.h
+++ b/src/dnode/inc/dnodeMInfos.h
@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_MINFOS_H
+#define TDENGINE_DNODE_MINFOS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitMInfos();
+void    dnodeCleanupMInfos();
+void    dnodeUpdateMInfos(SMnodeInfos *minfos);
+void    dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetMInfos(SMnodeInfos *minfos);
+bool    dnodeIsMasterEp(char *ep);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeMPeer.h
+++ b/src/dnode/inc/dnodeMPeer.h
@ -20,11 +20,11 @@
 extern "C" {
 #endif

-int32_t dnodeInitMnodePeer();
-void    dnodeCleanupMnodePeer();
-int32_t dnodeAllocateMnodePqueue();
-void    dnodeFreeMnodePqueue();
-void    dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMPeer();
+void    dnodeCleanupMPeer();
+int32_t dnodeAllocateMPeerQueue();
+void    dnodeFreeMPeerQueue();
+void    dnodeDispatchToMPeerQueue(SRpcMsg *pMsg);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeMRead.h
+++ b/src/dnode/inc/dnodeMRead.h
@ -20,11 +20,11 @@
 extern "C" {
 #endif

-int32_t dnodeInitMnodeRead();
-void    dnodeCleanupMnodeRead();
-int32_t dnodeAllocateMnodeRqueue();
-void    dnodeFreeMnodeRqueue();
-void    dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
+int32_t dnodeInitMRead();
+void    dnodeCleanupMRead();
+int32_t dnodeAllocMReadQueue();
+void    dnodeFreeMReadQueue();
+void    dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeMWrite.h
+++ b/src/dnode/inc/dnodeMWrite.h
@ -20,11 +20,11 @@
 extern "C" {
 #endif

-int32_t dnodeInitMnodeWrite();
-void    dnodeCleanupMnodeWrite();
-int32_t dnodeAllocateMnodeWqueue();
-void    dnodeFreeMnodeWqueue();
-void    dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMWrite();
+void    dnodeCleanupMWrite();
+int32_t dnodeAllocMWritequeue();
+void    dnodeFreeMWritequeue();
+void    dnodeDispatchToMWriteQueue(SRpcMsg *pMsg);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeMgmt.h
+++ b/src/dnode/inc/dnodeMgmt.h
@ -20,6 +20,8 @@
 extern "C" {
 #endif

+#include "trpc.h"
+
 int32_t dnodeInitMgmt();
 void    dnodeCleanupMgmt();
 int32_t dnodeInitMgmtTimer();
@ -35,8 +37,8 @@ void*   dnodeGetVnodeTsdb(void *pVnode);
 void    dnodeReleaseVnode(void *pVnode);

 void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetMnodeEpSetForPeer(void *epSet);
-void    dnodeGetMnodeEpSetForShell(void *epSet);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeVRead.h
+++ b/src/dnode/inc/dnodeVRead.h
@ -20,9 +20,11 @@
 extern "C" {
 #endif

-int32_t dnodeInitVnodeRead();
-void    dnodeCleanupVnodeRead();
-void    dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVRead();
+void    dnodeCleanupVRead();
+void    dnodeDispatchToVReadQueue(SRpcMsg *pMsg);
+void *  dnodeAllocVReadQueue(void *pVnode);
+void    dnodeFreeVReadQueue(void *rqueue);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeVWrite.h
+++ b/src/dnode/inc/dnodeVWrite.h
@ -20,9 +20,12 @@
 extern "C" {
 #endif

-int32_t dnodeInitVnodeWrite();
-void    dnodeCleanupVnodeWrite();
-void    dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVWrite();
+void    dnodeCleanupVWrite();
+void    dnodeDispatchToVWriteQueue(SRpcMsg *pMsg);
+void *  dnodeAllocVWriteQueue(void *pVnode);
+void    dnodeFreeVWriteQueue(void *pWqueue);
+void    dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code);

 #ifdef __cplusplus
 }
--- a/src/dnode/src/dnodeCfg.c
+++ b/src/dnode/src/dnodeCfg.c
@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeCfg.h"
+
+static SDnodeCfg tsCfg = {0};
+static pthread_mutex_t tsCfgMutex;
+
+static int32_t dnodeReadCfg();
+static int32_t dnodeWriteCfg();
+static void    dnodeResetCfg(SDnodeCfg *cfg);
+static void    dnodePrintCfg(SDnodeCfg *cfg);
+
+int32_t dnodeInitCfg() {
+  pthread_mutex_init(&tsCfgMutex, NULL);
+  dnodeResetCfg(NULL);
+  int32_t ret = dnodeReadCfg();
+  if (ret == 0) {
+    dInfo("dnode cfg is initialized");
+  }
+  return ret;
+}
+
+void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); }
+
+void dnodeUpdateCfg(SDnodeCfg *cfg) {
+  if (tsCfg.dnodeId != 0) return;
+  dnodeResetCfg(cfg);
+}
+
+int32_t dnodeGetDnodeId() {
+  int32_t dnodeId = 0;
+  pthread_mutex_lock(&tsCfgMutex);
+  dnodeId = tsCfg.dnodeId;
+  pthread_mutex_unlock(&tsCfgMutex);
+  return dnodeId;
+}
+
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+  pthread_mutex_lock(&tsCfgMutex);
+  *dnodeId = tsCfg.dnodeId;
+  tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodeResetCfg(SDnodeCfg *cfg) {
+  if (cfg == NULL) return;
+  if (cfg->dnodeId == 0) return;
+
+  pthread_mutex_lock(&tsCfgMutex);
+  tsCfg.dnodeId = cfg->dnodeId;
+  tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
+  dnodePrintCfg(cfg);
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodePrintCfg(SDnodeCfg *cfg) {
+  dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+}
+
+static int32_t dnodeReadCfg() {
+  int32_t   len = 0;
+  int32_t   maxLen = 200;
+  char *    content = calloc(1, maxLen + 1);
+  cJSON *   root = NULL;
+  FILE *    fp = NULL;
+  SDnodeCfg cfg = {0};
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_Number) {
+    dError("failed to read %s, dnodeId not found", file);
+    goto PARSE_CFG_OVER;
+  }
+  cfg.dnodeId = dnodeId->valueint;
+
+  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_String) {
+    dError("failed to read %s, clusterId not found", file);
+    goto PARSE_CFG_OVER;
+  }
+  tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+
+  dInfo("read file %s successed", file);
+
+PARSE_CFG_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  terrno = 0;
+
+  dnodeResetCfg(&cfg);
+  return 0;
+}
+
+static int32_t dnodeWriteCfg() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 200;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsCfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsCfg.clusterId);
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeCheck.c
+++ b/src/dnode/src/dnodeCheck.c
@ -15,9 +15,7 @@

 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosdef.h"
 #include "tglobal.h"
-#include "mnode.h"
 #include "dnodeInt.h"
 #include "dnodeCheck.h"

@ -30,8 +28,8 @@ typedef struct {
  void    (*stopFp)();
 } SCheckItem;

-static    SCheckItem  tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
-int64_t   tsMinFreeMemSizeForStart = 0;
+static  SCheckItem  tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
+int64_t tsMinFreeMemSizeForStart = 0;

 static int bindTcpPort(int port) {
  int    serverSocket;
@ -264,8 +262,6 @@ int32_t dnodeInitCheck() {
    }
  }

+  dInfo("dnode check is initialized");
  return 0;
 }
-
-
-
--- a/src/dnode/src/dnodeEps.c
+++ b/src/dnode/src/dnodeEps.c
@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "hash.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeEps.h"
+
+static SDnodeEps *tsEps = NULL;
+static SHashObj * tsEpsHash = NULL;
+static pthread_mutex_t tsEpsMutex;
+
+static int32_t dnodeReadEps();
+static int32_t dnodeWriteEps();
+static void    dnodeResetEps(SDnodeEps *eps);
+static void    dnodePrintEps(SDnodeEps *eps);
+
+int32_t dnodeInitEps() {
+  pthread_mutex_init(&tsEpsMutex, NULL);
+  tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  dnodeResetEps(NULL);
+  int32_t ret = dnodeReadEps();
+  if (ret == 0) {
+    dInfo("dnode eps is initialized");
+  }
+  return ret;
+}
+
+void dnodeCleanupEps() {
+  pthread_mutex_lock(&tsEpsMutex);
+  if (tsEps) {
+    free(tsEps);
+    tsEps = NULL;
+  }
+  if (tsEpsHash) {
+    taosHashCleanup(tsEpsHash);
+    tsEpsHash = NULL;
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+  pthread_mutex_destroy(&tsEpsMutex);
+}
+
+void dnodeUpdateEps(SDnodeEps *eps) {
+  if (eps == NULL) return;
+
+  eps->dnodeNum = htonl(eps->dnodeNum);
+  for (int32_t i = 0; i < eps->dnodeNum; ++i) {
+    eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId);
+    eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort);
+  }
+
+  pthread_mutex_lock(&tsEpsMutex);
+  if (eps->dnodeNum != tsEps->dnodeNum) {
+    dnodeResetEps(eps);
+    dnodeWriteEps();
+  } else {
+    int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp);
+    if (memcmp(eps, tsEps, size) != 0) {
+      dnodeResetEps(eps);
+      dnodeWriteEps();
+    }
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+}
+
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) {
+  bool changed = false;
+  pthread_mutex_lock(&tsEpsMutex);
+  SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+  if (ep != NULL) {
+    char epSaved[TSDB_EP_LEN + 1];
+    snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+    changed = strcmp(epstr, epSaved) != 0;
+    tstrncpy(epstr, epSaved, TSDB_EP_LEN);
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+  return changed;
+}
+
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
+  pthread_mutex_lock(&tsEpsMutex);
+  SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+  if (ep != NULL) {
+    if (port) *port = ep->dnodePort;
+    if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
+    if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+}
+
+static void dnodeResetEps(SDnodeEps *eps) {
+  if (eps == NULL) {
+    int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp);
+    if (tsEps == NULL) {
+      tsEps = calloc(1, size);
+    } else {
+      tsEps->dnodeNum = 0;
+    }
+  } else {
+    assert(tsEps);
+
+    int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum;
+    if (eps->dnodeNum > tsEps->dnodeNum) {
+      tsEps = realloc(tsEps, size);
+    }
+    memcpy(tsEps, eps, size);
+    dnodePrintEps(eps);
+  }
+
+  for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+    SDnodeEp *ep = &tsEps->dnodeEps[i];
+    taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+  }
+}
+
+static void dnodePrintEps(SDnodeEps *eps) {
+  dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
+  for (int32_t i = 0; i < eps->dnodeNum; i++) {
+    SDnodeEp *ep = &eps->dnodeEps[i];
+    dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
+  }
+}
+
+static int32_t dnodeReadEps() {
+  int32_t    ret = -1;
+  int32_t    len = 0;
+  int32_t    maxLen = 30000;
+  char *     content = calloc(1, maxLen + 1);
+  cJSON *    root = NULL;
+  FILE *     fp = NULL;
+  SDnodeEps *eps = NULL;
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
+  if (!dnodeNum || dnodeNum->type != cJSON_Number) {
+    dError("failed to read %s, dnodeNum not found", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
+  if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
+    dError("failed to read %s, dnodeInfos not found", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
+  if (dnodeInfosSize != dnodeNum->valueint) {
+    dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize,
+           (int32_t)dnodeNum->valueint);
+    goto PRASE_EPS_OVER;
+  }
+
+  int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp);
+  eps = calloc(1, epsSize);
+  eps->dnodeNum = dnodeInfosSize;
+
+  for (int32_t i = 0; i < dnodeInfosSize; ++i) {
+    cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
+    if (dnodeInfo == NULL) break;
+
+    SDnodeEp *ep = &eps->dnodeEps[i];
+
+    cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
+    if (!dnodeId || dnodeId->type != cJSON_Number) {
+      dError("failed to read %s, dnodeId not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    ep->dnodeId = dnodeId->valueint;
+
+    cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
+    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+      dError("failed to read %s, dnodeFqdn not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
+    if (!dnodePort || dnodePort->type != cJSON_Number) {
+      dError("failed to read %s, dnodePort not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    ep->dnodePort = (uint16_t)dnodePort->valueint;
+  }
+
+  ret = 0;
+
+  dInfo("read file %s successed", file);
+  dnodePrintEps(eps);
+
+PRASE_EPS_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  if (ret != 0) {
+    if (eps) free(eps);
+    eps = NULL;
+  }
+
+  dnodeResetEps(eps);
+  if (eps) free(eps);
+
+  dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+
+  terrno = 0;
+  return 0;
+}
+
+static int32_t dnodeWriteEps() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 30000;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", tsEps->dnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
+  for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+    SDnodeEp *ep = &tsEps->dnodeEps[i];
+    len += snprintf(content + len, maxLen - len, "    \"dnodeId\": %d,\n", ep->dnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
+    len += snprintf(content + len, maxLen - len, "    \"dnodePort\": %u\n", ep->dnodePort);
+    if (i < tsEps->dnodeNum - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeMInfos.c
+++ b/src/dnode/src/dnodeMInfos.c
@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMInfos.h"
+
+static SMnodeInfos tsMInfos;
+static SRpcEpSet   tsMEpSet;
+static pthread_mutex_t tsMInfosMutex;
+
+static void    dnodeResetMInfos(SMnodeInfos *minfos);
+static void    dnodePrintMInfos(SMnodeInfos *minfos);
+static int32_t dnodeReadMInfos();
+static int32_t dnodeWriteMInfos();
+
+int32_t dnodeInitMInfos() {
+  pthread_mutex_init(&tsMInfosMutex, NULL);
+  dnodeResetMInfos(NULL);
+  int32_t ret = dnodeReadMInfos();
+  if (ret == 0) {
+    dInfo("dnode minfos is initialized");
+  }
+
+  return ret;
+}
+
+void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
+
+void dnodeUpdateMInfos(SMnodeInfos *minfos) {
+  if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
+    dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+    return;
+  }
+
+  for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
+    SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+    minfo->mnodeId = htonl(minfo->mnodeId);
+    if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
+      dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
+      return;
+    }
+  }
+
+  pthread_mutex_lock(&tsMInfosMutex);
+  if (minfos->mnodeNum != tsMInfos.mnodeNum) {
+    dnodeResetMInfos(minfos);
+    dnodeWriteMInfos();
+    sdbUpdateAsync();
+  } else {
+    int32_t size = sizeof(SMnodeInfos);
+    if (memcmp(minfos, &tsMInfos, size) != 0) {
+      dnodeResetMInfos(minfos);
+      dnodeWriteMInfos();
+      sdbUpdateAsync();
+    }
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
+  if (ep->numOfEps <= 0) {
+    dError("minfos is changed, but content is invalid, discard it");
+    return;
+  }
+
+  pthread_mutex_lock(&tsMInfosMutex);
+  dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
+  for (int i = 0; i < ep->numOfEps; ++i) {
+    ep->port[i] -= TSDB_PORT_DNODEDNODE;
+    dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
+  }
+  tsMEpSet = *ep;
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+bool dnodeIsMasterEp(char *ep) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0;
+  pthread_mutex_unlock(&tsMInfosMutex);
+
+  return isMaster;
+}
+
+void dnodeGetMInfos(SMnodeInfos *minfos) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
+    minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  *epSet = tsMEpSet;
+  for (int i = 0; i < epSet->numOfEps; ++i) {
+    epSet->port[i] += TSDB_PORT_DNODEDNODE;
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  *epSet = tsMEpSet;
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+static void dnodePrintMInfos(SMnodeInfos *minfos) {
+  dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
+  for (int32_t i = 0; i < minfos->mnodeNum; i++) {
+    dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+  }
+}
+
+static void dnodeResetMInfos(SMnodeInfos *minfos) {
+  if (minfos == NULL) {
+    tsMEpSet.numOfEps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
+
+    if (strcmp(tsSecond, tsFirst) != 0) {
+      tsMEpSet.numOfEps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]);
+    }
+    return;
+  }
+
+  if (minfos->mnodeNum == 0) return;
+
+  int32_t size = sizeof(SMnodeInfos);
+  memcpy(&tsMInfos, minfos, size);
+
+  tsMEpSet.inUse = tsMInfos.inUse;
+  tsMEpSet.numOfEps = tsMInfos.mnodeNum;
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+    taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
+  }
+
+  dnodePrintMInfos(minfos);
+}
+
+static int32_t dnodeReadMInfos() {
+  int32_t     len = 0;
+  int32_t     maxLen = 2000;
+  char *      content = calloc(1, maxLen + 1);
+  cJSON *     root = NULL;
+  FILE *      fp = NULL;
+  SMnodeInfos minfos = {0};
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  cJSON *inUse = cJSON_GetObjectItem(root, "inUse");
+  if (!inUse || inUse->type != cJSON_Number) {
+    dError("failed to read mnodeEpSet.json, inUse not found");
+    goto PARSE_MINFOS_OVER;
+  }
+  tsMInfos.inUse = inUse->valueint;
+
+  cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
+  if (!nodeNum || nodeNum->type != cJSON_Number) {
+    dError("failed to read mnodeEpSet.json, nodeNum not found");
+    goto PARSE_MINFOS_OVER;
+  }
+  minfos.mnodeNum = nodeNum->valueint;
+
+  cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+    dError("failed to read mnodeEpSet.json, nodeInfos not found");
+    goto PARSE_MINFOS_OVER;
+  }
+
+  int size = cJSON_GetArraySize(nodeInfos);
+  if (size != minfos.mnodeNum) {
+    dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
+    goto PARSE_MINFOS_OVER;
+  }
+
+  for (int i = 0; i < size; ++i) {
+    cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+    if (nodeInfo == NULL) continue;
+
+    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+    if (!nodeId || nodeId->type != cJSON_Number) {
+      dError("failed to read mnodeEpSet.json, nodeId not found");
+      goto PARSE_MINFOS_OVER;
+    }
+    minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
+
+    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+      dError("failed to read mnodeEpSet.json, nodeName not found");
+      goto PARSE_MINFOS_OVER;
+    }
+    strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+  }
+
+  dInfo("read file %s successed", file);
+  dnodePrintMInfos(&minfos);
+
+PARSE_MINFOS_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  terrno = 0;
+
+  for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
+    SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+    dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
+  }
+  dnodeResetMInfos(&minfos);
+  return 0;
+}
+
+static int32_t dnodeWriteMInfos() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 2000;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsMInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsMInfos.mnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp);
+    if (i < tsMInfos.mnodeNum - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeMPeer.c
+++ b/src/dnode/src/dnodeMPeer.c
@ -35,44 +35,44 @@ typedef struct {
 typedef struct {
  int32_t curNum;
  int32_t maxNum;
-  SMPeerWorker *peerWorker;
+  SMPeerWorker *worker;
 } SMPeerWorkerPool;

-static SMPeerWorkerPool tsMPeerPool;
+static SMPeerWorkerPool tsMPeerWP;
 static taos_qset        tsMPeerQset;
 static taos_queue       tsMPeerQueue;

-static void *dnodeProcessMnodePeerQueue(void *param);
+static void *dnodeProcessMPeerQueue(void *param);

-int32_t dnodeInitMnodePeer() {
+int32_t dnodeInitMPeer() {
  tsMPeerQset = taosOpenQset();
  
-  tsMPeerPool.maxNum = 1;
-  tsMPeerPool.curNum = 0;
-  tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
+  tsMPeerWP.maxNum = 1;
+  tsMPeerWP.curNum = 0;
+  tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum);

-  if (tsMPeerPool.peerWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  if (tsMPeerWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
    pWorker->workerId = i;
    dDebug("dnode mpeer worker:%d is created", i);
  }

-  dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
+  dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset);
  return 0;
 }

-void dnodeCleanupMnodePeer() {
-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+void dnodeCleanupMPeer() {
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
    if (pWorker->thread) {
      taosQsetThreadResume(tsMPeerQset);
    }
    dDebug("dnode mpeer worker:%d is closed", i);
  }

-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
    dDebug("dnode mpeer worker:%d start to join", i);
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() {

  taosCloseQset(tsMPeerQset);
  tsMPeerQset = NULL;
-  taosTFree(tsMPeerPool.peerWorker);
+  tfree(tsMPeerWP.worker);
 }

-int32_t dnodeAllocateMnodePqueue() {
+int32_t dnodeAllocateMPeerQueue() {
  tsMPeerQueue = taosOpenQueue();
  if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;

  taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);

-  for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
    pWorker->workerId = i;

    pthread_attr_t thAttr;
    pthread_attr_init(&thAttr);
    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) {
      dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
    }

    pthread_attr_destroy(&thAttr);

-    tsMPeerPool.curNum = i + 1;
-    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
+    tsMPeerWP.curNum = i + 1;
+    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum);
  }

  dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
  return TSDB_CODE_SUCCESS;
 }

-void dnodeFreeMnodePqueue() {
+void dnodeFreeMPeerQueue() {
  dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
  taosCloseQueue(tsMPeerQueue);
  tsMPeerQueue = NULL;
 }

-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) {
  if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
    dnodeSendRedirectMsg(pMsg, false);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pPeer = mnodeCreateMsg(pMsg);
+    taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
  }

-  SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pPeer, pMsg);
-  taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
+  rpcFreeCont(pMsg->pCont);
 }

-static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
+static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) {
  mnodeCleanupMsg(pPeer);
  taosFreeQitem(pPeer);
 }

-static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
+static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) {
  if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;

  SRpcMsg rpcRsp = {
@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
  };

  rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodePeerMsg(pPeer);
+  dnodeFreeMPeerMsg(pPeer);
 }

-static void *dnodeProcessMnodePeerQueue(void *param) {
+static void *dnodeProcessMPeerQueue(void *param) {
  SMnodeMsg *pPeerMsg;
  int32_t    type;
  void *     unUsed;
@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {

    dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);    
    int32_t code = mnodeProcessPeerReq(pPeerMsg);    
-    dnodeSendRpcMnodePeerRsp(pPeerMsg, code);    
+    dnodeSendRpcMPeerRsp(pPeerMsg, code);    
  }

  return NULL;
--- a/src/dnode/src/dnodeMRead.c
+++ b/src/dnode/src/dnodeMRead.c
@ -35,46 +35,46 @@ typedef struct {
 typedef struct {
  int32_t curNum;
  int32_t maxNum;
-  SMReadWorker *readWorker;
+  SMReadWorker *worker;
 } SMReadWorkerPool;

-static SMReadWorkerPool tsMReadPool;
+static SMReadWorkerPool tsMReadWP;
 static taos_qset        tsMReadQset;
 static taos_queue       tsMReadQueue;

-static void *dnodeProcessMnodeReadQueue(void *param);
+static void *dnodeProcessMReadQueue(void *param);

-int32_t dnodeInitMnodeRead() {
+int32_t dnodeInitMRead() {
  tsMReadQset = taosOpenQset();

-  tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
-  tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
-  tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
-  tsMReadPool.curNum = 0;
-  tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
+  tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+  tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
+  tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
+  tsMReadWP.curNum = 0;
+  tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum);

-  if (tsMReadPool.readWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  if (tsMReadWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
    pWorker->workerId = i;
    dDebug("dnode mread worker:%d is created", i);
  }

-  dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
+  dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset);
  return 0;
 }

-void dnodeCleanupMnodeRead() {
-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+void dnodeCleanupMRead() {
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
    if (pWorker->thread) {
      taosQsetThreadResume(tsMReadQset);
    }
    dDebug("dnode mread worker:%d is closed", i);
  }

-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
    dDebug("dnode mread worker:%d start to join", i);
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() {

  taosCloseQset(tsMReadQset);
  tsMReadQset = NULL;
-  free(tsMReadPool.readWorker);
+  free(tsMReadWP.worker);
 }

-int32_t dnodeAllocateMnodeRqueue() {
+int32_t dnodeAllocMReadQueue() {
  tsMReadQueue = taosOpenQueue();
  if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;

  taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);

-  for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
    pWorker->workerId = i;

    pthread_attr_t thAttr;
    pthread_attr_init(&thAttr);
    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) {
      dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
    }

    pthread_attr_destroy(&thAttr);
-    tsMReadPool.curNum = i + 1;
-    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
+    tsMReadWP.curNum = i + 1;
+    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum);
  }

  dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
  return TSDB_CODE_SUCCESS;
 }

-void dnodeFreeMnodeRqueue() {
+void dnodeFreeMReadQueue() {
  dDebug("dnode mread queue:%p is freed", tsMReadQueue);
  taosCloseQueue(tsMReadQueue);
  tsMReadQueue = NULL;
 }

-void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
  if (!mnodeIsRunning() || tsMReadQueue == NULL) {
    dnodeSendRedirectMsg(pMsg, true);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
+    taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
  }

-  SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pRead, pMsg);
-  taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
+  rpcFreeCont(pMsg->pCont);
 }

-static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
+static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {
  mnodeCleanupMsg(pRead);
  taosFreeQitem(pRead);
 }

-static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
+static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) {
  if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
  if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
    // may be a auto create req, should put into write queue
-    dnodeReprocessMnodeWriteMsg(pRead);
+    dnodeReprocessMWriteMsg(pRead);
    return;
  }

@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
  };

  rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodeReadMsg(pRead);
+  dnodeFreeMReadMsg(pRead);
 }

-static void *dnodeProcessMnodeReadQueue(void *param) {
-  SMnodeMsg *pReadMsg;
+static void *dnodeProcessMReadQueue(void *param) {
+  SMnodeMsg *pRead;
  int32_t    type;
  void *     unUsed;
-  
+
  while (1) {
-    if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
+    if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) {
      dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
      break;
    }

-    dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);    
-    int32_t code = mnodeProcessRead(pReadMsg);    
-    dnodeSendRpcMnodeReadRsp(pReadMsg, code);   
+    dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
+    int32_t code = mnodeProcessRead(pRead);
+    dnodeSendRpcMReadRsp(pRead, code);
  }

  return NULL;
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@ -36,45 +36,45 @@ typedef struct {
 typedef struct {
  int32_t curNum;
  int32_t maxNum;
-  SMWriteWorker *writeWorker;
+  SMWriteWorker *worker;
 } SMWriteWorkerPool;

-static SMWriteWorkerPool tsMWritePool;
+static SMWriteWorkerPool tsMWriteWP;
 static taos_qset         tsMWriteQset;
 static taos_queue        tsMWriteQueue;
 extern void *            tsDnodeTmr;

-static void *dnodeProcessMnodeWriteQueue(void *param);
+static void *dnodeProcessMWriteQueue(void *param);

-int32_t dnodeInitMnodeWrite() {
+int32_t dnodeInitMWrite() {
  tsMWriteQset = taosOpenQset();

-  tsMWritePool.maxNum = 1;
-  tsMWritePool.curNum = 0;
-  tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
+  tsMWriteWP.maxNum = 1;
+  tsMWriteWP.curNum = 0;
+  tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum);

-  if (tsMWritePool.writeWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  if (tsMWriteWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
    pWorker->workerId = i;
    dDebug("dnode mwrite worker:%d is created", i);
  }

-  dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
+  dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset);
  return 0;
 }

-void dnodeCleanupMnodeWrite() {
-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+void dnodeCleanupMWrite() {
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
    if (pWorker->thread) {
      taosQsetThreadResume(tsMWriteQset);
    }
    dDebug("dnode mwrite worker:%d is closed", i);
  }

-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
    dDebug("dnode mwrite worker:%d start to join", i);
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() {

  taosCloseQset(tsMWriteQset);
  tsMWriteQset = NULL;
-  taosTFree(tsMWritePool.writeWorker);
+  tfree(tsMWriteWP.worker);
 }

-int32_t dnodeAllocateMnodeWqueue() {
+int32_t dnodeAllocMWritequeue() {
  tsMWriteQueue = taosOpenQueue();
  if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;

  taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);

-  for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
    pWorker->workerId = i;

    pthread_attr_t thAttr;
    pthread_attr_init(&thAttr);
    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) {
      dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
    }

    pthread_attr_destroy(&thAttr);
-    tsMWritePool.curNum = i + 1;
-    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
+    tsMWriteWP.curNum = i + 1;
+    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum);
  }

  dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
  return TSDB_CODE_SUCCESS;
 }

-void dnodeFreeMnodeWqueue() {
+void dnodeFreeMWritequeue() {
  dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
  taosCloseQueue(tsMWriteQueue);
  tsMWriteQueue = NULL;
 }

-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
  if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
    dnodeSendRedirectMsg(pMsg, true);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
+    dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+           taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
+    taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
  }

-  SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pWrite, pMsg);
-
-  dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
-         taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
-  taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+  rpcFreeCont(pMsg->pCont);
 }

-static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
  dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
         taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);

@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
  taosFreeQitem(pWrite);
 }

-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) {
  SMnodeMsg *pWrite = pMsg;
  if (pWrite == NULL) return;
  if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
  if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
-    dnodeReprocessMnodeWriteMsg(pWrite);
+    dnodeReprocessMWriteMsg(pWrite);
    return;
  }

@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
  };

  rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodeWriteMsg(pWrite);
+  dnodeFreeMWriteMsg(pWrite);
 }

-static void *dnodeProcessMnodeWriteQueue(void *param) {
+static void *dnodeProcessMWriteQueue(void *param) {
  SMnodeMsg *pWrite;
  int32_t    type;
  void *     unUsed;
@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
           taosMsg[pWrite->rpcMsg.msgType]);

    int32_t code = mnodeProcessWrite(pWrite);
-    dnodeSendRpcMnodeWriteRsp(pWrite, code);
+    dnodeSendRpcMWriteRsp(pWrite, code);
  }

  return NULL;
 }

-void dnodeReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeReprocessMWriteMsg(void *pMsg) {
  SMnodeMsg *pWrite = pMsg;

  if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
           taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);

    dnodeSendRedirectMsg(pMsg, true);
-    dnodeFreeMnodeWriteMsg(pWrite);
+    dnodeFreeMWriteMsg(pWrite);
  } else {
    dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
           taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
  }
 }

-static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
-  dnodeReprocessMnodeWriteMsg(param);
+static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) {
+  dnodeReprocessMWriteMsg(param);
 }

-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeDelayReprocessMWriteMsg(void *pMsg) {
  SMnodeMsg *mnodeMsg = pMsg;
  void *unUsed = NULL;
-  taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
+  taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
 }
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@ -19,11 +19,16 @@
 #include "tutil.h"
 #include "tconfig.h"
 #include "tglobal.h"
+#include "twal.h"
+#include "trpc.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
 #include "dnodePeer.h"
 #include "dnodeModule.h"
+#include "dnodeEps.h"
+#include "dnodeMInfos.h"
+#include "dnodeCfg.h"
 #include "dnodeCheck.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
@ -35,29 +40,36 @@
 #include "tpath.h"
 #include "tdisk.h"

+static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
+
 static int32_t dnodeInitStorage();
-static void dnodeCleanupStorage();
-static void dnodeSetRunStatus(SDnodeRunStatus status);
-static void dnodeCheckDataDirOpenned(char *dir);
-static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+static void    dnodeCleanupStorage();
+static void    dnodeSetRunStatus(SRunStatus status);
+static void    dnodeCheckDataDirOpenned(char *dir);
 static int32_t dnodeInitComponents();
-static void dnodeCleanupComponents(int32_t stepId);
-static int dnodeCreateDir(const char *dir);
+static void    dnodeCleanupComponents(int32_t stepId);
+static int     dnodeCreateDir(const char *dir);

 typedef struct {
  const char *const name;
-  int               (*init)();
-  void              (*cleanup)();
+  int32_t (*init)();
+  void (*cleanup)();
 } SDnodeComponent;

 static const SDnodeComponent tsDnodeComponents[] = {
+  {"rpc",       rpcInit,             rpcCleanup},
  {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
+  {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
+  {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
+  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
+  {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
+  {"wal",       walInit,             walCleanUp},
  {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
-  {"vread",     dnodeInitVnodeRead,  dnodeCleanupVnodeRead},
-  {"vwrite",    dnodeInitVnodeWrite, dnodeCleanupVnodeWrite},
-  {"mread",     dnodeInitMnodeRead,  dnodeCleanupMnodeRead},
-  {"mwrite",    dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
-  {"mpeer",     dnodeInitMnodePeer,  dnodeCleanupMnodePeer},  
+  {"vread",     dnodeInitVRead,      dnodeCleanupVRead},
+  {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
+  {"mread",     dnodeInitMRead,      dnodeCleanupMRead},
+  {"mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
+  {"mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
  {"client",    dnodeInitClient,     dnodeCleanupClient},
  {"server",    dnodeInitServer,     dnodeCleanupServer},
  {"mgmt",      dnodeInitMgmt,       dnodeCleanupMgmt},
@ -77,7 +89,9 @@ static int dnodeCreateDir(const char *dir) {

 static void dnodeCleanupComponents(int32_t stepId) {
  for (int32_t i = stepId; i >= 0; i--) {
-    tsDnodeComponents[i].cleanup();
+    if (tsDnodeComponents[i].cleanup) {
+      (*tsDnodeComponents[i].cleanup)();
+    }
  }
 }

@ -94,7 +108,7 @@ static int32_t dnodeInitComponents() {
 }

 int32_t dnodeInitSystem() {
-  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
+  dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
  tscEmbedded  = 1;
  taosBlockSIGPIPE();
  taosResolveCRC();
@ -114,21 +128,20 @@ int32_t dnodeInitSystem() {
    printf("failed to init log file\n");
  }

-  if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) {
+  if (!taosReadGlobalCfg()) {
    taosPrintGlobalCfg();
    dError("TDengine read global config failed");
    return -1;
  }
-  taosPrintGlobalCfg();

-  dInfo("start to initialize TDengine on %s", tsLocalEp);
+  dInfo("start to initialize TDengine");

  if (dnodeInitComponents() != 0) {
    return -1;
  }

  dnodeStartModules();
-  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
+  dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);

  dInfo("TDengine is initialized successfully");

@ -136,20 +149,20 @@ int32_t dnodeInitSystem() {
 }

 void dnodeCleanUpSystem() {
-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
-    dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
+    dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
    dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
    taos_cleanup();
    taosCloseLog();
  }
 }

-SDnodeRunStatus dnodeGetRunStatus() {
-  return tsDnodeRunStatus;
+SRunStatus dnodeGetRunStatus() {
+  return tsRunStatus;
 }

-static void dnodeSetRunStatus(SDnodeRunStatus status) {
-  tsDnodeRunStatus = status;
+static void dnodeSetRunStatus(SRunStatus status) {
+  tsRunStatus = status;
 }

 static void dnodeCheckDataDirOpenned(char *dir) {
@ -219,7 +232,7 @@ static int32_t dnodeInitStorage() {

  dnodeCheckDataDirOpenned(tsDnodeDir);

-  dInfo("storage directory is initialized");
+  dInfo("dnode storage is initialized at %s", tsDnodeDir);
  return 0;
 }

--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@ -31,12 +31,13 @@
 #include "mnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
 #include "dnodeModule.h"

-#define MPEER_CONTENT_LEN 2000
-
 typedef struct {
  pthread_t thread;
  int32_t   threadIndex;
@ -46,23 +47,18 @@ typedef struct {
  int32_t * vnodeList;
 } SOpenVnodeThread;

-void *          tsDnodeTmr = NULL;
-static void *   tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
+typedef struct {
+  SRpcMsg rpcMsg;
+  char    pCont[];
+} SMgmtMsg;

-static SRpcEpSet     tsDMnodeEpSet = {0};
-static SDMMnodeInfos tsDMnodeInfos = {0};
-static SDMDnodeCfg   tsDnodeCfg = {0};
-static taos_qset     tsMgmtQset = NULL;
-static taos_queue    tsMgmtQueue = NULL;
-static pthread_t     tsQthread;
+void *            tsDnodeTmr = NULL;
+static void *     tsStatusTimer = NULL;
+static uint32_t   tsRebootTime;
+static taos_qset  tsMgmtQset = NULL;
+static taos_queue tsMgmtQueue = NULL;
+static pthread_t  tsQthread;

-static void   dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
-static bool   dnodeReadMnodeInfos();
-static void   dnodeSaveMnodeInfos();
-static void   dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
-static bool   dnodeReadDnodeCfg();
-static void   dnodeSaveDnodeCfg();
 static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
 static void   dnodeSendStatusMsg(void *handle, void *tmrId);
 static void  *dnodeProcessMgmtQueue(void *param);
@ -74,7 +70,7 @@ static int32_t  dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
+static int32_t  dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
 static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);

 int32_t dnodeInitMgmt() {
@ -86,28 +82,8 @@ int32_t dnodeInitMgmt() {
  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;

  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
-  dnodeReadDnodeCfg();
  tsRebootTime = taosGetTimestampSec();

-  if (!dnodeReadMnodeInfos()) {
-    memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet));
-    memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
-
-    tsDMnodeEpSet.numOfEps = 1;
-    taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]);
-    
-    if (strcmp(tsSecond, tsFirst) != 0) {
-      tsDMnodeEpSet.numOfEps = 2;
-      taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]);
-    }
-  } else {
-    tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
-    tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
-    for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-      taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
-    }
-  }
-
  int32_t code = vnodeInitResources();
  if (code != TSDB_CODE_SUCCESS) {
    dnodeCleanupMgmt();
@ -201,38 +177,46 @@ void dnodeCleanupMgmt() {
  vnodeCleanupResources();
 }

-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
-  void *item;
-
-  item = taosAllocateQitem(sizeof(SRpcMsg));
-  if (item) {
-    memcpy(item, pMsg, sizeof(SRpcMsg));
-    taosWriteQitem(tsMgmtQueue, 1, item);
-  } else {
-    SRpcMsg rsp = {
-      .handle = pMsg->handle,
-      .pCont  = NULL,
-      .code   = TSDB_CODE_DND_OUT_OF_MEMORY
-    };
-    
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pMsg->pCont);
+static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
+  SMgmtMsg *pMgmt = taosAllocateQitem(size);
+  if (pMgmt == NULL) {
+    return TSDB_CODE_DND_OUT_OF_MEMORY;
  }
+
+  pMgmt->rpcMsg = *pMsg;
+  pMgmt->rpcMsg.pCont = pMgmt->pCont;
+  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+  taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t code = dnodeWriteToMgmtQueue(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
 }

 static void *dnodeProcessMgmtQueue(void *param) {
-  SRpcMsg *pMsg;
-  SRpcMsg  rsp = {0};
-  int      type;
-  void *   handle;
+  SMgmtMsg *pMgmt;
+  SRpcMsg * pMsg;
+  SRpcMsg   rsp = {0};
+  int32_t   qtype;
+  void *    handle;

  while (1) {
-    if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
+    if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
      dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
      break;
    }

-    dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]);    
+    pMsg = &pMgmt->rpcMsg;
+    dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
    if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
      rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
    } else {
@ -240,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) {
    }

    rsp.handle = pMsg->handle;
-    rsp.pCont  = NULL;
+    rsp.pCont = NULL;
    rpcSendResponse(&rsp);

-    rpcFreeCont(pMsg->pCont);
    taosFreeQitem(pMsg);
  }

@ -381,7 +364,7 @@ static void dnodeCloseVnodes() {
 }

 static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
  pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
@ -404,7 +387,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
+  SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);

  void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
  if (pVnode != NULL) {
@ -418,7 +401,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
+  SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);

  void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
  if (pVnode != NULL) {
@ -433,14 +416,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDDropVnodeMsg *pDrop = rpcMsg->pCont;
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
  pDrop->vgId = htonl(pDrop->vgId);

  return vnodeDrop(pDrop->vgId);
 }

 static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-//  SMDAlterStreamMsg *pStream = pCont;
+//  SAlterStreamMsg *pStream = pCont;
 //  pStream->uid    = htobe64(pStream->uid);
 //  pStream->stime  = htobe64(pStream->stime);
 //  pStream->vnode  = htonl(pStream->vnode);
@ -453,12 +436,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
 }

 static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
-  SMDCfgDnodeMsg *pCfg = pMsg->pCont;
+  SCfgDnodeMsg *pCfg = pMsg->pCont;
  return taosCfgDynamicOptions(pCfg->config);
 }

 static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
-  SMDCreateMnodeMsg *pCfg = pMsg->pCont;
+  SCreateMnodeMsg *pCfg = pMsg->pCont;
  pCfg->dnodeId = htonl(pCfg->dnodeId);
  if (pCfg->dnodeId != dnodeGetDnodeId()) {
    dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
@ -470,10 +453,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
  }

-  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum);
-  for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) {
-    pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId);
-    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp);
+  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
+  for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
+    pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
+    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
  }

  dnodeStartMnode(&pCfg->mnodes);
@ -481,34 +464,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
  return TSDB_CODE_SUCCESS;
 }

-void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
-  if (pEpSet->numOfEps <= 0) {
-    dError("mnode EP list for peer is changed, but content is invalid, discard it");
-    return;
-  }
-
-  dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse);
-  for (int i = 0; i < pEpSet->numOfEps; ++i) {
-    pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
-    dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
-  }
-
-  tsDMnodeEpSet = *pEpSet;
-}
-
-void dnodeGetMnodeEpSetForPeer(void *epSetRaw) {
-  SRpcEpSet *epSet = epSetRaw;
-  *epSet = tsDMnodeEpSet;
-
-  for (int i=0; i<epSet->numOfEps; ++i) 
-    epSet->port[i] += TSDB_PORT_DNODEDNODE;
-}
-
-void dnodeGetMnodeEpSetForShell(void *epSetRaw) {
-  SRpcEpSet *epSet = epSetRaw;
-  *epSet = tsDMnodeEpSet;
-}
-
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
  if (pMsg->code != TSDB_CODE_SUCCESS) {
    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
@ -516,202 +471,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
    return;
  }

-  SDMStatusRsp *pStatusRsp = pMsg->pCont;
-  SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
-  if (pMnodes->nodeNum <= 0) {
-    dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
+  SStatusRsp *pStatusRsp = pMsg->pCont;
+  SMnodeInfos *minfos = &pStatusRsp->mnodes;
+  dnodeUpdateMInfos(minfos);

-  SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
+  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
  pCfg->dnodeId = htonl(pCfg->dnodeId);
-
-  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
-    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
-    pMnodeInfo->nodeId   = htonl(pMnodeInfo->nodeId);
-  }
+  dnodeUpdateCfg(pCfg);

  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);

-  // will not set mnode in status msg
-  // dnodeProcessModuleStatus(pCfg->moduleStatus);
-  dnodeUpdateDnodeCfg(pCfg);
+  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+  dnodeUpdateEps(pEps);

-  dnodeUpdateMnodeInfos(pMnodes);
  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
 }

-static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) {
-  if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) {
-    dError("invalid mnode infos, num:%d", pMnodes->nodeNum);
-    return false;
-  }
-
-  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
-    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
-    if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) {
-      dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp);
-      return false;
-    }
-  }
-
-  return true;
-}
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
-  bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
-  bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
-  if (!(mnodesChanged || mnodesNotInit)) return;
-
-  if (!dnodeCheckMnodeInfos(pMnodes)) return;
-
-  memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
-  dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-  tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
-  tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
-  }
-
-  dnodeSaveMnodeInfos();
-  sdbUpdateAsync();
-}
-
-static bool dnodeReadMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN*2] = {0};
-  
-  sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "r");
-  if (!fp) {
-    dDebug("failed to read mnodeEpSet.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 2000;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read mnodeEpSet.json, content is null");
-    return false;
-  }
-
-  content[len] = 0;
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read mnodeEpSet.json, invalid json format");
-    goto PARSE_OVER;
-  }
-
-  cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
-  if (!inUse || inUse->type != cJSON_Number) {
-    dError("failed to read mnodeEpSet.json, inUse not found");
-    goto PARSE_OVER;
-  }
-  tsDMnodeInfos.inUse = inUse->valueint;
-
-  cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
-  if (!nodeNum || nodeNum->type != cJSON_Number) {
-    dError("failed to read mnodeEpSet.json, nodeNum not found");
-    goto PARSE_OVER;
-  }
-  tsDMnodeInfos.nodeNum = nodeNum->valueint;
-
-  cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
-  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
-    dError("failed to read mnodeEpSet.json, nodeInfos not found");
-    goto PARSE_OVER;
-  }
-
-  int size = cJSON_GetArraySize(nodeInfos);
-  if (size != tsDMnodeInfos.nodeNum) {
-    dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
-    goto PARSE_OVER;
-  }
-
-  for (int i = 0; i < size; ++i) {
-    cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
-    if (nodeInfo == NULL) continue;
-
-    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
-    if (!nodeId || nodeId->type != cJSON_Number) {
-      dError("failed to read mnodeEpSet.json, nodeId not found");
-      goto PARSE_OVER;
-    }
-    tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
-
-    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
-    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
-      dError("failed to read mnodeEpSet.json, nodeName not found");
-      goto PARSE_OVER;
-    }
-    strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
-  }
-
-  ret = true;
-
-  dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-PARSE_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 2000;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsDMnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum);
-  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp);
-    if (i < tsDMnodeInfos.nodeNum -1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");  
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");  
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n"); 
-
-  fwrite(content, 1, len, fp);
-  fflush(fp);
-  fclose(fp);
-  free(content);
-  
-  dInfo("save mnode epSet successed");
-}
-
-char *dnodeGetMnodeMasterEp() {
-  return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp;
-}
-
-void* dnodeGetMnodeInfos() {
-  return &tsDMnodeInfos;
-}
-
 static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  if (tsDnodeTmr == NULL) {
    dError("dnode timer is already released");
@ -724,22 +501,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
    return;
  }

-  int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SDMStatusMsg *pStatus = rpcMallocCont(contLen);
+  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SStatusMsg *pStatus = rpcMallocCont(contLen);
  if (pStatus == NULL) {
    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
    dError("failed to malloc status message");
    return;
  }

-  //strcpy(pStatus->dnodeName, tsDnodeName);
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
  pStatus->version          = htonl(tsVersion);
-  pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
  pStatus->lastReboot       = htonl(tsRebootTime);
  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
  pStatus->diskAvailable    = tsAvailDataDirGB;
  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN);
  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);

  // fill cluster cfg parameters
@ -759,7 +535,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
  
  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
  pStatus->openVnodes = htons(pStatus->openVnodes);
  
  SRpcMsg rpcMsg = {
@ -769,110 +545,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  };

  SRpcEpSet epSet;
-  dnodeGetMnodeEpSetForPeer(&epSet);
+  dnodeGetEpSetForPeer(&epSet);
  dnodeSendMsgToDnode(&epSet, &rpcMsg);
 }

-static bool dnodeReadDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0};
-  
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "r");
-  if (!fp) {
-    dDebug("failed to read dnodeCfg.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 100;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read dnodeCfg.json, content is null");
-    return false;
-  }
-
-  content[len] = 0;
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read dnodeCfg.json, invalid json format");
-    goto PARSE_CFG_OVER;
-  }
-
-  cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read dnodeCfg.json, dnodeId not found");
-    goto PARSE_CFG_OVER;
-  }
-  tsDnodeCfg.dnodeId = dnodeId->valueint;
-
-  cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read dnodeCfg.json, clusterId not found");
-    goto PARSE_CFG_OVER;
-  }
-  tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
-
-  ret = true;
-
-  dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
-
-PARSE_CFG_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 200;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  fwrite(content, 1, len, fp);
-  fflush(fp);
-  fclose(fp);
-  free(content);
-  
-  dInfo("save dnodeId successed");
-}
-
-void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
-  if (tsDnodeCfg.dnodeId == 0) {
-    dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId);  
-    tsDnodeCfg.dnodeId = pCfg->dnodeId;
-    tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN);
-    dnodeSaveDnodeCfg();
-  }
-}
-
-int32_t dnodeGetDnodeId() {
-  return tsDnodeCfg.dnodeId;
-}
-
 void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
  SRpcConnInfo connInfo = {0};
  rpcGetConnInfo(rpcMsg->handle, &connInfo);

  SRpcEpSet epSet = {0};
  if (forShell) {
-    dnodeGetMnodeEpSetForShell(&epSet);
+    dnodeGetEpSetForShell(&epSet);
  } else {
-    dnodeGetMnodeEpSetForPeer(&epSet);
+    dnodeGetEpSetForPeer(&epSet);
  }
  
  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@ -114,6 +114,7 @@ int32_t dnodeInitModules() {
    }
  }

+  dInfo("dnode modules is initialized");
  return 0;
 }

@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
  }
 }

-bool dnodeStartMnode(void *pMnodes) {
-  SDMMnodeInfos *mnodes = pMnodes;
+bool dnodeStartMnode(SMnodeInfos *minfos) {
+  SMnodeInfos *mnodes = minfos;

  if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
    dDebug("mnode module is already started, module status:%d", tsModuleStatus);
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@ -19,6 +19,7 @@
 * to dnode. All theses messages are handled from here
 */

+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
 #include "tglobal.h"
@ -28,20 +29,20 @@
 #include "dnodeMgmt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMPeer.h"
+#include "dnodeMInfos.h"

-extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet);
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
 static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
-static void *tsDnodeServerRpc = NULL;
-static void *tsDnodeClientRpc = NULL;
+static void *tsServerRpc = NULL;
+static void *tsClientRpc = NULL;

 int32_t dnodeInitServer() {
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeDispatchToVnodeWriteQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeDispatchToVWriteQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVWriteQueue;

  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; 
  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToMgmtQueue; 
@ -50,11 +51,11 @@ int32_t dnodeInitServer() {
  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;

-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMPeerQueue;
  
  SRpcInit rpcInit;
  memset(&rpcInit, 0, sizeof(rpcInit));
@ -66,20 +67,20 @@ int32_t dnodeInitServer() {
  rpcInit.connType     = TAOS_CONN_SERVER;
  rpcInit.idleTime     = tsShellActivityTimer * 1000;

-  tsDnodeServerRpc = rpcOpen(&rpcInit);
-  if (tsDnodeServerRpc == NULL) {
+  tsServerRpc = rpcOpen(&rpcInit);
+  if (tsServerRpc == NULL) {
    dError("failed to init inter-dnodes RPC server");
    return -1;
  }

-  dInfo("inter-dnodes RPC server is opened");
+  dInfo("dnode inter-dnodes RPC server is initialized");
  return 0;
 }

 void dnodeCleanupServer() {
-  if (tsDnodeServerRpc) {
-    rpcClose(tsDnodeServerRpc);
-    tsDnodeServerRpc = NULL;
+  if (tsServerRpc) {
+    rpcClose(tsServerRpc);
+    tsServerRpc = NULL;
    dInfo("inter-dnodes RPC server is closed");
  }
 }
@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
  
  if (pMsg->pCont == NULL) return;

-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
    rspMsg.code = TSDB_CODE_APP_NOT_READY;
    rpcSendResponse(&rspMsg);
    rpcFreeCont(pMsg->pCont);
@ -131,27 +132,34 @@ int32_t dnodeInitClient() {
  rpcInit.ckey         = "key";
  rpcInit.secret       = secret;

-  tsDnodeClientRpc = rpcOpen(&rpcInit);
-  if (tsDnodeClientRpc == NULL) {
+  tsClientRpc = rpcOpen(&rpcInit);
+  if (tsClientRpc == NULL) {
    dError("failed to init mnode rpc client");
    return -1;
  }

-  dInfo("inter-dnodes rpc client is opened");
+  dInfo("dnode inter-dnodes rpc client is initialized");
  return 0;
 }

 void dnodeCleanupClient() {
-  if (tsDnodeClientRpc) {
-    rpcClose(tsDnodeClientRpc);
-    tsDnodeClientRpc = NULL;
-    dInfo("inter-dnodes rpc client is closed");
+  if (tsClientRpc) {
+    rpcClose(tsClientRpc);
+    tsClientRpc = NULL;
+    dInfo("dnode inter-dnodes rpc client is closed");
  }
 }

 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
+    if (pMsg == NULL || pMsg->pCont == NULL) return;
+    dDebug("msg:%p is ignored since dnode not running", pMsg);
+    rpcFreeCont(pMsg->pCont);
+    return;
+  }
+
  if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
-    dnodeUpdateMnodeEpSetForPeer(pEpSet);
+    dnodeUpdateEpSetForPeer(pEpSet);
  }

  if (dnodeProcessRspMsgFp[pMsg->msgType]) {    
@ -168,15 +176,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
 }

 void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
-  rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
+  rpcSendRequest(tsClientRpc, epSet, rpcMsg, NULL);
 }

 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
  SRpcEpSet epSet = {0};
-  dnodeGetMnodeEpSetForPeer(&epSet);
-  rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
+  dnodeGetEpSetForPeer(&epSet);
+  rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
 }

 void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
-  rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
-}
+  rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
+}
--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@ -33,46 +33,46 @@
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
 static int     dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-static void  * tsDnodeShellRpc = NULL;
-static int32_t tsDnodeQueryReqNum  = 0;
-static int32_t tsDnodeSubmitReqNum = 0;
+static void  * tsShellRpc = NULL;
+static int32_t tsQueryReqNum  = 0;
+static int32_t tsSubmitReqNum = 0;

 int32_t dnodeInitShell() {
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]         = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]          = dnodeDispatchToVnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH]          = dnodeDispatchToVnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]         = dnodeDispatchToVWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]          = dnodeDispatchToVReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH]          = dnodeDispatchToVReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
  
  // the following message shall be treated as mnode write
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = dnodeDispatchToMnodeWriteQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = dnodeDispatchToMWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
  
  // the following message shall be treated as mnode query 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = dnodeDispatchToMnodeReadQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;   
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = dnodeDispatchToMReadQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;   
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMReadQueue;

  int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
  numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@ -91,24 +91,24 @@ int32_t dnodeInitShell() {
  rpcInit.idleTime     = tsShellActivityTimer * 1000;
  rpcInit.afp          = dnodeRetrieveUserAuthInfo;

-  tsDnodeShellRpc = rpcOpen(&rpcInit);
-  if (tsDnodeShellRpc == NULL) {
+  tsShellRpc = rpcOpen(&rpcInit);
+  if (tsShellRpc == NULL) {
    dError("failed to init shell rpc server");
    return -1;
  }

-  dInfo("shell rpc server is opened");
+  dInfo("dnode shell rpc server is initialized");
  return 0;
 }

 void dnodeCleanupShell() {
-  if (tsDnodeShellRpc) {
-    rpcClose(tsDnodeShellRpc);
-    tsDnodeShellRpc = NULL;
+  if (tsShellRpc) {
+    rpcClose(tsShellRpc);
+    tsShellRpc = NULL;
  }
 }

-void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
  SRpcMsg rpcMsg = {
    .handle  = pMsg->handle,
    .pCont   = NULL,
@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {

  if (pMsg->pCont == NULL) return;

-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
    dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
    rpcMsg.code = TSDB_CODE_APP_NOT_READY;
    rpcSendResponse(&rpcMsg);
@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
  }

  if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
-    atomic_fetch_add_32(&tsDnodeQueryReqNum, 1);
+    atomic_fetch_add_32(&tsQueryReqNum, 1);
  } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
-    atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1);
+    atomic_fetch_add_32(&tsSubmitReqNum, 1);
  } else {}

  if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
  int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
  if (code != TSDB_CODE_APP_NOT_READY) return code;

-  SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
+  SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
  tstrncpy(pMsg->user, user, sizeof(pMsg->user));

  SRpcMsg rpcMsg = {0};
  rpcMsg.pCont = pMsg;
-  rpcMsg.contLen = sizeof(SDMAuthMsg);
+  rpcMsg.contLen = sizeof(SAuthMsg);
  rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
  
  dDebug("user:%s, send auth msg to mnodes", user);
@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
  if (rpcRsp.code != 0) {
    dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
  } else {
-    SDMAuthRsp *pRsp = rpcRsp.pCont;
+    SAuthRsp *pRsp = rpcRsp.pCont;
    dDebug("user:%s, auth msg received from mnodes", user);
    memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
    memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
 void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
  dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid);

-  int32_t contLen = sizeof(SDMConfigTableMsg);
-  SDMConfigTableMsg *pMsg = rpcMallocCont(contLen);
+  int32_t contLen = sizeof(SConfigTableMsg);
+  SConfigTableMsg *pMsg = rpcMallocCont(contLen);

  pMsg->dnodeId = htonl(dnodeGetDnodeId());
  pMsg->vgId = htonl(vgId);
@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
  }
 }

-SDnodeStatisInfo dnodeGetStatisInfo() {
-  SDnodeStatisInfo info = {0};
-  if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
+SStatisInfo dnodeGetStatisInfo() {
+  SStatisInfo info = {0};
+  if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) {
    info.httpReqNum   = httpGetReqCount();
-    info.queryReqNum  = atomic_exchange_32(&tsDnodeQueryReqNum, 0);
-    info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0);
+    info.queryReqNum  = atomic_exchange_32(&tsQueryReqNum, 0);
+    info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0);
  }

  return info;
--- a/src/dnode/src/dnodeTelemetry.c
+++ b/src/dnode/src/dnodeTelemetry.c
@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) {
    return;
  }
  
-  if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
+  if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
    dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
  } 

@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() {
    dTrace("failed to create telemetry thread, reason:%s", strerror(errno));
  }

+  dInfo("dnode telemetry is initialized");
  return 0;
 }

--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@ -17,225 +17,184 @@
 #include "os.h"
 #include "taoserror.h"
 #include "taosmsg.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "twal.h"
 #include "tglobal.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeVRead.h"
+#include "tqueue.h"
 #include "vnode.h"
+#include "dnodeInt.h"

 typedef struct {
-  pthread_t  thread;    // thread
-  int32_t    workerId;  // worker ID
-} SReadWorker;
+  pthread_t thread;    // thread
+  int32_t   workerId;  // worker ID
+} SVReadWorker;

 typedef struct {
-  int32_t    max;       // max number of workers
-  int32_t    min;       // min number of workers
-  int32_t    num;       // current number of workers
-  SReadWorker *readWorker;
+  int32_t max;  // max number of workers
+  int32_t min;  // min number of workers
+  int32_t num;  // current number of workers
+  SVReadWorker *  worker;
  pthread_mutex_t mutex;
-} SReadWorkerPool;
+} SVReadWorkerPool;

 static void *dnodeProcessReadQueue(void *param);
-static void  dnodeHandleIdleReadWorker(SReadWorker *);

 // module global variable
-static SReadWorkerPool readPool;
-static taos_qset       readQset;
+static SVReadWorkerPool tsVReadWP;
+static taos_qset        tsVReadQset;

-int32_t dnodeInitVnodeRead() {
-  readQset = taosOpenQset();
+int32_t dnodeInitVRead() {
+  tsVReadQset = taosOpenQset();

-  readPool.min = tsNumOfCores;
-  readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
-  if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
-  readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
-  pthread_mutex_init(&readPool.mutex, NULL);
+  tsVReadWP.min = tsNumOfCores;
+  tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore;
+  if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min;
+  tsVReadWP.worker = (SVReadWorker *)calloc(sizeof(SVReadWorker), tsVReadWP.max);
+  pthread_mutex_init(&tsVReadWP.mutex, NULL);

-  if (readPool.readWorker == NULL) return -1;
-  for (int i = 0; i < readPool.max; ++i) {
-    SReadWorker *pWorker = readPool.readWorker + i;
+  if (tsVReadWP.worker == NULL) return -1;
+  for (int i = 0; i < tsVReadWP.max; ++i) {
+    SVReadWorker *pWorker = tsVReadWP.worker + i;
    pWorker->workerId = i;
  }

-  dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max);
+  dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max);
  return 0;
 }

-void dnodeCleanupVnodeRead() {
-  for (int i = 0; i < readPool.max; ++i) {
-    SReadWorker *pWorker = readPool.readWorker + i;
+void dnodeCleanupVRead() {
+  for (int i = 0; i < tsVReadWP.max; ++i) {
+    SVReadWorker *pWorker = tsVReadWP.worker + i;
    if (pWorker->thread) {
-      taosQsetThreadResume(readQset);
+      taosQsetThreadResume(tsVReadQset);
    }
  }

-  for (int i = 0; i < readPool.max; ++i) {
-    SReadWorker *pWorker = readPool.readWorker + i;
+  for (int i = 0; i < tsVReadWP.max; ++i) {
+    SVReadWorker *pWorker = tsVReadWP.worker + i;
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
    }
  }

-  free(readPool.readWorker);
-  taosCloseQset(readQset);
-  pthread_mutex_destroy(&readPool.mutex);
+  free(tsVReadWP.worker);
+  taosCloseQset(tsVReadQset);
+  pthread_mutex_destroy(&tsVReadWP.mutex);

-  dInfo("dnode read is closed");
+  dInfo("dnode vread is closed");
 }

-void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
-  int32_t     queuedMsgNum = 0;
-  int32_t     leftLen      = pMsg->contLen;
-  char        *pCont       = (char *) pMsg->pCont;
+void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
+  int32_t queuedMsgNum = 0;
+  int32_t leftLen = pMsg->contLen;
+  char *  pCont = (char *)pMsg->pCont;

  while (leftLen > 0) {
-    SMsgHead *pHead = (SMsgHead *) pCont;
-    pHead->vgId    = htonl(pHead->vgId);
+    SMsgHead *pHead = (SMsgHead *)pCont;
+    pHead->vgId = htonl(pHead->vgId);
    pHead->contLen = htonl(pHead->contLen);

-    taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
-
-    if (queue == NULL) {
-      leftLen -= pHead->contLen;
-      pCont -= pHead->contLen;
-      continue;
+    void *pVnode = vnodeAcquire(pHead->vgId);
+    if (pVnode != NULL) {
+      int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+      if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
+      vnodeRelease(pVnode);
    }

-    // put message into queue
-    SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
-    pRead->rpcMsg      = *pMsg;
-    pRead->pCont       = pCont;
-    pRead->contLen     = pHead->contLen;
-
-    // next vnode
    leftLen -= pHead->contLen;
    pCont -= pHead->contLen;
-    queuedMsgNum++;
-
-    taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
  }

  if (queuedMsgNum == 0) {
-    SRpcMsg rpcRsp = {
-        .handle  = pMsg->handle,
-        .pCont   = NULL,
-        .contLen = 0,
-        .code    = TSDB_CODE_VND_INVALID_VGROUP_ID,
-        .msgType = 0
-    };
+    SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
    rpcSendResponse(&rpcRsp);
-    rpcFreeCont(pMsg->pCont);
  }
+
+  rpcFreeCont(pMsg->pCont);
 }

-void *dnodeAllocateVnodeRqueue(void *pVnode) {
-  pthread_mutex_lock(&readPool.mutex);
+void *dnodeAllocVReadQueue(void *pVnode) {
+  pthread_mutex_lock(&tsVReadWP.mutex);
  taos_queue queue = taosOpenQueue();
  if (queue == NULL) {
-    pthread_mutex_unlock(&readPool.mutex);
+    pthread_mutex_unlock(&tsVReadWP.mutex);
    return NULL;
  }

-  taosAddIntoQset(readQset, queue, pVnode);
+  taosAddIntoQset(tsVReadQset, queue, pVnode);

  // spawn a thread to process queue
-  if (readPool.num < readPool.max) {
+  if (tsVReadWP.num < tsVReadWP.max) {
    do {
-      SReadWorker *pWorker = readPool.readWorker + readPool.num;
+      SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num;

      pthread_attr_t thAttr;
      pthread_attr_init(&thAttr);
      pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

      if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
-        dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+        dError("failed to create thread to process vread vqueue since %s", strerror(errno));
      }

      pthread_attr_destroy(&thAttr);
-      readPool.num++;
-      dDebug("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num);
-    } while (readPool.num < readPool.min);
+      tsVReadWP.num++;
+      dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num);
+    } while (tsVReadWP.num < tsVReadWP.min);
  }

-  pthread_mutex_unlock(&readPool.mutex);
-  dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue);
+  pthread_mutex_unlock(&tsVReadWP.mutex);
+  dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue);

  return queue;
 }

-void dnodeFreeVnodeRqueue(void *rqueue) {
+void dnodeFreeVReadQueue(void *rqueue) {
  taosCloseQueue(rqueue);
-
-  // dynamically adjust the number of threads
 }

-void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
+void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
  SRpcMsg rpcRsp = {
-    .handle  = pRead->rpcMsg.handle,
+    .handle  = pRead->rpcHandle,
    .pCont   = pRead->rspRet.rsp,
    .contLen = pRead->rspRet.len,
    .code    = code,
  };

  rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pRead->rpcMsg.pCont);
  vnodeRelease(pVnode);
 }

-void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) {
-  rpcFreeCont(pRead->rpcMsg.pCont);
+void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
  vnodeRelease(pVnode);
 }

 static void *dnodeProcessReadQueue(void *param) {
-  SReadMsg    *pReadMsg;
-  int          type;
-  void        *pVnode;
+  SVReadMsg *pRead;
+  int32_t    qtype;
+  void *     pVnode;

  while (1) {
-    if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
-      dDebug("qset:%p dnode read got no message from qset, exiting", readQset);
+    if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) {
+      dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset);
      break;
    }

-    dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle,
-           taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg);
+    dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
+           taosMsg[pRead->msgType], qtype);

-    int32_t code = vnodeProcessRead(pVnode, pReadMsg);
+    int32_t code = vnodeProcessRead(pVnode, pRead);

-    if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
-      dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
+    if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
+      dnodeSendRpcVReadRsp(pVnode, pRead, code);
    } else {
      if (code == TSDB_CODE_QRY_HAS_RSP) {
-        dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code);
-      } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
-        assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5));
-        dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+        dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code);
+      } else {  // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
+        assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
+        dnodeDispatchNonRspMsg(pVnode, pRead, code);
      }
    }

-    taosFreeQitem(pReadMsg);
+    taosFreeQitem(pRead);
  }

  return NULL;
 }
-
-
-UNUSED_FUNC
-static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
-  int32_t num = taosGetQueueNumber(readQset);
-
-  if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) {
-    readPool.num--;
-    dDebug("read worker:%d is released, total:%d", pWorker->workerId, readPool.num);
-    pthread_exit(NULL);
-  } else {
-    usleep(30000);
-    sched_yield();
-  }
-}
-
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@ -15,74 +15,55 @@

 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
 #include "taoserror.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "trpc.h"
-#include "tsdb.h"
-#include "twal.h"
-#include "tdataformat.h"
+#include "taosmsg.h"
 #include "tglobal.h"
-#include "tsync.h"
+#include "tqueue.h"
+#include "twal.h"
 #include "vnode.h"
 #include "dnodeInt.h"
-#include "syncInt.h"
-#include "dnodeVWrite.h"
-#include "dnodeMgmt.h"

 typedef struct {
-  taos_qall  qall;
-  taos_qset  qset;      // queue set
-  pthread_t  thread;    // thread
-  int32_t    workerId;  // worker ID
-} SWriteWorker;
+  taos_qall qall;
+  taos_qset qset;      // queue set
+  int32_t   workerId;  // worker ID
+  pthread_t thread;    // thread
+} SVWriteWorker;

 typedef struct {
-  SRspRet  rspRet;
-  int32_t  processedCount;
-  int32_t  code;
-  void    *pCont;
-  int32_t  contLen;
-  SRpcMsg  rpcMsg;
-} SWriteMsg;
-
-typedef struct {
-  int32_t        max;        // max number of workers
-  int32_t        nextId;     // from 0 to max-1, cyclic
-  SWriteWorker  *writeWorker;
+  int32_t max;     // max number of workers
+  int32_t nextId;  // from 0 to max-1, cyclic
+  SVWriteWorker * worker;
  pthread_mutex_t mutex;
-} SWriteWorkerPool;
+} SVWriteWorkerPool;

-static void *dnodeProcessWriteQueue(void *param);
-static void  dnodeHandleIdleWorker(SWriteWorker *pWorker);
+static SVWriteWorkerPool tsVWriteWP;
+static void *dnodeProcessVWriteQueue(void *pWorker);

-SWriteWorkerPool wWorkerPool;
+int32_t dnodeInitVWrite() {
+  tsVWriteWP.max = tsNumOfCores;
+  tsVWriteWP.worker = tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max);
+  if (tsVWriteWP.worker == NULL) return -1;
+  pthread_mutex_init(&tsVWriteWP.mutex, NULL);

-int32_t dnodeInitVnodeWrite() {
-  wWorkerPool.max = tsNumOfCores;
-  wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max);
-  if (wWorkerPool.writeWorker == NULL) return -1;
-  pthread_mutex_init(&wWorkerPool.mutex, NULL);
-
-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    wWorkerPool.writeWorker[i].workerId = i;
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    tsVWriteWP.worker[i].workerId = i;
  }

-  dInfo("dnode write is opened, max worker %d", wWorkerPool.max);
+  dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max);
  return 0;
 }

-void dnodeCleanupVnodeWrite() {
-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+void dnodeCleanupVWrite() {
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    SVWriteWorker *pWorker = tsVWriteWP.worker + i;
    if (pWorker->thread) {
      taosQsetThreadResume(pWorker->qset);
    }
  }

-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    SVWriteWorker *pWorker = tsVWriteWP.worker + i;
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
      taosFreeQall(pWorker->qall);
@ -90,52 +71,51 @@ void dnodeCleanupVnodeWrite() {
    }
  }

-  pthread_mutex_destroy(&wWorkerPool.mutex);
-  free(wWorkerPool.writeWorker);
-  dInfo("dnode write is closed");
+  pthread_mutex_destroy(&tsVWriteWP.mutex);
+  tfree(tsVWriteWP.worker);
+  dInfo("dnode vwrite is closed");
 }

-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
-  char *pCont = (char *)pMsg->pCont;
+void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
+  int32_t code;
+  char *pCont = pRpcMsg->pCont;

-  if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
+  if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
    SMsgDesc *pDesc = (SMsgDesc *)pCont;
    pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
    pCont += sizeof(SMsgDesc);
  }

-  SMsgHead *pHead = (SMsgHead *) pCont;
-  pHead->vgId     = htonl(pHead->vgId);
-  pHead->contLen  = htonl(pHead->contLen);
+  SMsgHead *pMsg = (SMsgHead *)pCont;
+  pMsg->vgId = htonl(pMsg->vgId);
+  pMsg->contLen = htonl(pMsg->contLen);

-  taos_queue queue = vnodeAcquireWqueue(pHead->vgId);
-  if (queue) {
-    // put message into queue
-    SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg));
-    pWrite->rpcMsg    = *pMsg;
-    pWrite->pCont     = pCont;
-    pWrite->contLen   = pHead->contLen;
-
-    taosWriteQitem(queue, TAOS_QTYPE_RPC, pWrite);
+  void *pVnode = vnodeAcquire(pMsg->vgId);
+  if (pVnode == NULL) {
+    code = TSDB_CODE_VND_INVALID_VGROUP_ID;
  } else {
-    SRpcMsg rpcRsp = {
-      .handle  = pMsg->handle,
-      .pCont   = NULL,
-      .contLen = 0,
-      .code    = TSDB_CODE_VND_INVALID_VGROUP_ID,
-      .msgType = 0
-    };
-    rpcSendResponse(&rpcRsp);
-    rpcFreeCont(pMsg->pCont);
+    SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead));
+    pHead->msgType = pRpcMsg->msgType;
+    pHead->version = 0;
+    pHead->len = pMsg->contLen;
+    code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg);
  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code};
+    rpcSendResponse(&rpcRsp);
+  }
+
+  vnodeRelease(pVnode);
+  rpcFreeCont(pRpcMsg->pCont);
 }

-void *dnodeAllocateVnodeWqueue(void *pVnode) {
-  pthread_mutex_lock(&wWorkerPool.mutex);
-  SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
-  void *queue = taosOpenQueue();
+void *dnodeAllocVWriteQueue(void *pVnode) {
+  pthread_mutex_lock(&tsVWriteWP.mutex);
+  SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
+  taos_queue *queue = taosOpenQueue();
  if (queue == NULL) {
-    pthread_mutex_unlock(&wWorkerPool.mutex);
+    pthread_mutex_unlock(&tsVWriteWP.mutex);
    return NULL;
  }

@ -143,7 +123,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
    pWorker->qset = taosOpenQset();
    if (pWorker->qset == NULL) {
      taosCloseQueue(queue);
-      pthread_mutex_unlock(&wWorkerPool.mutex);
+      pthread_mutex_unlock(&tsVWriteWP.mutex);
      return NULL;
    }

@ -152,45 +132,43 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
    if (pWorker->qall == NULL) {
      taosCloseQset(pWorker->qset);
      taosCloseQueue(queue);
-      pthread_mutex_unlock(&wWorkerPool.mutex);
+      pthread_mutex_unlock(&tsVWriteWP.mutex);
      return NULL;
    }
    pthread_attr_t thAttr;
    pthread_attr_init(&thAttr);
    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
-      dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) {
+      dError("failed to create thread to process vwrite queue since %s", strerror(errno));
      taosFreeQall(pWorker->qall);
      taosCloseQset(pWorker->qset);
      taosCloseQueue(queue);
      queue = NULL;
    } else {
-      dDebug("write worker:%d is launched", pWorker->workerId);
-      wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+      dDebug("dnode vwrite worker:%d is launched", pWorker->workerId);
+      tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
    }

    pthread_attr_destroy(&thAttr);
  } else {
    taosAddIntoQset(pWorker->qset, queue, pVnode);
-    wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+    tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
  }

-  pthread_mutex_unlock(&wWorkerPool.mutex);
-  dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue);
+  pthread_mutex_unlock(&tsVWriteWP.mutex);
+  dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue);

  return queue;
 }

-void dnodeFreeVnodeWqueue(void *wqueue) {
-  taosCloseQueue(wqueue);
-
-  // dynamically adjust the number of threads
+void dnodeFreeVWriteQueue(void *pWqueue) {
+  taosCloseQueue(pWqueue);
 }

-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
-  SWriteMsg *pWrite = (SWriteMsg *)param;
-  if (pWrite == NULL) return;
+void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) {
+  if (wparam == NULL) return;
+  SVWriteMsg *pWrite = wparam;

  if (code < 0) pWrite->code = code;
  int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
@ -198,106 +176,64 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
  if (count <= 1) return;

  SRpcMsg rpcRsp = {
-    .handle  = pWrite->rpcMsg.handle,
+    .handle  = pWrite->rpcHandle,
    .pCont   = pWrite->rspRet.rsp,
    .contLen = pWrite->rspRet.len,
    .code    = pWrite->code,
  };

  rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pWrite->rpcMsg.pCont);
-  taosFreeQitem(pWrite);
-
-  vnodeRelease(pVnode);
+  vnodeFreeFromWQueue(pVnode, pWrite);
 }

-static void *dnodeProcessWriteQueue(void *param) {
-  SWriteWorker *pWorker = (SWriteWorker *)param;
-  SWriteMsg *   pWrite;
-  SWalHead *    pHead;
-  int32_t       numOfMsgs;
-  int           type;
-  void *        pVnode, *item;
-  SRspRet *     pRspRet;
+static void *dnodeProcessVWriteQueue(void *wparam) {
+  SVWriteWorker *pWorker = wparam;
+  SVWriteMsg *   pWrite;
+  void *         pVnode;
+  int32_t        numOfMsgs;
+  int32_t        qtype;

-  dDebug("write worker:%d is running", pWorker->workerId);
+  dDebug("dnode vwrite worker:%d is running", pWorker->workerId);

  while (1) {
    numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
    if (numOfMsgs == 0) {
-      dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset);
+      dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset);
      break;
    }

+    bool forceFsync = false;
    for (int32_t i = 0; i < numOfMsgs; ++i) {
-      pWrite = NULL;
-      pRspRet = NULL;
-      taosGetQitem(pWorker->qall, &type, &item);
-      if (type == TAOS_QTYPE_RPC) {
-        pWrite = (SWriteMsg *)item;
-        pRspRet = &pWrite->rspRet;
-        pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
-        pHead->msgType = pWrite->rpcMsg.msgType;
-        pHead->version = 0;
-        pHead->len = pWrite->contLen;
-        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
-               taosMsg[pWrite->rpcMsg.msgType]);
-      } else if (type == TAOS_QTYPE_CQ) {
-        pHead = (SWalHead *)((char*)item + sizeof(SSyncHead));
-        dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
-               pHead->version);
-      } else {
-        pHead = (SWalHead *)item;
-        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
-               pHead->version);
-      }
+      taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+      dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite,
+             taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);

-      int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
-      dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
-             pHead->version, tstrerror(code));
+      pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
+      if (pWrite->code <= 0) pWrite->processedCount = 1;
+      if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;

-      if (pWrite) {
-        pWrite->rpcMsg.code = code;
-        if (code <= 0) pWrite->processedCount = 1;
-      }
+      dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
    }

-    walFsync(vnodeGetWal(pVnode));
+    walFsync(vnodeGetWal(pVnode), forceFsync);

    // browse all items, and process them one by one
    taosResetQitems(pWorker->qall);
    for (int32_t i = 0; i < numOfMsgs; ++i) {
-      taosGetQitem(pWorker->qall, &type, &item);
-      if (type == TAOS_QTYPE_RPC) {
-        pWrite = (SWriteMsg *)item;
-        dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code);
-      } else if (type == TAOS_QTYPE_FWD) {
-        pHead = (SWalHead *)item;
-        vnodeConfirmForward(pVnode, pHead->version, 0);
-        taosFreeQitem(item);
-        vnodeRelease(pVnode);
+      taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+      if (qtype == TAOS_QTYPE_RPC) {
+        dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
      } else {
-        taosFreeQitem(item);
-        vnodeRelease(pVnode);
+        if (qtype == TAOS_QTYPE_FWD) {
+          vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
+        }
+        if (pWrite->rspRet.rsp) {
+          rpcFreeCont(pWrite->rspRet.rsp);
+        }
+        vnodeFreeFromWQueue(pVnode, pWrite);
      }
    }
  }

  return NULL;
 }
-
-UNUSED_FUNC
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
-  int32_t num = taosGetQueueNumber(pWorker->qset);
-
-  if (num > 0) {
-    usleep(30000);
-    sched_yield();
-  } else {
-    taosFreeQall(pWorker->qall);
-    taosCloseQset(pWorker->qset);
-    pWorker->qset = NULL;
-    dDebug("write worker:%d is released", pWorker->workerId);
-    pthread_exit(NULL);
-  }
-}
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@ -25,29 +25,31 @@ extern "C" {
 #include "hash.h"
 #include "taoserror.h"
 #include "trpc.h"
+#include "taosmsg.h"

 typedef struct {
  int32_t queryReqNum;
  int32_t submitReqNum;
  int32_t httpReqNum;
-} SDnodeStatisInfo;
+} SStatisInfo;

 typedef enum {
-  TSDB_DNODE_RUN_STATUS_INITIALIZE,
-  TSDB_DNODE_RUN_STATUS_RUNING,
-  TSDB_DNODE_RUN_STATUS_STOPPED
-} SDnodeRunStatus;
+  TSDB_RUN_STATUS_INITIALIZE,
+  TSDB_RUN_STATUS_RUNING,
+  TSDB_RUN_STATUS_STOPPED
+} SRunStatus;

-SDnodeRunStatus dnodeGetRunStatus();
-SDnodeStatisInfo dnodeGetStatisInfo();
+SRunStatus  dnodeGetRunStatus();
+SStatisInfo dnodeGetStatisInfo();

 bool    dnodeIsFirstDeploy();
-char *  dnodeGetMnodeMasterEp();
-void    dnodeGetMnodeEpSetForPeer(void *epSet);
-void    dnodeGetMnodeEpSetForShell(void *epSet);
-void *  dnodeGetMnodeInfos();
+bool    dnodeIsMasterEp(char *ep);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
 int32_t dnodeGetDnodeId();
-bool    dnodeStartMnode(void *pModes);
+void    dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+bool    dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+bool    dnodeStartMnode(SMnodeInfos *minfos);

 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
@ -55,21 +57,21 @@ void  dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
 void  dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);
 void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid);

-void *dnodeAllocateVnodeWqueue(void *pVnode);
-void  dnodeFreeVnodeWqueue(void *queue);
-void *dnodeAllocateVnodeRqueue(void *pVnode);
-void  dnodeFreeVnodeRqueue(void *rqueue);
-void  dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
+void *dnodeAllocVWriteQueue(void *pVnode);
+void  dnodeFreeVWriteQueue(void *pWqueue);
+void  dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code);
+void *dnodeAllocVReadQueue(void *pVnode);
+void  dnodeFreeVReadQueue(void *rqueue);

-int32_t dnodeAllocateMnodePqueue();
-void    dnodeFreeMnodePqueue();
-int32_t dnodeAllocateMnodeRqueue();
-void    dnodeFreeMnodeRqueue();
-int32_t dnodeAllocateMnodeWqueue();
-void    dnodeFreeMnodeWqueue();
-void    dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
-void    dnodeReprocessMnodeWriteMsg(void *pMsg);
-void    dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
+int32_t dnodeAllocateMPeerQueue();
+void    dnodeFreeMPeerQueue();
+int32_t dnodeAllocMReadQueue();
+void    dnodeFreeMReadQueue();
+int32_t dnodeAllocMWritequeue();
+void    dnodeFreeMWritequeue();
+void    dnodeSendRpcMWriteRsp(void *pMsg, int32_t code);
+void    dnodeReprocessMWriteMsg(void *pMsg);
+void    dnodeDelayReprocessMWriteMsg(void *pMsg);

 void    dnodeSendStatusMsgToMnode();

--- a/src/inc/mnode.h
+++ b/src/inc/mnode.h
@ -35,24 +35,26 @@ typedef struct {
 } SMnodeRsp;

 typedef struct SMnodeMsg {
-  SRpcMsg   rpcMsg;
-  SMnodeRsp rpcRsp;
-  int8_t    received;
-  int8_t    successed;
-  int8_t    expected;
-  int8_t    retry;
-  int32_t   code;
-  void *    pObj;
  struct SAcctObj * pAcct;
  struct SDnodeObj *pDnode;
  struct SUserObj * pUser;
  struct SDbObj *   pDb;
  struct SVgObj *   pVgroup;
  struct STableObj *pTable;
-  struct SSuperTableObj *pSTable;
+  struct SSTableObj*pSTable;
+  SMnodeRsp rpcRsp;
+  int8_t    received;
+  int8_t    successed;
+  int8_t    expected;
+  int8_t    retry;
+  int32_t   incomingTs;
+  int32_t   code;
+  void *    pObj;
+  SRpcMsg   rpcMsg;
+  char      pCont[];
 } SMnodeMsg;

-void    mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
+void *  mnodeCreateMsg(SRpcMsg *pRpcMsg);
 int32_t mnodeInitMsg(SMnodeMsg *pMsg);
 void    mnodeCleanupMsg(SMnodeMsg *pMsg);

--- a/src/inc/taos.h
+++ b/src/inc/taos.h
@ -64,7 +64,7 @@ typedef struct taosField {
 #endif

 DLL_EXPORT void  taos_init();
-DLL_EXPORT void  taos_cleanup();
+DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@ -61,13 +61,23 @@ typedef struct tstr {

 // Bytes for each type.
 extern const int32_t TYPE_BYTES[11];
+
 // TODO: replace and remove code below
-#define CHAR_BYTES   sizeof(char)
-#define SHORT_BYTES  sizeof(int16_t)
-#define INT_BYTES    sizeof(int32_t)
-#define LONG_BYTES   sizeof(int64_t)
-#define FLOAT_BYTES  sizeof(float)
-#define DOUBLE_BYTES sizeof(double)
+#define CHAR_BYTES    sizeof(char)
+#define SHORT_BYTES   sizeof(int16_t)
+#define INT_BYTES     sizeof(int32_t)
+#define LONG_BYTES    sizeof(int64_t)
+#define FLOAT_BYTES   sizeof(float)
+#define DOUBLE_BYTES  sizeof(double)
+#define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)
+
+#define TSDB_KEYSIZE            sizeof(TSKEY)
+
+#if LINUX
+#define TSDB_NCHAR_SIZE         sizeof(wchar_t)
+#else
+#define TSDB_NCHAR_SIZE         sizeof(int32_t)
+#endif

 // NULL definition
 #define TSDB_DATA_BOOL_NULL             0x02
@ -75,6 +85,7 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_DATA_SMALLINT_NULL         0x8000
 #define TSDB_DATA_INT_NULL              0x80000000L
 #define TSDB_DATA_BIGINT_NULL           0x8000000000000000L
+#define TSDB_DATA_TIMESTAMP_NULL        TSDB_DATA_BIGINT_NULL

 #define TSDB_DATA_FLOAT_NULL            0x7FF00000              // it is an NAN
 #define TSDB_DATA_DOUBLE_NULL           0x7FFFFF0000000000L     // an NAN
@ -101,10 +112,12 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1
 #define TSDB_TIME_PRECISION_NANO  2
-#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))

 #define TSDB_TIME_PRECISION_MILLI_STR "ms"
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
+#define TSDB_TIME_PRECISION_NANO_STR  "ns"
+
+#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))

 #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
 #define T_APPEND_MEMBER(dst, ptr, type, member) \
@ -118,15 +131,6 @@ do { \
  (src) = (void *)((char *)src + sizeof(type));\
 } while(0)

-#define TSDB_KEYSIZE              sizeof(TSKEY)
-
-#if LINUX
-  #define TSDB_NCHAR_SIZE         sizeof(wchar_t)
-#else
-  #define TSDB_NCHAR_SIZE         4
-#endif
-//#define TSDB_CHAR_TERMINATED_SPACE 1
-
 #define GET_INT8_VAL(x)   (*(int8_t *)(x))
 #define GET_INT16_VAL(x)  (*(int16_t *)(x))
 #define GET_INT32_VAL(x)  (*(int32_t *)(x))
@ -172,7 +176,6 @@ typedef struct tDataTypeDescriptor {
 } tDataTypeDescriptor;

 extern tDataTypeDescriptor tDataTypeDesc[11];
-#define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)

 bool isValidDataType(int32_t type);
 //bool isNull(const char *val, int32_t type);
@ -266,10 +269,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_AUTH_LEN             16
 #define TSDB_KEY_LEN              16
 #define TSDB_VERSION_LEN          12
-#define TSDB_STREET_LEN           64
-#define TSDB_CITY_LEN             20
-#define TSDB_STATE_LEN            20
-#define TSDB_COUNTRY_LEN          20
 #define TSDB_LOCALE_LEN           64
 #define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
@ -333,7 +332,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf

 #define TSDB_MIN_DAYS_PER_FILE          1
 #define TSDB_MAX_DAYS_PER_FILE          3650 
-#define TSDB_DEFAULT_DAYS_PER_FILE      2 
+#define TSDB_DEFAULT_DAYS_PER_FILE      10

 #define TSDB_MIN_KEEP                   1        // data in db to be reserved.
 #define TSDB_MAX_KEEP                   365000   // data in db to be reserved.
@ -363,6 +362,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_MAX_WAL_LEVEL              2
 #define TSDB_DEFAULT_WAL_LEVEL          1

+#define TSDB_MIN_DB_UPDATE              0
+#define TSDB_MAX_DB_UPDATE              1
+#define TSDB_DEFAULT_DB_UPDATE_OPTION   0
+
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000   // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD       3000     // three second
@ -388,27 +391,27 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 * 1. ordinary sub query for select * from super_table
 * 2. all sqlobj generated by createSubqueryObj with this flag
 */
-#define TSDB_QUERY_TYPE_SUBQUERY                       0x02u
-#define TSDB_QUERY_TYPE_STABLE_SUBQUERY                0x04u     // two-stage subquery for super table
+#define TSDB_QUERY_TYPE_SUBQUERY               0x02u
+#define TSDB_QUERY_TYPE_STABLE_SUBQUERY        0x04u     // two-stage subquery for super table

-#define TSDB_QUERY_TYPE_TABLE_QUERY                    0x08u    // query ordinary table; below only apply to client side
-#define TSDB_QUERY_TYPE_STABLE_QUERY                   0x10u    // query on super table
-#define TSDB_QUERY_TYPE_JOIN_QUERY                     0x20u    // join query
-#define TSDB_QUERY_TYPE_PROJECTION_QUERY               0x40u    // select *,columns... query
-#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE                 0x80u    // join sub query at the second stage
+#define TSDB_QUERY_TYPE_TABLE_QUERY            0x08u    // query ordinary table; below only apply to client side
+#define TSDB_QUERY_TYPE_STABLE_QUERY           0x10u    // query on super table
+#define TSDB_QUERY_TYPE_JOIN_QUERY             0x20u    // join query
+#define TSDB_QUERY_TYPE_PROJECTION_QUERY       0x40u    // select *,columns... query
+#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE         0x80u    // join sub query at the second stage

-#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY              0x400u
-#define TSDB_QUERY_TYPE_INSERT                        0x100u    // insert type
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY              0x200u
-#define TSDB_QUERY_TYPE_STMT_INSERT                   0x800u    // stmt insert type
+#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY       0x400u
+#define TSDB_QUERY_TYPE_INSERT                 0x100u    // insert type
+#define TSDB_QUERY_TYPE_MULTITABLE_QUERY       0x200u
+#define TSDB_QUERY_TYPE_STMT_INSERT            0x800u    // stmt insert type

 #define TSDB_QUERY_HAS_TYPE(x, _type)         (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)         ((x) |= (_type))
 #define TSDB_QUERY_CLEAR_TYPE(x, _type)       ((x) &= (~_type))
 #define TSDB_QUERY_RESET_TYPE(x)              ((x) = TSDB_QUERY_TYPE_NON_TYPE)

-#define TSDB_ORDER_ASC   1
-#define TSDB_ORDER_DESC  2
+#define TSDB_ORDER_ASC                  1
+#define TSDB_ORDER_DESC                 2

 #define TSDB_DEFAULT_CLUSTER_HASH_SIZE  1
 #define TSDB_DEFAULT_MNODES_HASH_SIZE   5
@ -420,50 +423,60 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_DEFAULT_STABLES_HASH_SIZE  100
 #define TSDB_DEFAULT_CTABLES_HASH_SIZE  20000

-#define TSDB_PORT_DNODESHELL 0 
-#define TSDB_PORT_DNODEDNODE 5 
-#define TSDB_PORT_SYNC       10 
-#define TSDB_PORT_HTTP       11 
-#define TSDB_PORT_ARBITRATOR 12 
+#define TSDB_PORT_DNODESHELL            0
+#define TSDB_PORT_DNODEDNODE            5
+#define TSDB_PORT_SYNC                  10
+#define TSDB_PORT_HTTP                  11
+#define TSDB_PORT_ARBITRATOR            12
+#define TSDB_PORT_DNODESHELL 0
+#define TSDB_PORT_DNODEDNODE 5
+#define TSDB_PORT_SYNC       10
+#define TSDB_PORT_HTTP       11
+#define TSDB_PORT_ARBITRATOR 12

-#define TAOS_QTYPE_RPC      0
-#define TAOS_QTYPE_FWD      1
-#define TAOS_QTYPE_WAL      2 
-#define TAOS_QTYPE_CQ       3
-#define TAOS_QTYPE_QUERY    4
+#define TSDB_MAX_WAL_SIZE    (1024*1024)
+
+typedef enum {
+  TAOS_QTYPE_RPC   = 0,
+  TAOS_QTYPE_FWD   = 1,
+  TAOS_QTYPE_WAL   = 2,
+  TAOS_QTYPE_CQ    = 3,
+  TAOS_QTYPE_QUERY = 4
+} EQType;

 #define TSDB_MAX_TIERS           3
 #define TSDB_MAX_DISKS_PER_TIER 16
 #define TSDB_MAX_DISKS           (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)

 typedef enum {
-  TSDB_SUPER_TABLE        = 0,  // super table
-  TSDB_CHILD_TABLE        = 1,  // table created from super table
-  TSDB_NORMAL_TABLE       = 2,  // ordinary table
-  TSDB_STREAM_TABLE       = 3,  // table created from stream computing
-  TSDB_TABLE_MAX          = 4
+  TSDB_SUPER_TABLE  = 0,   // super table
+  TSDB_CHILD_TABLE  = 1,   // table created from super table
+  TSDB_NORMAL_TABLE = 2,  // ordinary table
+  TSDB_STREAM_TABLE = 3,  // table created from stream computing
+  TSDB_TABLE_MAX    = 4
 } ETableType;

 typedef enum {
-  TSDB_MOD_MNODE,
-  TSDB_MOD_HTTP,
-  TSDB_MOD_MONITOR,
-  TSDB_MOD_MQTT,
-  TSDB_MOD_MAX
+  TSDB_MOD_MNODE   = 0,
+  TSDB_MOD_HTTP    = 1,
+  TSDB_MOD_MONITOR = 2,
+  TSDB_MOD_MQTT    = 3,
+  TSDB_MOD_MAX     = 4
 } EModuleType;

-  typedef enum {
-    TSDB_CHECK_ITEM_NETWORK,
-    TSDB_CHECK_ITEM_MEM,
-    TSDB_CHECK_ITEM_CPU,
-    TSDB_CHECK_ITEM_DISK,
-    TSDB_CHECK_ITEM_OS,    
-    TSDB_CHECK_ITEM_ACCESS,    
-    TSDB_CHECK_ITEM_VERSION,
-    TSDB_CHECK_ITEM_DATAFILE,
-    TSDB_CHECK_ITEM_MAX
-  } ECheckItemType;
+typedef enum {
+  TSDB_CHECK_ITEM_NETWORK,
+  TSDB_CHECK_ITEM_MEM,
+  TSDB_CHECK_ITEM_CPU,
+  TSDB_CHECK_ITEM_DISK,
+  TSDB_CHECK_ITEM_OS,
+  TSDB_CHECK_ITEM_ACCESS,
+  TSDB_CHECK_ITEM_VERSION,
+  TSDB_CHECK_ITEM_DATAFILE,
+  TSDB_CHECK_ITEM_MAX
+} ECheckItemType;

+extern char *qtypeStr[];

 #ifdef __cplusplus
 }
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@ -28,7 +28,7 @@ extern "C" {
 #else
 #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (0x80000000 | ((mod)<<16) | (code));
 #endif
-
+ 
 #define TAOS_SYSTEM_ERROR(code)             (0x80ff0000 | (code))
 #define TAOS_SUCCEEDED(err)                 ((err) >= 0)
 #define TAOS_FAILED(err)                    ((err) < 0)
@ -37,7 +37,7 @@ const char* tstrerror(int32_t err);

 int32_t* taosGetErrno();
 #define terrno                              (*taosGetErrno())
-
+ 
 #define TSDB_CODE_SUCCESS                   0

 #ifdef TAOS_ERROR_C
@ -74,6 +74,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED,         0, 0x0101, "Memory cor
 TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY,            0, 0x0102, "Out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG,          0, 0x0103, "Invalid config message")
 TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED,           0, 0x0104, "Data file corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                0, 0x0105, "Ref out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     0, 0x0106, "too many Ref Objs")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED,               0, 0x0107, "Ref ID is removed")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID,               0, 0x0108, "Invalid Ref ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST,            0, 0x0109, "Ref is already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST,                0, 0x010A, "Ref is not there")

 //client
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL,              0, 0x0200, "Invalid SQL statement")
@ -175,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB,               0, 0x0383, "Invalid da
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN,     0, 0x0384, "Cannot delete monitor database")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES,       0, 0x0385, "Too many databases for account")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database not available")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         0, 0x0387, "Database unsynced")

 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")
@ -190,7 +197,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_DUPLICATE_PRIMARY_DISK,   0, 0x0409, "Duplicate
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_LACK_PRIMARY_DISK,        0, 0x040A, "Lack primary disk")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_DISK_AT_TIER,          0, 0x040B, "No disk at tier")

-// vnode
+// vnode 
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS,       0, 0x0500, "Action in progress")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED,        0, 0x0501, "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED,  0, 0x0502, "Action need to be reprocessed")
@ -202,6 +209,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "Missing da
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "Out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "Unexpected generic error in vnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE,     0, 0x050A, "Invalid version file")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL,                  0, 0x050B, "Vnode memory is full because commit failed")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")

@ -238,7 +246,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY,                0, 0x0707, "Query not
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP,                  0, 0x0708, "Query should response")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC,                  0, 0x0709, "Multiple retrieval of this query")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW,      0, 0x070A, "Too many time window in query")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,      0, 0x070B, "Query buffer limit has reached")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,        0, 0x070B, "Query buffer limit has reached")

 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "License expired")
@ -262,6 +270,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sy
 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED,           0, 0x1001, "WAL file is corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT,               0, 0x1002, "WAL size exceeds limit")

 // http
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE,          0, 0x1100, "http server is not onlin")
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )

+  
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
+
+
 #ifndef TAOS_MESSAGE_C
  TSDB_MSG_TYPE_MAX  // 105
 #endif
@ -291,7 +295,7 @@ typedef struct {
  SSchema schema[];
  // tagVal is padded after schema
  // char    tagVal[];
-} SCMAlterTableMsg;
+} SAlterTableMsg;

 typedef struct {
  SMsgHead  head;
@ -308,12 +312,12 @@ typedef struct {
 } SUpdateTableTagValMsg;

 typedef struct {
-  char clientVersion[TSDB_VERSION_LEN];
-  char msgVersion[TSDB_VERSION_LEN];
-  char db[TSDB_TABLE_FNAME_LEN];
-  char appName[TSDB_APPNAME_LEN];
+  char    clientVersion[TSDB_VERSION_LEN];
+  char    msgVersion[TSDB_VERSION_LEN];
+  char    db[TSDB_TABLE_FNAME_LEN];
+  char    appName[TSDB_APPNAME_LEN];
  int32_t pid;
-} SCMConnectMsg;
+} SConnectMsg;

 typedef struct {
  char      acctId[TSDB_ACCT_LEN];
@ -324,7 +328,7 @@ typedef struct {
  int8_t    reserved2;
  int32_t   connId;
  SRpcEpSet epSet;
-} SCMConnectRsp;
+} SConnectRsp;

 typedef struct {
  int32_t maxUsers;
@ -344,18 +348,18 @@ typedef struct {
  char     user[TSDB_USER_LEN];
  char     pass[TSDB_KEY_LEN];
  SAcctCfg cfg;
-} SCMCreateAcctMsg, SCMAlterAcctMsg;
+} SCreateAcctMsg, SAlterAcctMsg;

 typedef struct {
  char user[TSDB_USER_LEN];
-} SCMDropUserMsg, SCMDropAcctMsg;
+} SDropUserMsg, SDropAcctMsg;

 typedef struct {
  char   user[TSDB_USER_LEN];
  char   pass[TSDB_KEY_LEN];
  int8_t privilege;
  int8_t flag;
-} SCMCreateUserMsg, SCMAlterUserMsg;
+} SCreateUserMsg, SAlterUserMsg;

 typedef struct {
  int32_t  contLen;
@ -370,11 +374,11 @@ typedef struct {
  int32_t  vgId;
  uint64_t uid;
  char    tableId[TSDB_TABLE_FNAME_LEN];
-} SMDDropSTableMsg;
+} SDropSTableMsg;

 typedef struct {
  int32_t vgId;
-} SMDDropVnodeMsg;
+} SDropVnodeMsg;

 typedef struct SColIndex {
  int16_t  colId;      // column id
@ -388,6 +392,7 @@ typedef struct SColIndex {
 typedef struct SSqlFuncMsg {
  int16_t functionId;
  int16_t numOfParams;
+  int16_t resColId;      // result column id, id of the current output column

  SColIndex colInfo;
  struct ArgElem {
@ -457,11 +462,6 @@ typedef struct STimeWindow {
  TSKEY ekey;
 } STimeWindow;

-/*
- * the outputCols is equalled to or larger than numOfCols
- * e.g., select min(colName), max(colName), avg(colName) from table
- * the outputCols will be 3 while the numOfCols is 1.
- */
 typedef struct {
  SMsgHead    head;
  STimeWindow window;
@ -477,10 +477,11 @@ typedef struct {
  int64_t     limit;
  int64_t     offset;
  uint32_t    queryType;        // denote another query process
-  int16_t     numOfOutput;  // final output columns numbers
+  int16_t     numOfOutput;      // final output columns numbers
  int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
  int16_t     fillType;      // interpolate type
  uint64_t    fillVal;       // default value array list
+  int32_t     secondStageOutput;
  int32_t     tsOffset;       // offset value in current msg body, NOTE: ts list is compressed
  int32_t     tsLen;          // total length of ts comp block
  int32_t     tsNumOfBlocks;  // ts comp block numbers
@ -540,12 +541,14 @@ typedef struct {
  int8_t   replications;
  int8_t   quorum;
  int8_t   ignoreExist;
-} SCMCreateDbMsg, SCMAlterDbMsg;
+  int8_t   update;
+  int8_t   reserve[9];
+} SCreateDbMsg, SAlterDbMsg;

 typedef struct {
  char    db[TSDB_TABLE_FNAME_LEN];
  uint8_t ignoreNotExists;
-} SCMDropDbMsg, SCMUseDbMsg;
+} SDropDbMsg, SUseDbMsg;

 // IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed
 // TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE
@ -560,7 +563,7 @@ typedef struct {
 typedef struct {
  int32_t  vgId;
  int8_t   accessState;
-} SDMVgroupAccess;
+} SVgroupAccess;

 typedef struct {
  int32_t  dnodeId;
@ -568,18 +571,29 @@ typedef struct {
  uint32_t numOfVnodes;
  char     clusterId[TSDB_CLUSTER_ID_LEN];
  char     reserved[16];
-} SDMDnodeCfg;
+} SDnodeCfg;

 typedef struct {
-  int32_t   nodeId;
-  char      nodeEp[TSDB_EP_LEN];
-} SDMMnodeInfo;
+  int32_t  dnodeId;
+  uint16_t dnodePort;
+  char     dnodeFqdn[TSDB_FQDN_LEN];
+} SDnodeEp;

 typedef struct {
-  int8_t       inUse;
-  int8_t       nodeNum;
-  SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
-} SDMMnodeInfos;
+  int32_t  dnodeNum;
+  SDnodeEp dnodeEps[];
+} SDnodeEps;
+
+typedef struct {
+  int32_t mnodeId;
+  char    mnodeEp[TSDB_EP_LEN];
+} SMnodeInfo;
+
+typedef struct {
+  int8_t     inUse;
+  int8_t     mnodeNum;
+  SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA];
+} SMnodeInfos;

 typedef struct {
  int32_t  numOfMnodes;               // tsNumOfMnodes
@ -611,13 +625,13 @@ typedef struct {
  uint8_t     reserve2[15];
  SClusterCfg clusterCfg;
  SVnodeLoad  load[];
-} SDMStatusMsg;
+} SStatusMsg;

 typedef struct {
-  SDMMnodeInfos    mnodes;
-  SDMDnodeCfg      dnodeCfg;
-  SDMVgroupAccess  vgAccess[];
-} SDMStatusRsp;
+  SMnodeInfos   mnodes;
+  SDnodeCfg     dnodeCfg;
+  SVgroupAccess vgAccess[];
+} SStatusRsp;

 typedef struct {
  uint32_t vgId;
@ -639,55 +653,56 @@ typedef struct {
  int8_t   replications;
  int8_t   wals;
  int8_t   quorum;
-  int8_t   reserved[16];
-} SMDVnodeCfg;
+  int8_t   update;
+  int8_t   reserved[15];
+} SVnodeCfg;

 typedef struct {
  int32_t  nodeId;
  char     nodeEp[TSDB_EP_LEN];
-} SMDVnodeDesc;
+} SVnodeDesc;

 typedef struct {
-  char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
-  SMDVnodeCfg  cfg;
-  SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
-} SMDCreateVnodeMsg, SMDAlterVnodeMsg;
+  char       db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  SVnodeCfg  cfg;
+  SVnodeDesc nodes[TSDB_MAX_REPLICA];
+} SCreateVnodeMsg, SAlterVnodeMsg;

 typedef struct {
  char    tableId[TSDB_TABLE_FNAME_LEN];
  int16_t createFlag;
  char    tags[];
-} SCMTableInfoMsg;
+} STableInfoMsg;

 typedef struct {
  int32_t numOfTables;
  char    tableIds[];
-} SCMMultiTableInfoMsg;
+} SMultiTableInfoMsg;

-typedef struct SCMSTableVgroupMsg {
+typedef struct SSTableVgroupMsg {
  int32_t numOfTables;
-} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg;
+} SSTableVgroupMsg, SSTableVgroupRspMsg;

 typedef struct {
  int32_t       vgId;
  int8_t        numOfEps;
  SEpAddr1      epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupInfo;
+} SVgroupInfo;

 typedef struct {
  int32_t    vgId;
  int8_t     numOfEps;
  SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupMsg;
+} SVgroupMsg;

 typedef struct {
  int32_t numOfVgroups;
-  SCMVgroupInfo vgroups[];
+  SVgroupInfo vgroups[];
 } SVgroupsInfo;

 typedef struct {
  int32_t numOfVgroups;
-  SCMVgroupMsg vgroups[];
+  SVgroupMsg vgroups[];
 } SVgroupsMsg;

 typedef struct STableMetaMsg {
@ -702,7 +717,7 @@ typedef struct STableMetaMsg {
  int16_t       tversion;
  int32_t       tid;
  uint64_t      uid;
-  SCMVgroupMsg  vgroup;
+  SVgroupMsg    vgroup;
  SSchema       schema[];
 } STableMetaMsg;

@ -728,38 +743,38 @@ typedef struct {
  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
  uint16_t payloadLen;
  char     payload[];
-} SCMShowMsg;
+} SShowMsg;

-typedef struct SCMShowRsp {
+typedef struct SShowRsp {
  uint64_t      qhandle;
  STableMetaMsg tableMeta;
-} SCMShowRsp;
+} SShowRsp;

 typedef struct {
-  char     ep[TSDB_EP_LEN];  // end point, hostname:port
-} SCMCreateDnodeMsg, SCMDropDnodeMsg;
+  char ep[TSDB_EP_LEN];  // end point, hostname:port
+} SCreateDnodeMsg, SDropDnodeMsg;

 typedef struct {
  int32_t dnodeId;
  char    dnodeEp[TSDB_EP_LEN];  // end point, hostname:port
-  SDMMnodeInfos mnodes;
-} SMDCreateMnodeMsg;
+  SMnodeInfos mnodes;
+} SCreateMnodeMsg;

 typedef struct {
  int32_t dnodeId;
  int32_t vgId;
  int32_t tid;
-} SDMConfigTableMsg;
+} SConfigTableMsg;

 typedef struct {
  uint32_t dnodeId;
  int32_t  vgId;
-} SDMConfigVnodeMsg;
+} SConfigVnodeMsg;

 typedef struct {
  char ep[TSDB_EP_LEN];  // end point, hostname:port
  char config[64];
-} SMDCfgDnodeMsg, SCMCfgDnodeMsg;
+} SCfgDnodeMsg;

 typedef struct {
  char     sql[TSDB_SHOW_SQL_LEN];
@ -781,13 +796,14 @@ typedef struct {
 } SStreamDesc;

 typedef struct {
+  char     clientVer[TSDB_VERSION_LEN];
  uint32_t connId;
  int32_t  pid;
  int32_t  numOfQueries;
  int32_t  numOfStreams;
  char     appName[TSDB_APPNAME_LEN];
  char     pData[];
-} SCMHeartBeatMsg;
+} SHeartBeatMsg;

 typedef struct {
  uint32_t  queryId;
@ -797,11 +813,11 @@ typedef struct {
  uint32_t  connId;
  int8_t    killConnection;
  SRpcEpSet epSet;
-} SCMHeartBeatRsp;
+} SHeartBeatRsp;

 typedef struct {
  char queryId[TSDB_KILL_MSG_LEN + 1];
-} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg;
+} SKillQueryMsg, SKillStreamMsg, SKillConnMsg;

 typedef struct {
  int32_t  vnode;
@ -810,7 +826,7 @@ typedef struct {
  uint64_t stime;  // stream starting time
  int32_t  status;
  char     tableId[TSDB_TABLE_FNAME_LEN];
-} SMDAlterStreamMsg;
+} SAlterStreamMsg;

 typedef struct {
  char user[TSDB_USER_LEN];
@ -818,7 +834,7 @@ typedef struct {
  char encrypt;
  char secret[TSDB_KEY_LEN];
  char ckey[TSDB_KEY_LEN];
-} SDMAuthMsg, SDMAuthRsp;
+} SAuthMsg, SAuthRsp;

 #pragma pack(pop)

--- a/src/inc/tcq.h
+++ b/src/inc/tcq.h
@ -21,10 +21,10 @@ extern "C" {

 #include "tdataformat.h"

-typedef int (*FCqWrite)(void *ahandle, void *pHead, int type);
+typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);

 typedef struct {
-  int      vgId;
+  int32_t  vgId;
  char     user[TSDB_USER_LEN];
  char     pass[TSDB_PASSWORD_LEN];
  char     db[TSDB_DB_NAME_LEN];
@ -42,12 +42,12 @@ void  cqStart(void *handle);
 void  cqStop(void *handle);

 // cqCreate is called by TSDB to start an instance of CQ 
-void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);

 // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
 void  cqDrop(void *handle);

-extern int cqDebugFlag;
+extern int32_t cqDebugFlag;


 #ifdef __cplusplus
--- a/src/inc/trpc.h
+++ b/src/inc/trpc.h
@ -78,18 +78,20 @@ typedef struct SRpcInit {
  int  (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
 } SRpcInit;

+int32_t rpcInit();
+void  rpcCleanup();
 void *rpcOpen(const SRpcInit *pRpc);
 void  rpcClose(void *);
 void *rpcMallocCont(int contLen);
 void  rpcFreeCont(void *pCont);
 void *rpcReallocCont(void *ptr, int contLen);
-void  rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
+void  rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
 void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
 int   rpcReportProgress(void *pConn, char *pCont, int contLen);
-void  rpcCancelRequest(void *pContext);
+void  rpcCancelRequest(int64_t rid);

 #ifdef __cplusplus
 }
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@ -46,7 +46,7 @@ extern "C" {
 typedef struct {
  void *appH;
  void *cqH;
-  int (*notifyStatus)(void *, int status);
+  int (*notifyStatus)(void *, int status, int eno);
  int (*eventCallBack)(void *);
  void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
  void (*cqDropFunc)(void *handle);
@ -65,6 +65,7 @@ typedef struct {
  int32_t maxRowsPerFileBlock;  // maximum rows per file block
  int8_t  precision;
  int8_t  compression;
+  int8_t  update;
 } STsdbCfg;

 // --------- TSDB REPOSITORY USAGE STATISTICS
@ -82,7 +83,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
 int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
 int32_t      tsdbDropRepo(char *rootDir);
 TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
-void         tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
+int          tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
 int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
 int          tsdbGetState(TSDB_REPO_T *repo);

@ -125,7 +126,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
  STsdbCfg tsdbCfg;
-  int64_t  version;            // version of the repository
+  uint64_t version;            // version of the repository
  int64_t  tsdbTotalDataSize;  // the original inserted data size
  int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
  // TODO: Other informations to add
@ -135,7 +136,7 @@ STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
 // the meter information report structure
 typedef struct {
  STableCfg tableCfg;
-  int64_t   version;
+  uint64_t  version;
  int64_t   tableTotalDataSize;  // In bytes
  int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;
@ -163,6 +164,12 @@ typedef struct STsdbQueryCond {
  SColumnInfo *colList;
 } STsdbQueryCond;

+typedef struct SMemRef {
+  int32_t  ref;
+  void    *mem;
+  void    *imem;
+} SMemRef;
+
 typedef struct SDataBlockInfo {
  STimeWindow window;
  int32_t     rows;
@ -192,7 +199,7 @@ typedef struct {
 * @param qinfo      query info handle from query processor
 * @return
 */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);

 /**
 * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@ -204,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
 * @param tableInfo  table list.
 * @return
 */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);

 /**
 * get the queried table object list
@ -222,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 * @return
 */
 TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo);
+                                               void *qinfo, SMemRef* pRef);

 /**
 * move to next block if exists
@ -314,6 +321,12 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
 */
 void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);

+int  tsdbInitCommitQueue(int nthreads);
+void tsdbDestroyCommitQueue();
+int  tsdbSyncCommit(TSDB_REPO_T *repo);
+void tsdbIncCommitRef(int vgId);
+void tsdbDecCommitRef(int vgId);
+
 #ifdef __cplusplus
 }
 #endif
--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@ -51,9 +51,9 @@ typedef struct {
 } SSyncCfg;

 typedef struct {
-  int       selfIndex;
-  uint32_t  nodeId[TAOS_SYNC_MAX_REPLICA];
-  int       role[TAOS_SYNC_MAX_REPLICA];  
+  int32_t  selfIndex;
+  uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
+  int32_t  role[TAOS_SYNC_MAX_REPLICA];
 } SNodesRole;

 /* 
@ -68,10 +68,10 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin

 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int      (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index); 
+typedef int32_t  (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); 
 
-// when a forward pkt is received, call this to handle data 
-typedef int      (*FWriteToCache)(void *ahandle, void *pHead, int type);
+// when a forward pkt is received, call this to handle data
+typedef int32_t  (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);

 // when forward is confirmed by peer, master call this API to notify app
 typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
@ -83,48 +83,47 @@ typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
 typedef void     (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);

 // when data file is synced successfully, notity app
-typedef int      (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
+typedef int32_t  (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);

 typedef struct {
-  int32_t    vgId;      // vgroup ID
-  uint64_t   version;   // initial version
-  SSyncCfg   syncCfg;   // configuration from mgmt
-  char       path[128]; // path to the file
- 
-  void      *ahandle;   // handle provided by APP 
-  FGetFileInfo    getFileInfo;
-  FGetWalInfo     getWalInfo;
-  FWriteToCache   writeToCache;
-  FConfirmForward confirmForward;
-  FNotifyRole     notifyRole;
-  FNotifyFlowCtrl notifyFlowCtrl;
+  int32_t  vgId;       // vgroup ID
+  uint64_t version;    // initial version
+  SSyncCfg syncCfg;    // configuration from mgmt
+  char     path[128];  // path to the file
+  void *   ahandle;    // handle provided by APP
+  FGetFileInfo      getFileInfo;
+  FGetWalInfo       getWalInfo;
+  FWriteToCache     writeToCache;
+  FConfirmForward   confirmForward;
+  FNotifyRole       notifyRole;
+  FNotifyFlowCtrl   notifyFlowCtrl;
  FNotifyFileSynced notifyFileSynced;
 } SSyncInfo;

-typedef void* tsync_h;
+typedef void *tsync_h;

 int32_t syncInit();
 void    syncCleanUp();

-tsync_h syncStart(const SSyncInfo *);
-void    syncStop(tsync_h shandle);
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
-void    syncRecover(tsync_h shandle);      // recover from other nodes:
-int     syncGetNodesRole(tsync_h shandle, SNodesRole *);
+int64_t syncStart(const SSyncInfo *);
+void    syncStop(int64_t rid);
+int32_t syncReconfig(int64_t rid, const SSyncCfg *);
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype);
+void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
+void    syncRecover(int64_t rid);  // recover from other nodes:
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *);

-extern  char *syncRole[];
+extern char *syncRole[];

 //global configurable parameters
-extern  int   tsMaxSyncNum;
-extern  int   tsSyncTcpThreads;
-extern  int   tsMaxWatchFiles;
-extern  int   tsSyncTimer;
-extern  int   tsMaxFwdInfo; 
-extern  int   sDebugFlag;
-extern  char  tsArbitrator[];
-extern  uint16_t tsSyncPort;
+extern int32_t  tsMaxSyncNum;
+extern int32_t  tsSyncTcpThreads;
+extern int32_t  tsMaxWatchFiles;
+extern int32_t  tsSyncTimer;
+extern int32_t  tsMaxFwdInfo;
+extern int32_t  sDebugFlag;
+extern char     tsArbitrator[];
+extern uint16_t tsSyncPort;

 #ifdef __cplusplus
 }
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@ -16,7 +16,6 @@
 #ifndef TDENGINE_TTOKENDEF_H
 #define TDENGINE_TTOKENDEF_H

-
 #define TK_ID                               1
 #define TK_BOOL                             2
 #define TK_TINYINT                          3
@ -114,114 +113,116 @@
 #define TK_FSYNC                           95
 #define TK_COMP                            96
 #define TK_PRECISION                       97
-#define TK_LP                              98
-#define TK_RP                              99
-#define TK_TAGS                           100
-#define TK_USING                          101
-#define TK_AS                             102
-#define TK_COMMA                          103
-#define TK_NULL                           104
-#define TK_SELECT                         105
-#define TK_UNION                          106
-#define TK_ALL                            107
-#define TK_FROM                           108
-#define TK_VARIABLE                       109
-#define TK_INTERVAL                       110
-#define TK_FILL                           111
-#define TK_SLIDING                        112
-#define TK_ORDER                          113
-#define TK_BY                             114
-#define TK_ASC                            115
-#define TK_DESC                           116
-#define TK_GROUP                          117
-#define TK_HAVING                         118
-#define TK_LIMIT                          119
-#define TK_OFFSET                         120
-#define TK_SLIMIT                         121
-#define TK_SOFFSET                        122
-#define TK_WHERE                          123
-#define TK_NOW                            124
-#define TK_RESET                          125
-#define TK_QUERY                          126
-#define TK_ADD                            127
-#define TK_COLUMN                         128
-#define TK_TAG                            129
-#define TK_CHANGE                         130
-#define TK_SET                            131
-#define TK_KILL                           132
-#define TK_CONNECTION                     133
-#define TK_STREAM                         134
-#define TK_COLON                          135
-#define TK_ABORT                          136
-#define TK_AFTER                          137
-#define TK_ATTACH                         138
-#define TK_BEFORE                         139
-#define TK_BEGIN                          140
-#define TK_CASCADE                        141
-#define TK_CLUSTER                        142
-#define TK_CONFLICT                       143
-#define TK_COPY                           144
-#define TK_DEFERRED                       145
-#define TK_DELIMITERS                     146
-#define TK_DETACH                         147
-#define TK_EACH                           148
-#define TK_END                            149
-#define TK_EXPLAIN                        150
-#define TK_FAIL                           151
-#define TK_FOR                            152
-#define TK_IGNORE                         153
-#define TK_IMMEDIATE                      154
-#define TK_INITIALLY                      155
-#define TK_INSTEAD                        156
-#define TK_MATCH                          157
-#define TK_KEY                            158
-#define TK_OF                             159
-#define TK_RAISE                          160
-#define TK_REPLACE                        161
-#define TK_RESTRICT                       162
-#define TK_ROW                            163
-#define TK_STATEMENT                      164
-#define TK_TRIGGER                        165
-#define TK_VIEW                           166
-#define TK_COUNT                          167
-#define TK_SUM                            168
-#define TK_AVG                            169
-#define TK_MIN                            170
-#define TK_MAX                            171
-#define TK_FIRST                          172
-#define TK_LAST                           173
-#define TK_TOP                            174
-#define TK_BOTTOM                         175
-#define TK_STDDEV                         176
-#define TK_PERCENTILE                     177
-#define TK_APERCENTILE                    178
-#define TK_LEASTSQUARES                   179
-#define TK_HISTOGRAM                      180
-#define TK_DIFF                           181
-#define TK_SPREAD                         182
-#define TK_TWA                            183
-#define TK_INTERP                         184
-#define TK_LAST_ROW                       185
-#define TK_RATE                           186
-#define TK_IRATE                          187
-#define TK_SUM_RATE                       188
-#define TK_SUM_IRATE                      189
-#define TK_AVG_RATE                       190
-#define TK_AVG_IRATE                      191
-#define TK_TBID                           192
-#define TK_SEMI                           193
-#define TK_NONE                           194
-#define TK_PREV                           195
-#define TK_LINEAR                         196
-#define TK_IMPORT                         197
-#define TK_METRIC                         198
-#define TK_TBNAME                         199
-#define TK_JOIN                           200
-#define TK_METRICS                        201
-#define TK_STABLE                         202
-#define TK_INSERT                         203
-#define TK_INTO                           204
-#define TK_VALUES                         205
+#define TK_UPDATE                          98
+#define TK_LP                              99
+#define TK_RP                             100
+#define TK_TAGS                           101
+#define TK_USING                          102
+#define TK_AS                             103
+#define TK_COMMA                          104
+#define TK_NULL                           105
+#define TK_SELECT                         106
+#define TK_UNION                          107
+#define TK_ALL                            108
+#define TK_FROM                           109
+#define TK_VARIABLE                       110
+#define TK_INTERVAL                       111
+#define TK_FILL                           112
+#define TK_SLIDING                        113
+#define TK_ORDER                          114
+#define TK_BY                             115
+#define TK_ASC                            116
+#define TK_DESC                           117
+#define TK_GROUP                          118
+#define TK_HAVING                         119
+#define TK_LIMIT                          120
+#define TK_OFFSET                         121
+#define TK_SLIMIT                         122
+#define TK_SOFFSET                        123
+#define TK_WHERE                          124
+#define TK_NOW                            125
+#define TK_RESET                          126
+#define TK_QUERY                          127
+#define TK_ADD                            128
+#define TK_COLUMN                         129
+#define TK_TAG                            130
+#define TK_CHANGE                         131
+#define TK_SET                            132
+#define TK_KILL                           133
+#define TK_CONNECTION                     134
+#define TK_STREAM                         135
+#define TK_COLON                          136
+#define TK_ABORT                          137
+#define TK_AFTER                          138
+#define TK_ATTACH                         139
+#define TK_BEFORE                         140
+#define TK_BEGIN                          141
+#define TK_CASCADE                        142
+#define TK_CLUSTER                        143
+#define TK_CONFLICT                       144
+#define TK_COPY                           145
+#define TK_DEFERRED                       146
+#define TK_DELIMITERS                     147
+#define TK_DETACH                         148
+#define TK_EACH                           149
+#define TK_END                            150
+#define TK_EXPLAIN                        151
+#define TK_FAIL                           152
+#define TK_FOR                            153
+#define TK_IGNORE                         154
+#define TK_IMMEDIATE                      155
+#define TK_INITIALLY                      156
+#define TK_INSTEAD                        157
+#define TK_MATCH                          158
+#define TK_KEY                            159
+#define TK_OF                             160
+#define TK_RAISE                          161
+#define TK_REPLACE                        162
+#define TK_RESTRICT                       163
+#define TK_ROW                            164
+#define TK_STATEMENT                      165
+#define TK_TRIGGER                        166
+#define TK_VIEW                           167
+#define TK_COUNT                          168
+#define TK_SUM                            169
+#define TK_AVG                            170
+#define TK_MIN                            171
+#define TK_MAX                            172
+#define TK_FIRST                          173
+#define TK_LAST                           174
+#define TK_TOP                            175
+#define TK_BOTTOM                         176
+#define TK_STDDEV                         177
+#define TK_PERCENTILE                     178
+#define TK_APERCENTILE                    179
+#define TK_LEASTSQUARES                   180
+#define TK_HISTOGRAM                      181
+#define TK_DIFF                           182
+#define TK_SPREAD                         183
+#define TK_TWA                            184
+#define TK_INTERP                         185
+#define TK_LAST_ROW                       186
+#define TK_RATE                           187
+#define TK_IRATE                          188
+#define TK_SUM_RATE                       189
+#define TK_SUM_IRATE                      190
+#define TK_AVG_RATE                       191
+#define TK_AVG_IRATE                      192
+#define TK_TBID                           193
+#define TK_SEMI                           194
+#define TK_NONE                           195
+#define TK_PREV                           196
+#define TK_LINEAR                         197
+#define TK_IMPORT                         198
+#define TK_METRIC                         199
+#define TK_TBNAME                         200
+#define TK_JOIN                           201
+#define TK_METRICS                        202
+#define TK_STABLE                         203
+#define TK_INSERT                         204
+#define TK_INTO                           205
+#define TK_VALUES                         206
+

 #define TK_SPACE                          300
 #define TK_COMMENT                        301
--- a/src/inc/ttype.h
+++ b/src/inc/ttype.h
@ -0,0 +1,36 @@
+#ifndef TDENGINE_TTYPE_H
+#define TDENGINE_TTYPE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosdef.h"
+
+#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
+  switch (_type) {                       \
+    case TSDB_DATA_TYPE_TINYINT:         \
+      (_v) = (_finalType)GET_INT8_VAL(_data);        \
+      break;                             \
+    case TSDB_DATA_TYPE_SMALLINT:        \
+      (_v) = (_finalType)GET_INT16_VAL(_data);       \
+      break;                             \
+    case TSDB_DATA_TYPE_BIGINT:          \
+      (_v) = (_finalType)(GET_INT64_VAL(_data));     \
+      break;                             \
+    case TSDB_DATA_TYPE_FLOAT:           \
+      (_v) = (_finalType)GET_FLOAT_VAL(_data);       \
+      break;                             \
+    case TSDB_DATA_TYPE_DOUBLE:          \
+      (_v) = (_finalType)GET_DOUBLE_VAL(_data);      \
+      break;                             \
+    default:                             \
+      (_v) = (_finalType)GET_INT32_VAL(_data);       \
+      break;                             \
+  };
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TTYPE_H
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`Subproject commit b62a26ecc164a310104df57691691b237e091c89`