Merge branch 'develop' into test/jenkins

2020-11-05 11:01:54 +08:00 · 2020-11-05 11:01:54 +08:00 · 9d45a352e6
parent 3dc6e228ee a8bdb87c90
commit 9d45a352e6
154 changed files with 5927 additions and 3398 deletions
--- a/119
+++ b/119
@ -86,6 +86,13 @@ pipeline {
                ./crash_gen.sh -a -p -t 4 -s 2000
                '''
            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4
+                ./handle_crash_gen_val_log.sh
+                '''
+            }
            sh '''
            date
            cd ${WKC}/tests
@ -131,14 +138,33 @@ pipeline {
            sh'''
            cd ${WORKSPACE}
            git checkout develop
-            cd tests/gotest
-            bash batchtest.sh
-            cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
-            mvn clean package assembly:single >/dev/null 
-            java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
-            cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
-            python3 PythonChecker.py
            '''
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/gotest
+                bash batchtest.sh
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+                python3 PythonChecker.py
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
+                mvn clean package assembly:single >/dev/null 
+                java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
+                dotnet run
+                '''
+            }
+          
         }
       }

@ -146,5 +172,82 @@ pipeline {
    }

  }
-  
+  post {             
+        success {
+            emailext (
+                subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                body: '''<!DOCTYPE html>
+                <html>
+                <head>
+                <meta charset="UTF-8">
+                </head>
+                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
+                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
+                        <tr>
+                            <td><br />
+                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
+                                <hr size="2" width="100%" align="center" /></td>
+                        </tr>
+                        <tr>
+                            <td>
+                                <ul>
+                                <div style="font-size:18px">
+                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建编号：${BUILD_NUMBER}</li>
+                                    <li>触发用户：${CAUSE}</li>
+                                    <li>变更概要：${CHANGES}</li>
+                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
+                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
+                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                </div>
+                                </ul>
+                            </td>
+                        </tr>
+                    </table></font>
+                </body>
+                </html>''',
+                to: "yqliu@taosdata.com,pxiao@taosdata.com",
+                from: "support@taosdata.com"
+            )
+        }
+        failure {
+            emailext (
+                subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+                body: '''<!DOCTYPE html>
+                <html>
+                <head>
+                <meta charset="UTF-8">
+                </head>
+                <body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
+                    <table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
+                        <tr>
+                            <td><br />
+                                <b><font color="#0B610B"><font size="6">构建信息</font></font></b>
+                                <hr size="2" width="100%" align="center" /></td>
+                        </tr>
+                        <tr>
+                            <td>
+                                <ul>
+                                <div style="font-size:18px">
+                                    <li>构建名称>>分支：${PROJECT_NAME}</li>
+                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建编号：${BUILD_NUMBER}</li>
+                                    <li>触发用户：${CAUSE}</li>
+                                    <li>变更概要：${CHANGES}</li>
+                                    <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
+                                    <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
+                                    <li>变更集：${JELLY_SCRIPT}</li>
+                                </div>
+                                </ul>
+                            </td>
+                        </tr>
+                    </table></font>
+                </body>
+                </html>''',
+                to: "yqliu@taosdata.com,pxiao@taosdata.com",
+                from: "support@taosdata.com"
+            )
+        }
+    }
 }
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。
 - dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。
 - logDir：日志文件目录，客户端和服务器的运行日志文件将写入该目录。默认值：/var/log/taos。
+- tempDir：临时文件目录，客户端和服务器的临时文件（主要是查询时用于保存中间结果的问题）将写入该目录。 默认值：Linux下为 /tmp/，Windows下为环境变量 tmp 或 temp 指向的目录。
 - arbitrator：系统中裁决器的end point, 缺省值为空。
 - role：dnode的可选角色。0-any; 既可作为mnode，也可分配vnode；1-mgmt;只能作为mnode，不能分配vnode；2-dnode;不能作为mnode，只能分配vnode
 - debugFlag：运行日志开关。131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志）。默认值：131或135（不同模块有不同的默认值）。
--- a/documentation20/webdocs/markdowndocs/TAOS
+++ b/documentation20/webdocs/markdowndocs/TAOS
@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数

 - **PERCENTILE**
    ```mysql
-    SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause];
+    SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
    ```
    功能说明：统计表中某列的值百分比分位数。  
    返回结果数据类型： 双精度浮点数Double。  
--- a/documentation20/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation20/webdocs/markdowndocs/administrator-ch.md
@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景
 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
 ```

-示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。 
+示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 

 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。

--- a/documentation20/webdocs/markdowndocs/connector-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-ch.md
@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间
 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式
 - httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131

+## CSharp Connector
+
+在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
+
+#### 安装TDengine客户端
+
+C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。
+
+安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件
+
+- TDengineDriver.cs 调用taos.dll文件的Native C方法
+- TDengineTest.cs 参考程序示例
+
+在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件
+
+#### 使用方法
+
+- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
+- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法
+- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中
+
+#### 注意事项
+
+- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。
+- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。
+
+#### 第三方驱动
+
+Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考
+
+```
+//接口下载
+https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos   
+//用法说明    
+https://www.taosdata.com/blog/2020/11/02/1901.html                    
+```
+

 ## Go Connector

--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@ -38,9 +38,9 @@

 6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的

-7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
+7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 

-8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)

 9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
   检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@ -20,6 +20,9 @@
 # data file's directory
 # dataDir                   /var/lib/taos

+# temporary file's directory
+# tempDir                   /tmp/
+
 # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only   
 # arbitrator                arbitrator_hostname:6042     

@ -256,3 +259,5 @@
 # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
 # maxBinaryDisplayWidth 30

+# enable/disable telemetry reporting
+# telemetryReporting    1
--- a/packaging/rpm/tdengine.spec
+++ b/packaging/rpm/tdengine.spec
@ -2,7 +2,7 @@
 %define cfg_install_dir  /etc/taos
 %define __strip /bin/true

-Name:		TDengine
+Name:		tdengine
 Version:	%{_version}
 Release:	3%{?dist}
 Summary:	tdengine from taosdata
--- a/packaging/tools/post.sh
+++ b/packaging/tools/post.sh
@ -121,8 +121,11 @@ function install_config() {
    echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}"
    echo
    echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
-    read firstEp
-    while true; do
+    #read firstEp
+    if exec < /dev/tty; then
+    	read firstEp;
+	fi
+	while true; do
        if [ ! -z "$firstEp" ]; then
            # check the format of the firstEp
            #if [[ $firstEp == $FQDN_PATTERN ]]; then
--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@ -490,7 +490,7 @@ static bool balanceMontiorDropping() {

    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
-      if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; 
+      if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; 
      if (mnodeGetDnodesNum() <= 1) continue;

      mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
--- a/src/client/inc/tscLocalMerge.h
+++ b/src/client/inc/tscLocalMerge.h
@ -64,9 +64,8 @@ typedef struct SLocalReducer {
  SColumnModel *         resColModel;
  tExtMemBuffer **       pExtMemBuffer;      // disk-based buffer
  SFillInfo*             pFillInfo;          // interpolation support structure
-  char *                 pFinalRes;          // result data after interpo
-  tFilePage *            discardData;
-  SResultInfo *          pResInfo;
+  char*                  pFinalRes;          // result data after interpo
+  tFilePage*             discardData;
  bool                   discard;
  int32_t                offset;             // limit offset value
  bool                   orderPrjOnSTable;   // projection query on stable
--- a/src/client/inc/tscSubquery.h
+++ b/src/client/inc/tscSubquery.h
@ -23,7 +23,7 @@ extern "C" {
 #include "tscUtil.h"
 #include "tsclient.h"

-void tscFetchDatablockFromSubquery(SSqlObj* pSql);
+void tscFetchDatablockForSubquery(SSqlObj* pSql);

 void tscSetupOutputColumnIndex(SSqlObj* pSql);
 void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@ -75,6 +75,7 @@ typedef struct SJoinSupporter {
  SArray*         exprList;
  SFieldInfo      fieldsInfo;
  STagCond        tagCond;
+  SSqlGroupbyExpr groupInfo;       // group by info
  struct STSBuf*  pTSBuf;          // the TSBuf struct that holds the compressed timestamp array
  FILE*           f;               // temporary file in order to create TSBuf
  char            path[PATH_MAX];  // temporary file path, todo dynamic allocate memory
@ -86,8 +87,8 @@ typedef struct SJoinSupporter {
 } SJoinSupporter;

 typedef struct SVgroupTableInfo {
-  SCMVgroupInfo vgInfo;
-  SArray*       itemList;   //SArray<STableIdInfo>
+  SVgroupInfo vgInfo;
+  SArray*     itemList;   //SArray<STableIdInfo>
 } SVgroupTableInfo;

 static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
@ -225,8 +226,9 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo);

 void tscClearSubqueryInfo(SSqlCmd* pCmd);
 void tscFreeVgroupTableInfo(SArray* pVgroupTables);
-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables);
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables);
 void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);

 int  tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
 int  tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
@ -237,7 +239,7 @@ void tscDoQuery(SSqlObj* pSql);

 SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
 void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src);
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
 /**
 * The create object function must be successful expect for the out of memory issue.
 *
@ -265,6 +267,7 @@ void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub
 void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);

 int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);

 void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);

--- a/src/client/inc/tschemautil.h
+++ b/src/client/inc/tschemautil.h
@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
 * @param colId
 * @return
 */
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);

 /**
 * check if the schema is valid or not, including following aspects:
@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema();
 */
 STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);

-//todo tags value as well as the table id structure needs refactor
-char *tsGetTagsValue(STableMeta *pMeta);
-
 #ifdef __cplusplus
 }
 #endif
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@ -90,12 +90,12 @@ typedef struct STableComInfo {
  int32_t rowSize;
 } STableComInfo;

-typedef struct SCMCorVgroupInfo {
-  int32_t    version;
-  int8_t     inUse;
-  int8_t     numOfEps;
-  SEpAddr1   epAddr[TSDB_MAX_REPLICA];
-} SCMCorVgroupInfo;
+typedef struct SCorVgroupInfo {
+  int32_t  version;
+  int8_t   inUse;
+  int8_t   numOfEps;
+  SEpAddr1 epAddr[TSDB_MAX_REPLICA];
+} SCorVgroupInfo;

 typedef struct STableMeta {
  STableComInfo  tableInfo;
@ -103,8 +103,8 @@ typedef struct STableMeta {
  int16_t        sversion;
  int16_t        tversion;
  char           sTableId[TSDB_TABLE_FNAME_LEN];
-  SCMVgroupInfo  vgroupInfo;
-  SCMCorVgroupInfo  corVgroupInfo;
+  SVgroupInfo    vgroupInfo;
+  SCorVgroupInfo corVgroupInfo;
  STableId       id;
  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
 } STableMeta;
@ -128,7 +128,7 @@ typedef struct STableMetaInfo {
 typedef struct SSqlExpr {
  char      aliasName[TSDB_COL_NAME_LEN];  // as aliasName
  SColIndex colInfo;
-  int64_t   uid;            // refactor use the pointer
+  uint64_t  uid;            // refactor use the pointer
  int16_t   functionId;     // function id in aAgg array
  int16_t   resType;        // return value type
  int16_t   resBytes;       // length of return value
@ -339,9 +339,9 @@ typedef struct STscObj {
 } STscObj;

 typedef struct SSubqueryState {
-  int32_t          numOfRemain;         // the number of remain unfinished subquery
-  int32_t          numOfSub;            // the number of total sub-queries
-  uint64_t         numOfRetrievedRows;  // total number of points in this query
+  int32_t  numOfRemain;         // the number of remain unfinished subquery
+  int32_t  numOfSub;            // the number of total sub-queries
+  uint64_t numOfRetrievedRows;  // total number of points in this query
 } SSubqueryState;

 typedef struct SSqlObj {
@ -431,14 +431,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
 */
 void tscFreeSqlResult(SSqlObj *pSql);

-/**
- * only free part of resources allocated during query.
- * TODO remove it later
- * Note: this function is multi-thread safe.
- * @param pObj
- */
-void tscPartiallyFreeSqlObj(SSqlObj *pSql);
-
 /**
 * free sql object, release allocated resource
 * @param pObj
@ -523,7 +515,6 @@ extern SRpcCorEpSet tscMgmtEpSet;

 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

-int32_t tscCompareTidTags(const void* p1, const void* p2);
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);

 #ifdef __cplusplus
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
  }

  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscFetchDatablockFromSubquery(pSql);
+    tscFetchDatablockForSubquery(pSql);
  } else {
    tscProcessSql(pSql);
  }
@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) {
  
  // handle the sub queries of join query
  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscFetchDatablockFromSubquery(pSql);
+    tscFetchDatablockForSubquery(pSql);
  } else if (pRes->completed) {
    if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) {
      if (hasMoreVnodesToTry(pSql)) {  // sequentially retrieve data from remain vnodes.
@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
  SSqlRes *pRes = &pSql->res;
  pRes->code = code;

-  const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
+  SSqlObj *sub = (SSqlObj*) res;
+  const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
  if (code != TSDB_CODE_SUCCESS) {
    tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
    goto _error;
@ -427,8 +428,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
      } else {
        assert(code == TSDB_CODE_SUCCESS);      
      }
-     
-      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL);
+      // param already freed by other routine and pSql in tscCache when ctrl + c 
+      if (atomic_load_ptr(&pSql->param) == NULL) {
+        return;
+      }
+      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));

      SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
      SSqlObj *         pParObj = trs->pParentSql;
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@ -49,82 +49,6 @@ typedef struct SCreateBuilder {
 } SCreateBuilder; 
 static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);

-static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) {
-  char buf[512] = {0};
-
-  int32_t len = 0;
-  int32_t MAX_BOOL_TYPE_LENGTH = 5;  // max(strlen("true"), strlen("false"));
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-      return length;
-    case TSDB_DATA_TYPE_NCHAR:
-      return length;
-    case TSDB_DATA_TYPE_DOUBLE: {
-      double dv = 0;
-      dv = GET_DOUBLE_VAL(pData);
-      len = sprintf(buf, "%lf", dv);
-      if (strncasecmp("nan", buf, 3) == 0) {
-        len = 4;
-      }
-    } break;
-    case TSDB_DATA_TYPE_FLOAT: {
-      float fv = 0;
-      fv = GET_FLOAT_VAL(pData);
-      len = sprintf(buf, "%f", fv);
-      if (strncasecmp("nan", buf, 3) == 0) {
-        len = 4;
-      }
-    } break;
-    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_BIGINT:
-      len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
-      break;
-    case TSDB_DATA_TYPE_BOOL:
-      len = MAX_BOOL_TYPE_LENGTH;
-      break;
-    default:
-      len = sprintf(buf, "%d", *(int32_t *)pData);
-      break;
-  };
-  return len;
-}
-
-/*
- * we need to convert all data into string, so we need to sprintf all kinds of
- * non-string data into string, and record its length to get the right
- * maximum length. The length may be less or greater than its original binary length:
- * For example:
- * length((short) 1) == 1, less than sizeof(short)
- * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t)
- */
-static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) {
-  STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta;
-
-  if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE ||
-      pMeta->tableType == TSDB_STREAM_TABLE) {
-    return 0;
-  }
-
-  char *   pTagValue = tsGetTagsValue(pMeta);
-  SSchema *pTagsSchema = tscGetTableTagSchema(pMeta);
-
-  int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type);
-
-  pTagValue += pTagsSchema[0].bytes;
-  int32_t numOfTags = tscGetNumOfTags(pMeta);
-  
-  for (int32_t i = 1; i < numOfTags; ++i) {
-    int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type);
-    if (len < tLen) {
-      len = tLen;
-    }
-
-    pTagValue += pTagsSchema[i].bytes;
-  }
-
-  return len;
-}
-
 static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
  SSqlRes *pRes = &pSql->res;

@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
    return 0;
  }

-  // the following is handle display tags value for meters created according to metric
-  char *pTagValue = tsGetTagsValue(pMeta);
+  // the following is handle display tags for table created according to super table
  for (int32_t i = numOfRows; i < totalNumOfRows; ++i) {
    // field name
    TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
    char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
    const char *src = "TAG";
    STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
-
-    pTagValue += pSchema[i].bytes;
  }

  return 0;
@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
  const int32_t TYPE_COLUMN_LENGTH = 16;
  const int32_t NOTE_COLUMN_MIN_LENGTH = 8;

-  int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql);
-  if (noteFieldLen == 0) {
-    noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
-  }
+  int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql);
+//  if (noteFieldLen == 0) {
+//    noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
+//  }

  int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
  tscFieldInfoUpdateOffset(pQueryInfo);
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@ -99,12 +99,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
      pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
    }

-    SResultInfo *pResInfo = &pReducer->pResInfo[i];
-    pResInfo->bufLen = pExpr->interBytes;
-    pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen);
-
-    pCtx->resultInfo = &pReducer->pResInfo[i];
-    pCtx->resultInfo->superTableQ = true;
+    pCtx->interBufBytes = pExpr->interBytes;
+    pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo));
+    pCtx->stableQuery = true;
  }

  int16_t          n = 0;
@ -345,7 +342,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
  size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
  
  pReducer->pTempBuffer->num = 0;
-  pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));

  tscCreateResPointerInfo(pRes, pQueryInfo);
  tscInitSqlContext(pCmd, pReducer, pDesc);
@ -489,13 +485,15 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
      tscDebug("%p waiting for delete procedure, status: %d", pSql, status);
    }

-    pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
+    pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo);

    if (pLocalReducer->pCtx != NULL) {
      for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
        SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];

        tVariantDestroy(&pCtx->tag);
+        taosTFree(pCtx->resultInfo);
+
        if (pCtx->tagInfo.pTagCtxList != NULL) {
          taosTFree(pCtx->tagInfo.pTagCtxList);
        }
@ -509,15 +507,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
    taosTFree(pLocalReducer->pTempBuffer);
    taosTFree(pLocalReducer->pResultBuf);

-    if (pLocalReducer->pResInfo != NULL) {
-      size_t num = tscSqlExprNumOfExprs(pQueryInfo);
-      for (int32_t i = 0; i < num; ++i) {
-        taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
-      }
-
-      taosTFree(pLocalReducer->pResInfo);
-    }
-
    if (pLocalReducer->pLoserTree) {
      taosTFree(pLocalReducer->pLoserTree->param);
      taosTFree(pLocalReducer->pLoserTree);
@ -1072,7 +1061,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
      continue;
    }

-    SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
+    SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
    if (maxOutput < pResInfo->numOfRes) {
      maxOutput = pResInfo->numOfRes;
    }
@ -1253,10 +1242,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
  return true;
 }

-void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {  // reset output buffer to the beginning
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    pLocalReducer->pCtx[i].aOutputBuf =
-        pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity;
+void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning
+  size_t t = tscSqlExprNumOfExprs(pQueryInfo);
+  for (int32_t i = 0; i < t; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity;
  }

  memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage));
@ -1501,8 +1491,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
          if (pLocalReducer->discard && sameGroup) {
            pLocalReducer->hasUnprocessedRow = false;
            tmpBuffer->num = 0;
-          } else {
-            // current row does not belongs to the previous group, so it is not be handled yet.
+          } else { // current row does not belongs to the previous group, so it is not be handled yet.
            pLocalReducer->hasUnprocessedRow = true;
          }

--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
  }

  int32_t code = TSDB_CODE_TSC_INVALID_SQL;
-  char *  tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
+  char *  tmpTokenBuf = calloc(1, 16*1024);  // used for deleting Escape character: \\, \', \"
  if (NULL == tmpTokenBuf) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
 }

 int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
-  SCMHeartBeatMsg *pHeartbeat = pMsg;
+  SHeartBeatMsg *pHeartbeat = pMsg;
  int allocedQueriesNum = pHeartbeat->numOfQueries;
  int allocedStreamsNum = pHeartbeat->numOfStreams;
  
@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
  }

  int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
-                   sizeof(SCMHeartBeatMsg);
+                   sizeof(SHeartBeatMsg);
  pHeartbeat->connId = htonl(pObj->connId);
  pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
  pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@ -65,7 +65,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
 static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);

 static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength);
-static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName);

 static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
@ -80,9 +79,9 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);

 static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);

-static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
 static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);

 static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem);

@ -114,7 +113,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString);
 static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
 static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
 static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate);
+static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate);

 static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex);

@ -417,7 +416,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {

      char* pMsg = pCmd->payload;

-      SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
+      SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg;
      pDCL->a[0].n = strdequote(pDCL->a[0].z);
      
      strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
@ -616,14 +615,20 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) {
  return false;
 }

-int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
  const char* msg1 = "invalid query expression";
  const char* msg2 = "interval cannot be less than 10 ms";
+  const char* msg3 = "sliding cannot be used without interval";
+
+  SSqlCmd* pCmd = &pSql->cmd;

  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
  
  if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) {
+    if (pQuerySql->sliding.n > 0) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+    }
    return TSDB_CODE_SUCCESS;
  }

@ -656,7 +661,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
      return TSDB_CODE_TSC_INVALID_SQL;
    }

-    if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+    if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
      return TSDB_CODE_TSC_INVALID_SQL;
    }

@ -709,7 +714,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
    return TSDB_CODE_TSC_INVALID_SQL;
  }

-  if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+  if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_TSC_INVALID_SQL;
  }

@ -767,13 +772,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue
  return TSDB_CODE_SUCCESS;
 }

-int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
  const char* msg0 = "sliding value too small";
  const char* msg1 = "sliding value no larger than the interval value";
  const char* msg2 = "sliding value can not less than 1% of interval value";
-  const char* msg3 = "does not support sliding when interval is natual month/year";
+  const char* msg3 = "does not support sliding when interval is natural month/year";
+  const char* msg4 = "sliding not support yet in ordinary query";

  const static int32_t INTERVAL_SLIDING_FACTOR = 100;
+  SSqlCmd* pCmd = &pSql->cmd;

  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
@ -806,6 +813,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
  }

+  if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+  }
+
  return TSDB_CODE_SUCCESS;
 }

@ -1211,7 +1222,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex*
  tscColumnListInsert(pQueryInfo->colList, &tsCol);
 }
 static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
-  const char* msg1 = "invalid column name, or illegal column type";
+  const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables";
  const char* msg2 = "invalid arithmetic expression in select clause";
  const char* msg3 = "tag columns can not be used in arithmetic expression";
  const char* msg4 = "columns from different table mixed up in arithmetic expression";
@ -1625,16 +1636,15 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
 }

 static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
-    char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
+                                       const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
  const char* msg1 = "not support column types";

  int16_t type = 0;
  int16_t bytes = 0;
-  char    columnName[TSDB_COL_NAME_LEN] = {0};
  int32_t functionID = cvtFunc.execFuncId;

  if (functionID == TSDB_FUNC_SPREAD) {
-    int32_t t1 = pSchema[pColIndex->columnIndex].type;
+    int32_t t1 = pSchema->type;
    if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) {
      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
      return -1;
@ -1643,18 +1653,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
      bytes = tDataTypeDesc[type].nSize;
    }
  } else {
-    type = pSchema[pColIndex->columnIndex].type;
-    bytes = pSchema[pColIndex->columnIndex].bytes;
+    type = pSchema->type;
+    bytes = pSchema->bytes;
  }
  
-  if (aliasName != NULL) {
-    tstrncpy(columnName, aliasName, sizeof(columnName));
-  } else {
-    getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
-  }
-
  SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
-  tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
+  tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName));

  if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) {
    pExpr->colInfo.flag |= TSDB_COL_NULL;
@ -1674,7 +1678,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
  // if it is not in the final result, do not add it
  SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
  if (finalResult) {
-    insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr);
+    insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr);
  } else {
    tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0]));
  }
@ -1682,6 +1686,23 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
  return TSDB_CODE_SUCCESS;
 }

+void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) {
+  if (pItem->aliasName != NULL) {
+    tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
+  } else {
+    char uname[TSDB_COL_NAME_LEN] = {0};
+    int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN);
+    tstrncpy(uname, pToken->z, len);
+
+    int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1;
+    char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0};
+
+    snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
+
+    tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
+  }
+}
+
 int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) {
  STableMetaInfo* pTableMetaInfo = NULL;
  int32_t optr = pItem->pNode->nSQLOptr;
@ -1939,8 +1960,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col

          SColumnIndex index = COLUMN_INDEX_INITIALIZER;

-          if (pParamElem->pNode->nSQLOptr == TK_ALL) {
-            // select table.*
+          if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.*
            SStrToken tmpToken = pParamElem->pNode->colInfo;

            if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
@ -1950,9 +1970,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
            pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
            SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);

+            char name[TSDB_COL_NAME_LEN] = {0};
            for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
              index.columnIndex = j;
-              if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) {
+              SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)};
+              setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+              if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
                return TSDB_CODE_TSC_INVALID_SQL;
              }
            }
@ -1963,14 +1987,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
            }

            pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
-            SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);

            // functions can not be applied to tags
            if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) {
              return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
            }

-            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) {
+            char name[TSDB_COL_NAME_LEN] = {0};
+
+            SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
+            setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo);
+
+            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) {
              return TSDB_CODE_TSC_INVALID_SQL;
            }

@ -2007,7 +2035,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col

          for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) {
            SColumnIndex index = {.tableIndex = j, .columnIndex = i};
-            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) {
+
+            char name[TSDB_COL_NAME_LEN] = {0};
+            SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)};
+            setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+            if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
              return TSDB_CODE_TSC_INVALID_SQL;
            }

@ -2240,10 +2273,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt
  }
 }

-void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) {
-  snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName);
-}
-
 static bool isTablenameToken(SStrToken* token) {
  SStrToken tmpToken = *token;
  SStrToken tableToken = {0};
@ -2728,7 +2757,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
 int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
  const char* msg1 = "too many columns in group by clause";
  const char* msg2 = "invalid column name in group by clause";
-//  const char* msg3 = "group by columns must belong to one table";
+  const char* msg3 = "columns from one table allowed as group by columns";
  const char* msg7 = "not support group by expression";
  const char* msg8 = "not allowed column type for group by";
  const char* msg9 = "tags not allowed for table query";
@ -2764,7 +2793,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
    }

-    tableIndex = index.tableIndex;
+    if (tableIndex == COLUMN_INDEX_INITIAL_VAL) {
+      tableIndex = index.tableIndex;
+    } else if (tableIndex != index.tableIndex) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+    }

    pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
    pTableMeta = pTableMetaInfo->pTableMeta;
@ -2803,7 +2836,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
      tscColumnListInsert(pTableMetaInfo->tagColList, &index);
    } else {
      // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by
-      if (pSchema->type > TSDB_DATA_TYPE_BINARY) {
+      if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
      }

@ -3349,7 +3382,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
  return TSDB_CODE_SUCCESS;
 }

-static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) {
+static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList,
+    int32_t* type, uint64_t* uid) {
  if (pExpr->nSQLOptr == TK_ID) {
    if (*type == NON_ARITHMEIC_EXPR) {
      *type = NORMAL_ARITHMETIC;
@ -3398,13 +3432,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
    }

    // Not supported data type in arithmetic expression
+    uint64_t id = -1;
    for(int32_t i = 0; i < inc; ++i) {
      SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex);
      int16_t t = p1->resType;
      if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) {
        return TSDB_CODE_TSC_INVALID_SQL;
      }
+
+      if (i == 0) {
+        id = p1->uid;
+      } else if (id != p1->uid){
+        return TSDB_CODE_TSC_INVALID_SQL;
+      }
    }
+
+    *uid = id;
  }

  return TSDB_CODE_SUCCESS;
@ -3416,13 +3459,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
  }

  tSQLExpr* pLeft = pExpr->pLeft;
+  uint64_t uidLeft = 0;
+  uint64_t uidRight = 0;
+
  if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) {
    int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
    if (ret != TSDB_CODE_SUCCESS) {
      return ret;
    }
  } else {
-    int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
+    int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft);
    if (ret != TSDB_CODE_SUCCESS) {
      return ret;
    }
@ -3435,10 +3481,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
      return ret;
    }
  } else {
-    int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type);
+    int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight);
    if (ret != TSDB_CODE_SUCCESS) {
      return ret;
    }
+
+    // the expression not from the same table, return error
+    if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
+      return TSDB_CODE_TSC_INVALID_SQL;
+    }
  }

  return TSDB_CODE_SUCCESS;
@ -4424,8 +4475,8 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
 int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) {
  const char* msg0 = "only support order by primary timestamp";
  const char* msg1 = "invalid column name";
-  const char* msg2 = "only support order by primary timestamp and queried column";
-  const char* msg3 = "only support order by primary timestamp and first tag in groupby clause";
+  const char* msg2 = "only support order by primary timestamp or queried column";
+  const char* msg3 = "only support order by primary timestamp or first tag in groupby clause";

  setDefaultOrderInfo(pQueryInfo);
  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -5186,7 +5237,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
  return TSDB_CODE_SUCCESS;
 }

-static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
  const char* msg = "invalid number of options";

  pMsg->daysToKeep = htonl(-1);
@ -5217,7 +5268,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo*
  return TSDB_CODE_SUCCESS;
 }

-static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
+static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
  const char* msg = "invalid time precision";

  pMsg->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond by default
@ -5241,7 +5292,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn
  return TSDB_CODE_SUCCESS;
 }

-static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
  pMsg->maxTables = htonl(-1);  // max tables can not be set anymore
  pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
  pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
@ -5258,7 +5309,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
 }

 int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
-  SCMCreateDbMsg* pMsg = (SCMCreateDbMsg*)(pCmd->payload);
+  SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload);
  setCreateDBOption(pMsg, pCreateDbSql);

  if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) {
@ -5281,20 +5332,26 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau

  if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) {
    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex);
+    SSqlExpr* pExpr = NULL;
+
    size_t size = taosArrayGetSize(pQueryInfo->exprList);
-  
-    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+    if (size > 0) {
+      pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+    }

-    if (pExpr->functionId != TSDB_FUNC_TAG) {
-      STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
-      int16_t         columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
-      SColumnIndex    index = {.tableIndex = 0, .columnIndex = columnInfo};
-      SSchema*        pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+    if (pExpr == NULL || pExpr->functionId != TSDB_FUNC_TAG) {
+      STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex);
+
+      int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
+
+      SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId);
+      int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId);
+      SColumnIndex    index = {.tableIndex = 0, .columnIndex = colIndex};
+
+      char*   name = pTagSchema->name;
+      int16_t type = pTagSchema->type;
+      int16_t bytes = pTagSchema->bytes;

-      int16_t type = pSchema[index.columnIndex].type;
-      int16_t bytes = pSchema[index.columnIndex].bytes;
-      char*   name = pSchema[index.columnIndex].name;
-  
      pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
      pExpr->colInfo.flag = TSDB_COL_TAG;

@ -5770,7 +5827,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
 }

 // can only perform the parameters based on the macro definitation
-int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
+int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) {
  char msg[512] = {0};

  if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
@ -6094,7 +6151,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
  }

  // set interval value
-  if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+  if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_TSC_INVALID_SQL;
  } else {
    if ((pQueryInfo->interval.interval > 0) &&
@ -6302,7 +6359,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
  }

  // set interval value
-  if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+  if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
    return TSDB_CODE_TSC_INVALID_SQL;
  } else {
    if ((pQueryInfo->interval.interval > 0) &&
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex)
 }

 // TODO for large number of columns, employ the binary search method
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
  STableComInfo tinfo = tscGetTableInfo(pTableMeta);

  for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
@ -140,7 +140,7 @@ struct SSchema tscGetTbnameColumnSchema() {
  strcpy(s.name, TSQL_TBNAME_L);
  return s;
 }
-static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) {
+static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
  corVgroupInfo->version = 0;
  corVgroupInfo->inUse = 0;
  corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
@ -166,7 +166,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
  pTableMeta->id.tid = pTableMetaMsg->tid;
  pTableMeta->id.uid = pTableMetaMsg->uid;

-  SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
+  SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
  pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
  pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;

@ -197,28 +197,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
  return pTableMeta;
 }

-/**
- * the TableMeta data format in memory is as follows:
- *
- * +--------------------+
- * |STableMeta Body data|  sizeof(STableMeta)
- * +--------------------+
- * |Schema data         |  numOfTotalColumns * sizeof(SSchema)
- * +--------------------+
- * |Tags data           |  tag_col_1.bytes + tag_col_2.bytes + ....
- * +--------------------+
- *
- * @param pTableMeta
- * @return
- */
-char* tsGetTagsValue(STableMeta* pTableMeta) {
-  int32_t offset = 0;
-//  int32_t  numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags;
-//  uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema);
-
-  return ((char*)pTableMeta + offset);
-}
-
 // todo refactor
 UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
  for (int32_t i = 0; i < num; ++i) {
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) {
  return initial * (2<<(count - 2));
 }

-static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
+static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
  assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);

  SRpcEpSet* pEpSet = &pSql->epSet;
@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
  tscMgmtEpSet.epSet = *pEpSet;
  taosCorEndWrite(&tscMgmtEpSet.version);
 }
-static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
+static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
  if (pVgroupInfo == NULL) { return;}
  taosCorBeginRead(&pVgroupInfo->version);
  int8_t inUse = pVgroupInfo->inUse;
@ -117,7 +117,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
  SSqlCmd *pCmd = &pObj->cmd;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
-  SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
+  SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;

  taosCorBeginWrite(&pVgroupInfo->version);
  tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
  if (pObj == NULL) return;

  if (pObj != pObj->signature) {
-    tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
+    tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
    return;
  }

@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
  SSqlRes *pRes = &pSql->res;

  if (code == 0) {
-    SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
-    SRpcEpSet *      epSet = &pRsp->epSet;
+    SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp;
+    SRpcEpSet *    epSet = &pRsp->epSet;
    if (epSet->numOfEps > 0) {
      tscEpSetHtons(epSet);
      tscUpdateMgmtEpSet(epSet);
-    } 
+    }

    pSql->pTscObj->connId = htonl(pRsp->connId);

@ -175,12 +175,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
      if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
    }
  } else {
-    tscDebug("heartbeat failed, code:%s", tstrerror(code));
+    tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
  }

  if (pObj->pHb != NULL) {
    int32_t waitingDuring = tsShellActivityTimer * 500;
-    tscDebug("%p start heartbeat in %dms", pSql, waitingDuring);
+    tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);

    taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
  } else {
@ -208,6 +208,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {

  assert(*pHB->self == pHB);

+  pHB->retry = 0;
  int32_t code = tscProcessSql(pHB);
  taosCacheRelease(tscObjCache, (void**) &p, false);

@ -552,11 +553,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
  SQueryInfo *         pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);

  int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-  
-  size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+  size_t  numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
  int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs);
-  
-  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
+
+  int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0;
+
+  int32_t tableSerialize = 0;
+  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  if (pTableMetaInfo->pVgroupTables != NULL) {
+    size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+
+    int32_t totalTables = 0;
+    for (int32_t i = 0; i < numOfGroups; ++i) {
+      SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i);
+      totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList);
+    }
+
+    tableSerialize = totalTables * sizeof(STableIdInfo);
+  }
+
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize +
+         tableSerialize + 4096;
 }

 static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
@ -566,7 +584,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
    
-    SCMVgroupInfo* pVgroupInfo = NULL;
+    SVgroupInfo* pVgroupInfo = NULL;
    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
      int32_t index = pTableMetaInfo->vgroupIndex;
      assert(index >= 0);
@ -860,37 +878,20 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

  // compressed ts block
  pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload));
-  int32_t tsLen = 0;
-  int32_t numOfBlocks = 0;

  if (pQueryInfo->tsBuf != NULL) {
-    int32_t vnodeId = htonl(pQueryMsg->head.vgId);
-    STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId);
-    assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL);  // this query should not be sent
-
-    // todo refactor
-    if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
-      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
-      tscError("%p: fseek failed: %s", pSql, tstrerror(code));
+    // note: here used the index instead of actual vnode id.
+    int32_t vnodeIndex = pTableMetaInfo->vgroupIndex;
+    int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks);
+    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }

-    size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
-    if (s != pBlockInfo->compLen) {
-      int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
-      tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
-      return code;
-    }
+    pMsg += pQueryMsg->tsLen;

-    pMsg += pBlockInfo->compLen;
-    tsLen = pBlockInfo->compLen;
-    numOfBlocks = pBlockInfo->numOfBlocks;
-  }
-
-  pQueryMsg->tsLen = htonl(tsLen);
-  pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks);
-  if (pQueryInfo->tsBuf != NULL) {
    pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
+    pQueryMsg->tsLen   = htonl(pQueryMsg->tsLen);
+    pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
  }

  int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
@ -907,10 +908,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateDbMsg);
+  pCmd->payloadLen = sizeof(SCreateDbMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;

-  SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload;
+  SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;

  assert(pCmd->numOfClause == 1);
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
@ -921,13 +922,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateDnodeMsg);
+  pCmd->payloadLen = sizeof(SCreateDnodeMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
+  SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload;
  strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
  
  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
@ -937,13 +938,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateAcctMsg);
+  pCmd->payloadLen = sizeof(SCreateAcctMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload;
+  SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload;

  SStrToken *pName = &pInfo->pDCLInfo->user.user;
  SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd;
@ -982,14 +983,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCreateUserMsg);
+  pCmd->payloadLen = sizeof(SCreateUserMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload;
+  SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload;

  SUserInfo *pUser = &pInfo->pDCLInfo->user;
  strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n);
@ -1014,21 +1015,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMCfgDnodeMsg);
+  pCmd->payloadLen = sizeof(SCfgDnodeMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE;
  return TSDB_CODE_SUCCESS;
 }

 int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropDbMsg);
+  pCmd->payloadLen = sizeof(SDropDbMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
+  SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload;

  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
@ -1058,13 +1059,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropDnodeMsg);
+  pCmd->payloadLen = sizeof(SDropDnodeMsg);
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
+  SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
@ -1074,7 +1075,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropUserMsg);
+  pCmd->payloadLen = sizeof(SDropUserMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@ -1082,7 +1083,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+  SDropUserMsg *  pDropMsg = (SDropUserMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));

@ -1091,7 +1092,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMDropUserMsg);
+  pCmd->payloadLen = sizeof(SDropUserMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@ -1099,7 +1100,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+  SDropUserMsg *  pDropMsg = (SDropUserMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));

@ -1108,14 +1109,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMUseDbMsg);
+  pCmd->payloadLen = sizeof(SUseDbMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload;
+  SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  strcpy(pUseDbMsg->db, pTableMetaInfo->name);
  pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB;
@ -1127,14 +1128,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  STscObj *pObj = pSql->pTscObj;
  SSqlCmd *pCmd = &pSql->cmd;
  pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW;
-  pCmd->payloadLen = sizeof(SCMShowMsg) + 100;
+  pCmd->payloadLen = sizeof(SShowMsg) + 100;

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload;
+  SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;

  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  size_t nameLen = strlen(pTableMetaInfo->name);
@ -1161,13 +1162,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    pShowMsg->payloadLen = htons(pEpAddr->n);
  }

-  pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen;
+  pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen;
  return TSDB_CODE_SUCCESS;
 }

 int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMKillQueryMsg);
+  pCmd->payloadLen = sizeof(SKillQueryMsg);

  switch (pCmd->command) {
    case TSDB_SQL_KILL_QUERY:
@ -1279,8 +1280,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) {
  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
-         TSDB_EXTRA_PAYLOAD_SIZE;
+  return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE;
 }

 int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@ -1299,7 +1299,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }
  
-  SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
+  SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload;
  tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);

  strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
@ -1348,10 +1348,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {

 int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SSqlCmd *pCmd = &pSql->cmd;
-  pCmd->payloadLen = sizeof(SCMAlterDbMsg);
+  pCmd->payloadLen = sizeof(SAlterDbMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;

-  SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
+  SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
  tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));

@ -1476,14 +1476,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  STscObj *pObj = pSql->pTscObj;
  SSqlCmd *pCmd = &pSql->cmd;
  pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT;
-  pCmd->payloadLen = sizeof(SCMConnectMsg);
+  pCmd->payloadLen = sizeof(SConnectMsg);

  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
    tscError("%p failed to malloc for query msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
+  SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload;

  // TODO refactor full_name
  char *db;  // ugly code to move the space
@ -1505,11 +1505,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

-  SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
+  STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload;
  strcpy(pInfoMsg->tableId, pTableMetaInfo->name);
  pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0);

-  char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
+  char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);

  size_t len = htonl(pCmd->tagData.dataLen);
  if (pSql->cmd.autoCreated) {
@ -1528,7 +1528,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

 /**
 *  multi table meta req pkg format:
- *  | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
+ *  | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
 *      no used         4B
 **/
 int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@ -1546,7 +1546,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize);
  memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN);  // server don't need the db

-  SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
+  SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
  pInfoMsg->numOfTables = htonl((int32_t)pCmd->count);

  if (pCmd->payloadLen > 0) {
@ -1555,7 +1555,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {

  taosTFree(tmpData);

-  pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg);
+  pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg);
  pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;

  assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize);
@ -1600,12 +1600,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
  
  char* pMsg = pCmd->payload;
  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  
-  SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg;
+
+  SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg;
  pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables);
-  pMsg += sizeof(SCMSTableVgroupMsg);
-  
-  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+  pMsg += sizeof(SSTableVgroupMsg);
+
+  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
    STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
    size_t size = sizeof(pTableMetaInfo->name);
    tstrncpy(pMsg, pTableMetaInfo->name, size);
@ -1638,14 +1638,17 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
    numOfStreams++;
  }

-  int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100;
+  int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100;
  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
    pthread_mutex_unlock(&pObj->mutex);
-    tscError("%p failed to malloc for heartbeat msg", pSql);
+    tscError("%p failed to create heartbeat msg", pSql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

-  SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
+  // TODO the expired hb and client can not be identified by server till now.
+  SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload;
+  tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer));
+
  pHeartbeat->numOfQueries = numOfQueries;
  pHeartbeat->numOfStreams = numOfStreams;

@ -1734,7 +1737,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {

 /**
 *  multi table meta rsp pkg format:
- *  | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
+ *  | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
 *  |...... 1B        1B            4B
 **/
 int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
@ -1751,9 +1754,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {

  rsp++;

-  SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp;
+  SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp;
  totalNum = htonl(pInfo->numOfTables);
-  rsp += sizeof(SCMMultiTableInfoMsg);
+  rsp += sizeof(SMultiTableInfoMsg);

  for (i = 0; i < totalNum; i++) {
    SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
@ -1845,10 +1848,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
  SSqlRes* pRes = &pSql->res;
  
  // NOTE: the order of several table must be preserved.
-  SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
+  SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp;
  pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables);
-  char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg);
-  
+  char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg);
+
  // master sqlObj locates in param
  SSqlObj* parent = pSql->param;
  assert(parent != NULL);
@ -1860,18 +1863,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
    SVgroupsMsg *  pVgroupMsg = (SVgroupsMsg *) pMsg;
    pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);

-    size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
+    size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);

-    size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
+    size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
    pInfo->vgroupList = calloc(1, vgroupsz);
    assert(pInfo->vgroupList != NULL);

    pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
    for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
      //just init, no need to lock
-      SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
+      SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];

-      SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
+      SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
      pVgroups->vgId = htonl(vmsg->vgId);
      pVgroups->numOfEps = vmsg->numOfEps;

@ -1893,10 +1896,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
 * current process do not use the cache at all
 */
 int tscProcessShowRsp(SSqlObj *pSql) {
-  STableMetaMsg * pMetaMsg;
-  SCMShowRsp *pShow;
-  SSchema *    pSchema;
-  char         key[20];
+  STableMetaMsg *pMetaMsg;
+  SShowRsp *     pShow;
+  SSchema *      pSchema;
+  char           key[20];

  SSqlRes *pRes = &pSql->res;
  SSqlCmd *pCmd = &pSql->cmd;
@ -1905,7 +1908,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {

  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

-  pShow = (SCMShowRsp *)pRes->pRsp;
+  pShow = (SShowRsp *)pRes->pRsp;
  pShow->qhandle = htobe64(pShow->qhandle);
  pRes->qhandle = pShow->qhandle;

@ -1998,11 +2001,12 @@ static void createHBObj(STscObj* pObj) {
 }

 int tscProcessConnectRsp(SSqlObj *pSql) {
-  char temp[TSDB_TABLE_FNAME_LEN * 2];
  STscObj *pObj = pSql->pTscObj;
  SSqlRes *pRes = &pSql->res;

-  SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
+  char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
+
+  SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp;
  tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId));  // copy acctId from response
  int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);

@ -2020,6 +2024,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
  pObj->connId = htonl(pConnect->connId);

  createHBObj(pObj);
+
+  //launch a timer to send heartbeat to maintain the connection and send status to mnode
  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);

  return 0;
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@ -32,11 +32,26 @@ typedef struct SInsertSupporter {
 static void freeJoinSubqueryObj(SSqlObj* pSql);
 static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);

-static bool tsCompare(int32_t order, int64_t left, int64_t right) {
+static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
+  if (left == right) {
+    return 0;
+  }
+
  if (order == TSDB_ORDER_ASC) {
-    return left < right;
+    return left < right? -1:1;
  } else {
-    return left > right;
+    return left > right? -1:1;
+  }
+}
+
+static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) {
+  while (tsBufNextPos(pTSBuf)) {
+    STSElem el1 = tsBufGetElem(pTSBuf);
+
+    int32_t res = tVariantCompare(el1.tag, tag1);
+    if (res != 0) { // it is a record with new tag
+      return;
+    }
  }
 }

@ -51,10 +66,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ

  SLimitVal* pLimit = &pQueryInfo->limit;
  int32_t    order = pQueryInfo->order.order;
-  
+
  SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0);
  SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0);
-  
+
  pSubQueryInfo1->tsBuf = output1;
  pSubQueryInfo2->tsBuf = output2;

@ -88,59 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
  int64_t numOfInput1 = 1;
  int64_t numOfInput2 = 1;

-  while (1) {
-    STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
-    STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+  while(1) {
+    STSElem elem = tsBufGetElem(pSupporter1->pTSBuf);

-#ifdef _DEBUG_VIEW
-    tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
-#endif
+    // no data in pSupporter1 anymore, jump out of loop
+    if (!tsBufIsValidElem(&elem)) {
+      break;
+    }

-    int32_t res = tVariantCompare(elem1.tag, elem2.tag);
-    if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) {
-      if (!tsBufNextPos(pSupporter1->pTSBuf)) {
-        break;
-      }
+    // find the data in supporter2 with the same tag value
+    STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag);

-      numOfInput1++;
-    } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) {
-      if (!tsBufNextPos(pSupporter2->pTSBuf)) {
-        break;
-      }
+    /**
+     * there are elements in pSupporter2 with the same tag, continue
+     */
+    tVariant tag1 = {0};
+    tVariantAssign(&tag1, elem.tag);

-      numOfInput2++;
-    } else {
-      /*
-       * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
-       * final results which is acquired after the secondry merge of in the client.
-       */
-      if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
-        if (win->skey > elem1.ts) {
-          win->skey = elem1.ts;
+    if (tsBufIsValidElem(&e2)) {
+      while (1) {
+        STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
+        STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+
+        // data with current are exhausted
+        if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) {
+          break;
        }
-  
-        if (win->ekey < elem1.ts) {
-          win->ekey = elem1.ts;
+
+        if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag
+          skipRemainValue(pSupporter1->pTSBuf, &tag1);
+          break;
        }
-        
-        tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
-        tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));

-      } else {
-        pLimit->offset -= 1;
+        /*
+         * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
+         * final results which is acquired after the secondary merge of in the client.
+         */
+        int32_t re = tsCompare(order, elem1.ts, elem2.ts);
+        if (re < 0) {
+          tsBufNextPos(pSupporter1->pTSBuf);
+          numOfInput1++;
+        } else if (re > 0) {
+          tsBufNextPos(pSupporter2->pTSBuf);
+          numOfInput2++;
+        } else {
+          if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
+            if (win->skey > elem1.ts) {
+              win->skey = elem1.ts;
+            }
+
+            if (win->ekey < elem1.ts) {
+              win->ekey = elem1.ts;
+            }
+
+            tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
+            tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
+          } else {
+            pLimit->offset -= 1;//offset apply to projection?
+          }
+
+          tsBufNextPos(pSupporter1->pTSBuf);
+          numOfInput1++;
+
+          tsBufNextPos(pSupporter2->pTSBuf);
+          numOfInput2++;
+        }
      }
-
-      if (!tsBufNextPos(pSupporter1->pTSBuf)) {
-        break;
-      }
-
-      numOfInput1++;
-
-      if (!tsBufNextPos(pSupporter2->pTSBuf)) {
-        break;
-      }
-
-      numOfInput2++;
+    } else {  // no data in pSupporter2, ignore current data in pSupporter2
+      skipRemainValue(pSupporter1->pTSBuf, &tag1);
    }
  }

@ -162,8 +192,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ

  TSKEY et = taosGetTimestampUs();
  tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks "
-           "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal,
-           output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st);
+           "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us",
+           pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfVnodes, win->skey, win->ekey,
+           tsBufGetNumOfVnodes(output1), et - st);

  return output1->numOfTotal;
 }
@ -248,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
  return false;
 }

+static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+  int32_t  num = 0;
+  int32_t* list = NULL;
+  tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list);
+
+  // The virtual node, of which all tables are disqualified after the timestamp intersection,
+  // is removed to avoid next stage query.
+  // TODO: If tables from some vnodes are not qualified for next stage query, discard them.
+  for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) {
+    SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k);
+
+    bool found = false;
+    for (int32_t f = 0; f < num; ++f) {
+      if (p->vgInfo.vgId == list[f]) {
+        found = true;
+        break;
+      }
+    }
+
+    if (!found) {
+      tscRemoveVgroupTableGroup(pVgroupTables, k);
+    } else {
+      k++;
+    }
+  }
+
+  assert(taosArrayGetSize(pVgroupTables) > 0);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+  taosTFree(list);
+}
+
+static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+  int32_t  num = 0;
+  int32_t* list = NULL;
+  tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list);
+
+  size_t numOfGroups = taosArrayGetSize(pVgroupTables);
+
+  SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo));
+
+  SVgroupTableInfo info;
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t vnodeId = list[i];
+
+    for (int32_t j = 0; j < numOfGroups; ++j) {
+      SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j);
+      if (p1->vgInfo.vgId == vnodeId) {
+        tscVgroupTableCopy(&info, p1);
+        break;
+      }
+    }
+
+    taosArrayPush(pNew, &info);
+  }
+
+  taosTFree(list);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+  return pNew;
+}
+
 /*
 * launch secondary stage query to fetch the result that contains timestamp in set
 */
@ -320,26 +413,27 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
    pQueryInfo->colList = pSupporter->colList;
    pQueryInfo->exprList = pSupporter->exprList;
    pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
+    pQueryInfo->groupbyExpr = pSupporter->groupInfo;
+
+    assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
+  
+    tscFieldInfoUpdateOffset(pQueryInfo);
+  
+    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+    pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;

    pSupporter->exprList = NULL;
    pSupporter->colList = NULL;
-    memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
-  
-    SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
-    assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
-  
-    tscFieldInfoUpdateOffset(pNewQueryInfo);
-  
-    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
-    pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;
    pSupporter->pVgroupTables = NULL;
+    memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
+    memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr));

    /*
     * When handling the projection query, the offset value will be modified for table-table join, which is changed
     * during the timestamp intersection.
     */
    pSupporter->limit = pQueryInfo->limit;
-    pNewQueryInfo->limit = pSupporter->limit;
+    pQueryInfo->limit = pSupporter->limit;

    SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
    SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0);
@ -354,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {

      tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
      tscPrintSelectClause(pNew, 0);
-      tscFieldInfoUpdateOffset(pNewQueryInfo);
+      tscFieldInfoUpdateOffset(pQueryInfo);

      pExpr = tscSqlExprGet(pQueryInfo, 0);
    }
@ -369,39 +463,21 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
      pExpr->numOfParams = 1;
    }

-    int32_t num = 0;
-    int32_t *list = NULL;
-    tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list);
-
-    if (pTableMetaInfo->pVgroupTables != NULL) {
-      for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) {
-        SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k);
-
-        bool found = false;
-        for(int32_t f = 0; f < num; ++f) {
-          if (p->vgInfo.vgId == list[f]) {
-            found = true;
-            break;
-          }
-        }
-
-        if (!found) {
-          tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k);
-        } else {
-          k++;
-        }
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+      assert(pTableMetaInfo->pVgroupTables != NULL);
+      if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+        SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables);
+        tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
+        pTableMetaInfo->pVgroupTables = p;
+      } else {
+        filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables);
      }
-
-      assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0);
-      TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
    }

-    taosTFree(list);
-
-    size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
+    size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
    tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s",
-             pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList),
-             numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
+             pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList),
+             numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
  }
  
  //prepare the subqueries object failed, abort
@ -460,18 +536,36 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {

 }

-int32_t tscCompareTidTags(const void* p1, const void* p2) {
-  const STidTags* t1 = (const STidTags*) varDataVal(p1);
-  const STidTags* t2 = (const STidTags*) varDataVal(p2);
+int32_t tidTagsCompar(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) (p1);
+  const STidTags* t2 = (const STidTags*) (p2);
  
  if (t1->vgId != t2->vgId) {
    return (t1->vgId > t2->vgId) ? 1 : -1;
  }

-  if (t1->tid != t2->tid) {
-    return (t1->tid > t2->tid) ? 1 : -1;
+  tstr* tag1 = (tstr*) t1->tag;
+  tstr* tag2 = (tstr*) t2->tag;
+
+  if (tag1->len != tag2->len) {
+    return (tag1->len > tag2->len)? 1: -1;
  }
-  return 0;
+
+  return strncmp(tag1->data, tag2->data, tag1->len);
+}
+
+int32_t tagValCompar(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) varDataVal(p1);
+  const STidTags* t2 = (const STidTags*) varDataVal(p2);
+
+  tstr* tag1 = (tstr*) t1->tag;
+  tstr* tag2 = (tstr*) t2->tag;
+
+  if (tag1->len != tag2->len) {
+    return (tag1->len > tag2->len)? 1: -1;
+  }
+
+  return strncmp(tag1->data, tag2->data, tag1->len);
 }

 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@ -489,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
      SVgroupTableInfo info = {{0}};
      for (int32_t m = 0; m < pvg->numOfVgroups; ++m) {
        if (tt->vgId == pvg->vgroups[m].vgId) {
-          tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
+          tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
          break;
        }
      }
@ -497,17 +591,29 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr

      vgTables = taosArrayInit(4, sizeof(STableIdInfo));
      info.itemList = vgTables;
+
+      if (taosArrayGetSize(result) > 0) {
+        SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1);
+        tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList));
+      }
+
      taosArrayPush(result, &info);
    }

-    tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId)
    STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN};
    taosArrayPush(vgTables, &item);
+
+    tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId);
    prev = tt;
  }

  pTableMetaInfo->pVgroupTables = result;
  pTableMetaInfo->vgroupIndex = 0;
+
+  if (taosArrayGetSize(result) > 0) {
+    SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1);
+    tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList));
+  }
 }

 static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@ -582,22 +688,24 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq
 }

 static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
-  tscDebug("%p all subqueries retrieve <tid, tags> complete, do tags match", pParentSql);
-
  SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
  SJoinSupporter* p2 = pParentSql->pSubs[1]->param;

-  qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
-  qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
+  tscDebug("%p all subquery retrieve <tid, tags> complete, do tags match, %d, %d", pParentSql, p1->num, p2->num);
+
+  // sort according to the tag value
+  qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar);
+  qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar);

  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
  int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);

-  SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+  SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);

  // int16_t for padding
-  *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t));
-  *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t));
+  int32_t size = p1->tagSize - sizeof(int16_t);
+  *s1 = taosArrayInit(p1->num, size);
+  *s2 = taosArrayInit(p2->num, size);

  if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) {
    return TSDB_CODE_QRY_DUP_JOIN_KEY;
@ -625,6 +733,27 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
    }
  }

+  // reorganize the tid-tag value according to both the vgroup id and tag values
+  // sort according to the tag value
+  size_t t1 = taosArrayGetSize(*s1);
+  size_t t2 = taosArrayGetSize(*s2);
+
+  qsort((*s1)->pData, t1, size, tidTagsCompar);
+  qsort((*s2)->pData, t2, size, tidTagsCompar);
+
+#if 0
+  for(int32_t k = 0; k < t1; ++k) {
+    STidTags* p =  (*s1)->pData + size * k;
+    printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+  }
+
+  for(int32_t k = 0; k < t1; ++k) {
+    STidTags* p =  (*s2)->pData + size * k;
+    printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+  }
+#endif
+
+  tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2);
  return TSDB_CODE_SUCCESS;
 }

@ -744,10 +873,10 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
    tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);

    SSqlObj* psub1 = pParentSql->pSubs[0];
-    ((SJoinSupporter*)psub1->param)->pVgroupTables =  tscCloneVgroupTableInfo(pTableMetaInfo1->pVgroupTables);
+    ((SJoinSupporter*)psub1->param)->pVgroupTables =  tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables);

    SSqlObj* psub2 = pParentSql->pSubs[1];
-    ((SJoinSupporter*)psub2->param)->pVgroupTables =  tscCloneVgroupTableInfo(pTableMetaInfo2->pVgroupTables);
+    ((SJoinSupporter*)psub2->param)->pVgroupTables =  tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables);

    pParentSql->subState.numOfSub = 2;
    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;
@ -928,6 +1057,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
    }
  }

+  assert(pState->numOfRemain > 0);
  if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
    tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub);
    return;
@ -941,6 +1071,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
  }

  // update the records for each subquery in parent sql object.
+  bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0);
  for (int32_t i = 0; i < pState->numOfSub; ++i) {
    if (pParentSql->pSubs[i] == NULL) {
      tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i);
@ -954,7 +1085,10 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
               pRes1->numOfRows, pRes1->numOfTotal);
      assert(pRes1->row < pRes1->numOfRows);
    } else {
-      pRes1->numOfClauseTotal += pRes1->numOfRows;
+      if (!stableQuery) {
+        pRes1->numOfClauseTotal += pRes1->numOfRows;
+      }
+
      tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i,
               pRes1->numOfRows, pRes1->numOfTotal);
    }
@ -964,7 +1098,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
  tscBuildResFromSubqueries(pParentSql);
 }

-void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
+void tscFetchDatablockForSubquery(SSqlObj* pSql) {
  assert(pSql->subState.numOfSub >= 1);
  
  int32_t numOfFetch = 0;
@ -1026,11 +1160,22 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
  if (numOfFetch <= 0) {
    bool tryNextVnode = false;

-    SSqlObj*    pp = pSql->pSubs[0];
-    SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0);
+    bool orderedPrjQuery = false;
+    for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+      SSqlObj* pSub = pSql->pSubs[i];
+      if (pSub == NULL) {
+        continue;
+      }
+
+      SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0);
+      orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0);
+      if (orderedPrjQuery) {
+        break;
+      }
+    }

    // get the number of subquery that need to retrieve the next vnode.
-    if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) {
+    if (orderedPrjQuery) {
      for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
        SSqlObj* pSub = pSql->pSubs[i];
        if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) {
@ -1134,7 +1279,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
  SSqlCmd* pCmd = &pSql->cmd;
  SSqlRes* pRes = &pSql->res;

-  tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex);

  // the column transfer support struct has been built
  if (pRes->pColumnIndex != NULL) {
@ -1230,21 +1374,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
    return;
  }

-  // wait for the other subqueries response from vnode
-  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
-    return;
+
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+  // In case of consequence query from other vnode, do not wait for other query response here.
+  if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) {
+    if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
+      return;
+    }
  }

  tscSetupOutputColumnIndex(pParentSql);
-  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);

  /**
   * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
   * data instead of returning to its invoker
   */
  if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-//    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;  // reset the record value
-
    pSql->fp = joinRetrieveFinalResCallback;  // continue retrieve data
    pSql->cmd.command = TSDB_SQL_FETCH;
    tscProcessSql(pSql);
@ -1313,6 +1459,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
      return TSDB_CODE_TSC_OUT_OF_MEMORY;
    }

+    pSupporter->groupInfo = pNewQueryInfo->groupbyExpr;
+    memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
+
    pNew->cmd.numOfCols = 0;
    pNewQueryInfo->interval.interval = 0;
    pSupporter->limit = pNewQueryInfo->limit;
@ -1333,17 +1482,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
      assert(pTagCond->joinInfo.hasJoin);

      int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
-      SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+      SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);

-      // get the tag colId column index
-      int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta);
-      SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
-      for(int32_t i = 0; i < numOfTags; ++i) {
-        if (pSchema[i].colId == tagColId) {
-          colIndex.columnIndex = i;
-          break;
-        }
-      }
+      colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId);

      int16_t bytes = 0;
      int16_t type  = 0;
@ -1424,6 +1565,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {

  tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+
    SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i);
    
    if (pSupporter == NULL) {  // failed to create support struct, abort current query
@ -1645,8 +1787,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
  SSqlObj *pParentSql = trsupport->pParentSql;
  int32_t  subqueryIndex = trsupport->subqueryIndex;

-  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+  STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
+  SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];

  tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);

@ -1848,7 +1990,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
  
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo  *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+  SVgroupInfo  *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];

  if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
    trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
@ -1959,7 +2101,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
  assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
  
  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
+  SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];

  // stable query killed or other subquery failed, all query stopped
  if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
@ -2165,7 +2307,8 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
    numOfRes = (int32_t)(MIN(numOfRes, remain));
  }

-  if (numOfRes == 0) {
+  if (numOfRes == 0) {  // no result any more, free all subquery objects
+    freeJoinSubqueryObj(pSql);
    return;
  }

--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@ -80,7 +80,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon


 void taos_init_imp(void) {
-  char temp[128];
+  char temp[128]  = {0};
  
  errno = TSDB_CODE_SUCCESS;
  srand(taosGetTimestampSec());
@ -104,7 +104,6 @@ void taos_init_imp(void) {

    taosReadGlobalCfg();
    taosCheckGlobalCfg();
-    taosPrintGlobalCfg();

    tscDebug("starting to initialize TAOS client ...");
    tscDebug("Local End Point is:%s", tsLocalEp);
@ -149,30 +148,42 @@ void taos_init_imp(void) {

  tscRefId = taosOpenRef(200, tscCloseTscObj);

+  // in other language APIs, taos_cleanup is not available yet.
+  // So, to make sure taos_cleanup will be invoked to clean up the allocated
+  // resource to suppress the valgrind warning.
+  atexit(taos_cleanup);
  tscDebug("client is initialized successfully");
 }

 void taos_init() { pthread_once(&tscinit, taos_init_imp); }

-void taos_cleanup() {
-  if (tscMetaCache != NULL) {
-    taosCacheCleanup(tscMetaCache);
-    tscMetaCache = NULL;
+// this function may be called by user or system, or by both simultaneously.
+void taos_cleanup(void) {
+  tscDebug("start to cleanup client environment");

-    taosCacheCleanup(tscObjCache);
-    tscObjCache = NULL;
+  void* m = tscMetaCache;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) {
+    taosCacheCleanup(m);
  }
-  
-  if (tscQhandle != NULL) {
-    taosCleanUpScheduler(tscQhandle);
-    tscQhandle = NULL;
+
+  m = tscObjCache;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+    taosCacheCleanup(m);
+  }
+
+  m = tscQhandle;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) {
+    taosCleanUpScheduler(m);
  }

  taosCloseRef(tscRefId);
  taosCleanupKeywordsTable();
  taosCloseLog();
-  
-  taosTmrCleanUp(tscTmr);
+
+  m = tscTmr;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {
+    taosTmrCleanUp(m);
+  }
 }

 static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@ -338,34 +338,6 @@ void tscFreeSqlResult(SSqlObj* pSql) {
  memset(&pSql->res, 0, sizeof(SSqlRes));
 }

-void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
-  if (pSql == NULL || pSql->signature != pSql) {
-    return;
-  }
-
-  SSqlCmd* pCmd = &pSql->cmd;
-  int32_t cmd = pCmd->command;
-  if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
-    tscRemoveFromSqlList(pSql);
-  }
-  
-  // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-//  if (pObj->signature == pObj) {
-    //pthread_mutex_lock(&pObj->mutex);
-    taosTFree(pSql->sqlstr);
-    //pthread_mutex_unlock(&pObj->mutex);
-//  }
-  
-  tscFreeSqlResult(pSql);
-  
-  taosTFree(pSql->pSubs);
-  pSql->subState.numOfSub = 0;
-  pSql->self = 0;
-  
-  tscResetSqlCmdObj(pCmd, false);
-}
-
 static void tscFreeSubobj(SSqlObj* pSql) {
  if (pSql->subState.numOfSub == 0) {
    return;
@ -434,22 +406,32 @@ void tscFreeSqlObj(SSqlObj* pSql) {
  tscDebug("%p start to free sqlObj", pSql);

  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
  tscFreeSubobj(pSql);

-  tscPartiallyFreeSqlObj(pSql);
+  SSqlCmd* pCmd = &pSql->cmd;
+  int32_t cmd = pCmd->command;
+  if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
+    tscRemoveFromSqlList(pSql);
+  }

  pSql->signature = NULL;
  pSql->fp = NULL;
-  
-  SSqlCmd* pCmd = &pSql->cmd;
+  taosTFree(pSql->sqlstr);
+
+  taosTFree(pSql->pSubs);
+  pSql->subState.numOfSub = 0;
+  pSql->self = 0;
+
+  tscFreeSqlResult(pSql);
+  tscResetSqlCmdObj(pCmd, false);

  memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
  taosTFree(pCmd->payload);
  pCmd->allocSize = 0;
  
-  taosTFree(pSql->sqlstr);
  tsem_destroy(&pSql->rspSem);
-
  free(pSql);
 }

@ -1665,6 +1647,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
  if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
    taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
    pQueryInfo->groupbyExpr.columnInfo = NULL;
+    pQueryInfo->groupbyExpr.numOfGroupCols = 0;
  }
  
  pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
@ -1713,7 +1696,18 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) {
  taosArrayRemove(pVgroupTable, index);
 }

-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
+  memset(info, 0, sizeof(SVgroupTableInfo));
+
+  info->vgInfo = pInfo->vgInfo;
+  for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+    info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
+  }
+
+  info->itemList = taosArrayClone(pInfo->itemList);
+}
+
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) {
  if (pVgroupTables == NULL) {
    return NULL;
  }
@ -1724,14 +1718,8 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
  SVgroupTableInfo info;
  for (size_t i = 0; i < num; i++) {
    SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
-    memset(&info, 0, sizeof(SVgroupTableInfo));
+    tscVgroupTableCopy(&info, pInfo);

-    info.vgInfo = pInfo->vgInfo;
-    for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
-      info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
-    }
-
-    info.itemList = taosArrayClone(pInfo->itemList);
    taosArrayPush(pa, &info);
  }

@ -1739,7 +1727,7 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
 }

 void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
-  tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
+  tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables);
  
  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@ -1779,6 +1767,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
    pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
  }

+  // TODO handle malloc failure
  pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
  if (pTableMetaInfo->tagColList == NULL) {
    return NULL;
@ -1788,7 +1777,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
    tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
  }

-  pTableMetaInfo->pVgroupTables = tscCloneVgroupTableInfo(pVgroupTables);
+  pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables);
  
  pQueryInfo->numOfTables += 1;
  return pTableMetaInfo;
@ -2155,6 +2144,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
  }
 }

+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) {
+  int32_t numOfTags = tscGetNumOfTags(pTableMeta);
+
+  SSchema* pSchema = tscGetTableTagSchema(pTableMeta);
+  for(int32_t i = 0; i < numOfTags; ++i) {
+    if (pSchema[i].colId == colId) {
+      return i;
+    }
+  }
+
+  // can not reach here
+  assert(0);
+  return INT16_MIN;
+}
+
 bool tscIsUpdateQuery(SSqlObj* pSql) {
  if (pSql == NULL || pSql->signature != pSql) {
    terrno = TSDB_CODE_TSC_DISCONNECTED;
@ -2424,7 +2428,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
    return NULL;
  }

-  size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
+  size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups;
  SVgroupsInfo* pNew = calloc(1, size);
  if (pNew == NULL) {
    return NULL;
@ -2433,9 +2437,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
  pNew->numOfVgroups = vgroupList->numOfVgroups;

  for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
-    SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i];
+    SVgroupInfo* pNewVInfo = &pNew->vgroups[i];

-    SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i];
+    SVgroupInfo* pvInfo = &vgroupList->vgroups[i];
    pNewVInfo->vgId = pvInfo->vgId;
    pNewVInfo->numOfEps = pvInfo->numOfEps;

@ -2454,7 +2458,7 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
  }

  for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
-    SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
+    SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];

    for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
      taosTFree(pVgroupInfo->epAddr[j].fqdn);
@ -2465,10 +2469,11 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
  return NULL;
 }

-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) {
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
  dst->vgId = src->vgId;
  dst->numOfEps = src->numOfEps;
  for(int32_t i = 0; i < dst->numOfEps; ++i) {
+    taosTFree(dst->epAddr[i].fqdn);
    dst->epAddr[i].port = src->epAddr[i].port;
    dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
  }
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@ -51,6 +51,7 @@ extern char     tsLocale[];
 extern char     tsCharset[];            // default encode string
 extern int32_t  tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
+extern char     tsTempDir[];

 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer for each data node during query processing
@ -183,13 +184,13 @@ extern int32_t debugFlag;

 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

-void taosInitGlobalCfg();
-bool taosCheckGlobalCfg();
-void taosSetAllDebugFlag();
-bool taosCfgDynamicOptions(char *msg);
-int  taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
-bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
- 
+void    taosInitGlobalCfg();
+int32_t taosCheckGlobalCfg();
+void    taosSetAllDebugFlag();
+bool    taosCfgDynamicOptions(char *msg);
+int     taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
+bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+
 #ifdef __cplusplus
 }
 #endif
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@ -58,6 +58,7 @@ char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int32_t tsEnableCoreFile = 0;
 int32_t tsMaxBinaryDisplayWidth = 30;
+char    tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";

 /*
 * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -1310,13 +1311,23 @@ static void doInitGlobalConfig(void) {
  cfg.ptrLength = 0;
  cfg.unitType = TAOS_CFG_UTYPE_NONE;
  taosInitConfigOption(cfg);
+
+  cfg.option = "tempDir";
+  cfg.ptr = tsTempDir;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = tListLen(tsTempDir);
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
 }

 void taosInitGlobalCfg() {
  pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
 }

-bool taosCheckGlobalCfg() {
+int32_t taosCheckGlobalCfg() {
  char fqdn[TSDB_FQDN_LEN];
  uint16_t port;

@ -1375,7 +1386,9 @@ bool taosCheckGlobalCfg() {
  tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
  tsHttpPort = tsServerPort + TSDB_PORT_HTTP;

-  return true;
+  taosPrintGlobalCfg();
+
+  return 0;
 }

 int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
--- a/src/common/src/tvariant.c
+++ b/src/common/src/tvariant.c
@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
      break;
    }
    case TSDB_DATA_TYPE_BINARY: {  // todo refactor, extract a method
-      pVar->pz = calloc(len, sizeof(char));
+      pVar->pz = calloc(len + 1, sizeof(char));
      memcpy(pVar->pz, pz, len);
      pVar->nLen = (int32_t)len;
      break;
--- a/src/dnode/inc/dnodeCfg.h
+++ b/src/dnode/inc/dnodeCfg.h
@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_CFG_H
+#define TDENGINE_DNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dnodeInitCfg();
+void    dnodeCleanupCfg();
+void    dnodeUpdateCfg(SDnodeCfg *cfg);
+int32_t dnodeGetDnodeId();
+void    dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeEps.h
+++ b/src/dnode/inc/dnodeEps.h
@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_EP_H
+#define TDENGINE_DNODE_EP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitEps();
+void    dnodeCleanupEps();
+void    dnodeUpdateEps(SDnodeEps *eps);
+void    dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
+bool    dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeMInfos.h
+++ b/src/dnode/inc/dnodeMInfos.h
@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_MINFOS_H
+#define TDENGINE_DNODE_MINFOS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitMInfos();
+void    dnodeCleanupMInfos();
+void    dnodeUpdateMInfos(SMnodeInfos *minfos);
+void    dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetMInfos(SMnodeInfos *minfos);
+bool    dnodeIsMasterEp(char *ep);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/dnode/inc/dnodeMgmt.h
+++ b/src/dnode/inc/dnodeMgmt.h
@ -20,6 +20,8 @@
 extern "C" {
 #endif

+#include "trpc.h"
+
 int32_t dnodeInitMgmt();
 void    dnodeCleanupMgmt();
 int32_t dnodeInitMgmtTimer();
@ -35,8 +37,8 @@ void*   dnodeGetVnodeTsdb(void *pVnode);
 void    dnodeReleaseVnode(void *pVnode);

 void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetMnodeEpSetForPeer(void *epSet);
-void    dnodeGetMnodeEpSetForShell(void *epSet);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);

 #ifdef __cplusplus
 }
--- a/src/dnode/inc/dnodeVWrite.h
+++ b/src/dnode/inc/dnodeVWrite.h
@ -20,9 +20,12 @@
 extern "C" {
 #endif

-int32_t dnodeInitVnodeWrite();
-void    dnodeCleanupVnodeWrite();
-void    dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVWrite();
+void    dnodeCleanupVWrite();
+void    dnodeDispatchToVWriteQueue(SRpcMsg *pMsg);
+void *  dnodeAllocVWriteQueue(void *pVnode);
+void    dnodeFreeVWriteQueue(void *wqueue);
+void    dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);

 #ifdef __cplusplus
 }
--- a/src/dnode/src/dnodeCfg.c
+++ b/src/dnode/src/dnodeCfg.c
@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeCfg.h"
+
+static SDnodeCfg tsCfg = {0};
+static pthread_mutex_t tsCfgMutex;
+
+static int32_t dnodeReadCfg();
+static int32_t dnodeWriteCfg();
+static void    dnodeResetCfg(SDnodeCfg *cfg);
+static void    dnodePrintCfg(SDnodeCfg *cfg);
+
+int32_t dnodeInitCfg() {
+  pthread_mutex_init(&tsCfgMutex, NULL);
+  dnodeResetCfg(NULL);
+  int32_t ret = dnodeReadCfg();
+  if (ret == 0) {
+    dInfo("dnode cfg is initialized");
+  }
+  return ret;
+}
+
+void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); }
+
+void dnodeUpdateCfg(SDnodeCfg *cfg) {
+  if (tsCfg.dnodeId != 0) return;
+  dnodeResetCfg(cfg);
+}
+
+int32_t dnodeGetDnodeId() {
+  int32_t dnodeId = 0;
+  pthread_mutex_lock(&tsCfgMutex);
+  dnodeId = tsCfg.dnodeId;
+  pthread_mutex_unlock(&tsCfgMutex);
+  return dnodeId;
+}
+
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+  pthread_mutex_lock(&tsCfgMutex);
+  *dnodeId = tsCfg.dnodeId;
+  tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodeResetCfg(SDnodeCfg *cfg) {
+  if (cfg == NULL) return;
+  if (cfg->dnodeId == 0) return;
+
+  pthread_mutex_lock(&tsCfgMutex);
+  tsCfg.dnodeId = cfg->dnodeId;
+  tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
+  dnodePrintCfg(cfg);
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodePrintCfg(SDnodeCfg *cfg) {
+  dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+}
+
+static int32_t dnodeReadCfg() {
+  int32_t   len = 0;
+  int32_t   maxLen = 200;
+  char *    content = calloc(1, maxLen + 1);
+  cJSON *   root = NULL;
+  FILE *    fp = NULL;
+  SDnodeCfg cfg = {0};
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PARSE_CFG_OVER;
+  }
+
+  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_Number) {
+    dError("failed to read %s, dnodeId not found", file);
+    goto PARSE_CFG_OVER;
+  }
+  cfg.dnodeId = dnodeId->valueint;
+
+  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_String) {
+    dError("failed to read %s, clusterId not found", file);
+    goto PARSE_CFG_OVER;
+  }
+  tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+
+  dInfo("read file %s successed", file);
+
+PARSE_CFG_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  terrno = 0;
+
+  dnodeResetCfg(&cfg);
+  return 0;
+}
+
+static int32_t dnodeWriteCfg() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 200;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsCfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsCfg.clusterId);
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeCheck.c
+++ b/src/dnode/src/dnodeCheck.c
@ -15,9 +15,7 @@

 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosdef.h"
 #include "tglobal.h"
-#include "mnode.h"
 #include "dnodeInt.h"
 #include "dnodeCheck.h"

@ -30,8 +28,8 @@ typedef struct {
  void    (*stopFp)();
 } SCheckItem;

-static    SCheckItem  tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
-int64_t   tsMinFreeMemSizeForStart = 0;
+static  SCheckItem  tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
+int64_t tsMinFreeMemSizeForStart = 0;

 static int bindTcpPort(int port) {
  int    serverSocket;
@ -264,8 +262,6 @@ int32_t dnodeInitCheck() {
    }
  }

+  dInfo("dnode check is initialized");
  return 0;
 }
-
-
-
--- a/src/dnode/src/dnodeEps.c
+++ b/src/dnode/src/dnodeEps.c
@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "hash.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeEps.h"
+
+static SDnodeEps *tsEps = NULL;
+static SHashObj * tsEpsHash = NULL;
+static pthread_mutex_t tsEpsMutex;
+
+static int32_t dnodeReadEps();
+static int32_t dnodeWriteEps();
+static void    dnodeResetEps(SDnodeEps *eps);
+static void    dnodePrintEps(SDnodeEps *eps);
+
+int32_t dnodeInitEps() {
+  pthread_mutex_init(&tsEpsMutex, NULL);
+  tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  dnodeResetEps(NULL);
+  int32_t ret = dnodeReadEps();
+  if (ret == 0) {
+    dInfo("dnode eps is initialized");
+  }
+  return ret;
+}
+
+void dnodeCleanupEps() {
+  pthread_mutex_lock(&tsEpsMutex);
+  if (tsEps) {
+    free(tsEps);
+    tsEps = NULL;
+  }
+  if (tsEpsHash) {
+    taosHashCleanup(tsEpsHash);
+    tsEpsHash = NULL;
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+  pthread_mutex_destroy(&tsEpsMutex);
+}
+
+void dnodeUpdateEps(SDnodeEps *eps) {
+  if (eps == NULL) return;
+
+  eps->dnodeNum = htonl(eps->dnodeNum);
+  for (int32_t i = 0; i < eps->dnodeNum; ++i) {
+    eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId);
+    eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort);
+  }
+
+  pthread_mutex_lock(&tsEpsMutex);
+  if (eps->dnodeNum != tsEps->dnodeNum) {
+    dnodeResetEps(eps);
+    dnodeWriteEps();
+  } else {
+    int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp);
+    if (memcmp(eps, tsEps, size) != 0) {
+      dnodeResetEps(eps);
+      dnodeWriteEps();
+    }
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+}
+
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) {
+  bool changed = false;
+  pthread_mutex_lock(&tsEpsMutex);
+  SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+  if (ep != NULL) {
+    char epSaved[TSDB_EP_LEN + 1];
+    snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+    changed = strcmp(epstr, epSaved) != 0;
+    tstrncpy(epstr, epSaved, TSDB_EP_LEN);
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+  return changed;
+}
+
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
+  pthread_mutex_lock(&tsEpsMutex);
+  SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+  if (ep != NULL) {
+    if (port) *port = ep->dnodePort;
+    if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
+    if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+  }
+  pthread_mutex_unlock(&tsEpsMutex);
+}
+
+static void dnodeResetEps(SDnodeEps *eps) {
+  if (eps == NULL) {
+    int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp);
+    if (tsEps == NULL) {
+      tsEps = calloc(1, size);
+    } else {
+      tsEps->dnodeNum = 0;
+    }
+  } else {
+    assert(tsEps);
+
+    int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum;
+    if (eps->dnodeNum > tsEps->dnodeNum) {
+      tsEps = realloc(tsEps, size);
+    }
+    memcpy(tsEps, eps, size);
+    dnodePrintEps(eps);
+  }
+
+  for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+    SDnodeEp *ep = &tsEps->dnodeEps[i];
+    taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+  }
+}
+
+static void dnodePrintEps(SDnodeEps *eps) {
+  dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
+  for (int32_t i = 0; i < eps->dnodeNum; i++) {
+    SDnodeEp *ep = &eps->dnodeEps[i];
+    dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
+  }
+}
+
+static int32_t dnodeReadEps() {
+  int32_t    ret = -1;
+  int32_t    len = 0;
+  int32_t    maxLen = 30000;
+  char *     content = calloc(1, maxLen + 1);
+  cJSON *    root = NULL;
+  FILE *     fp = NULL;
+  SDnodeEps *eps = NULL;
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
+  if (!dnodeNum || dnodeNum->type != cJSON_Number) {
+    dError("failed to read %s, dnodeNum not found", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
+  if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
+    dError("failed to read %s, dnodeInfos not found", file);
+    goto PRASE_EPS_OVER;
+  }
+
+  int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
+  if (dnodeInfosSize != dnodeNum->valueint) {
+    dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize,
+           (int32_t)dnodeNum->valueint);
+    goto PRASE_EPS_OVER;
+  }
+
+  int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp);
+  eps = calloc(1, epsSize);
+  eps->dnodeNum = dnodeInfosSize;
+
+  for (int32_t i = 0; i < dnodeInfosSize; ++i) {
+    cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
+    if (dnodeInfo == NULL) break;
+
+    SDnodeEp *ep = &eps->dnodeEps[i];
+
+    cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
+    if (!dnodeId || dnodeId->type != cJSON_Number) {
+      dError("failed to read %s, dnodeId not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    ep->dnodeId = dnodeId->valueint;
+
+    cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
+    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+      dError("failed to read %s, dnodeFqdn not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
+    if (!dnodePort || dnodePort->type != cJSON_Number) {
+      dError("failed to read %s, dnodePort not found", file);
+      goto PRASE_EPS_OVER;
+    }
+    ep->dnodePort = (uint16_t)dnodePort->valueint;
+  }
+
+  ret = 0;
+
+  dInfo("read file %s successed", file);
+  dnodePrintEps(eps);
+
+PRASE_EPS_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  if (ret != 0) {
+    if (eps) free(eps);
+    eps = NULL;
+  }
+
+  dnodeResetEps(eps);
+  if (eps) free(eps);
+
+  dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+
+  terrno = 0;
+  return 0;
+}
+
+static int32_t dnodeWriteEps() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 30000;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", tsEps->dnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
+  for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+    SDnodeEp *ep = &tsEps->dnodeEps[i];
+    len += snprintf(content + len, maxLen - len, "    \"dnodeId\": %d,\n", ep->dnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
+    len += snprintf(content + len, maxLen - len, "    \"dnodePort\": %u\n", ep->dnodePort);
+    if (i < tsEps->dnodeNum - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeMInfos.c
+++ b/src/dnode/src/dnodeMInfos.c
@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMInfos.h"
+
+static SMnodeInfos tsMInfos;
+static SRpcEpSet   tsMEpSet;
+static pthread_mutex_t tsMInfosMutex;
+
+static void    dnodeResetMInfos(SMnodeInfos *minfos);
+static void    dnodePrintMInfos(SMnodeInfos *minfos);
+static int32_t dnodeReadMInfos();
+static int32_t dnodeWriteMInfos();
+
+int32_t dnodeInitMInfos() {
+  pthread_mutex_init(&tsMInfosMutex, NULL);
+  dnodeResetMInfos(NULL);
+  int32_t ret = dnodeReadMInfos();
+  if (ret == 0) {
+    dInfo("dnode minfos is initialized");
+  }
+
+  return ret;
+}
+
+void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
+
+void dnodeUpdateMInfos(SMnodeInfos *minfos) {
+  if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
+    dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+    return;
+  }
+
+  for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
+    SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+    minfo->mnodeId = htonl(minfo->mnodeId);
+    if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
+      dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
+      return;
+    }
+  }
+
+  pthread_mutex_lock(&tsMInfosMutex);
+  if (minfos->mnodeNum != tsMInfos.mnodeNum) {
+    dnodeResetMInfos(minfos);
+    dnodeWriteMInfos();
+    sdbUpdateAsync();
+  } else {
+    int32_t size = sizeof(SMnodeInfos);
+    if (memcmp(minfos, &tsMInfos, size) != 0) {
+      dnodeResetMInfos(minfos);
+      dnodeWriteMInfos();
+      sdbUpdateAsync();
+    }
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
+  if (ep->numOfEps <= 0) {
+    dError("mnode EP list for peer is changed, but content is invalid, discard it");
+    return;
+  }
+
+  pthread_mutex_lock(&tsMInfosMutex);
+  dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
+  for (int i = 0; i < ep->numOfEps; ++i) {
+    ep->port[i] -= TSDB_PORT_DNODEDNODE;
+    dInfo("mnode index:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
+  }
+  tsMEpSet = *ep;
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+bool dnodeIsMasterEp(char *ep) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0;
+  pthread_mutex_unlock(&tsMInfosMutex);
+
+  return isMaster;
+}
+
+void dnodeGetMInfos(SMnodeInfos *minfos) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
+    minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  *epSet = tsMEpSet;
+  for (int i = 0; i < epSet->numOfEps; ++i) {
+    epSet->port[i] += TSDB_PORT_DNODEDNODE;
+  }
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsMInfosMutex);
+  *epSet = tsMEpSet;
+  pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+static void dnodePrintMInfos(SMnodeInfos *minfos) {
+  dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
+  for (int32_t i = 0; i < minfos->mnodeNum; i++) {
+    dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+  }
+}
+
+static void dnodeResetMInfos(SMnodeInfos *minfos) {
+  if (minfos == NULL) {
+    tsMEpSet.numOfEps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
+
+    if (strcmp(tsSecond, tsFirst) != 0) {
+      tsMEpSet.numOfEps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]);
+    }
+    return;
+  }
+
+  if (minfos->mnodeNum == 0) return;
+
+  int32_t size = sizeof(SMnodeInfos);
+  memcpy(&tsMInfos, minfos, size);
+
+  tsMEpSet.inUse = tsMInfos.inUse;
+  tsMEpSet.numOfEps = tsMInfos.mnodeNum;
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+    taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
+  }
+
+  dnodePrintMInfos(minfos);
+}
+
+static int32_t dnodeReadMInfos() {
+  int32_t     len = 0;
+  int32_t     maxLen = 2000;
+  char *      content = calloc(1, maxLen + 1);
+  cJSON *     root = NULL;
+  FILE *      fp = NULL;
+  SMnodeInfos minfos = {0};
+
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("failed to read %s, file not exist", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  len = fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s, content is null", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s, invalid json format", file);
+    goto PARSE_MINFOS_OVER;
+  }
+
+  cJSON *inUse = cJSON_GetObjectItem(root, "inUse");
+  if (!inUse || inUse->type != cJSON_Number) {
+    dError("failed to read mnodeEpSet.json, inUse not found");
+    goto PARSE_MINFOS_OVER;
+  }
+  tsMInfos.inUse = inUse->valueint;
+
+  cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
+  if (!nodeNum || nodeNum->type != cJSON_Number) {
+    dError("failed to read mnodeEpSet.json, nodeNum not found");
+    goto PARSE_MINFOS_OVER;
+  }
+  minfos.mnodeNum = nodeNum->valueint;
+
+  cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+    dError("failed to read mnodeEpSet.json, nodeInfos not found");
+    goto PARSE_MINFOS_OVER;
+  }
+
+  int size = cJSON_GetArraySize(nodeInfos);
+  if (size != minfos.mnodeNum) {
+    dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
+    goto PARSE_MINFOS_OVER;
+  }
+
+  for (int i = 0; i < size; ++i) {
+    cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+    if (nodeInfo == NULL) continue;
+
+    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+    if (!nodeId || nodeId->type != cJSON_Number) {
+      dError("failed to read mnodeEpSet.json, nodeId not found");
+      goto PARSE_MINFOS_OVER;
+    }
+    minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
+
+    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+      dError("failed to read mnodeEpSet.json, nodeName not found");
+      goto PARSE_MINFOS_OVER;
+    }
+    strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+  }
+
+  dInfo("read file %s successed", file);
+  dnodePrintMInfos(&minfos);
+
+PARSE_MINFOS_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+  terrno = 0;
+
+  for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
+    SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+    dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
+  }
+  dnodeResetMInfos(&minfos);
+  return 0;
+}
+
+static int32_t dnodeWriteMInfos() {
+  char file[TSDB_FILENAME_LEN + 20] = {0};
+  sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+  FILE *fp = fopen(file, "w");
+  if (!fp) {
+    dError("failed to write %s, reason:%s", file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 2000;
+  char *  content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsMInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsMInfos.mnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
+  for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp);
+    if (i < tsMInfos.mnodeNum - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  fflush(fp);
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", file);
+  return 0;
+}
--- a/src/dnode/src/dnodeMPeer.c
+++ b/src/dnode/src/dnodeMPeer.c
@ -58,7 +58,7 @@ int32_t dnodeInitMnodePeer() {
    dDebug("dnode mpeer worker:%d is created", i);
  }

-  dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
+  dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
  return 0;
 }

--- a/src/dnode/src/dnodeMRead.c
+++ b/src/dnode/src/dnodeMRead.c
@ -60,7 +60,7 @@ int32_t dnodeInitMnodeRead() {
    dDebug("dnode mread worker:%d is created", i);
  }

-  dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
+  dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
  return 0;
 }

--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@ -60,7 +60,7 @@ int32_t dnodeInitMnodeWrite() {
    dDebug("dnode mwrite worker:%d is created", i);
  }

-  dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
+  dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
  return 0;
 }

--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@ -19,11 +19,15 @@
 #include "tutil.h"
 #include "tconfig.h"
 #include "tglobal.h"
+#include "twal.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
 #include "dnodePeer.h"
 #include "dnodeModule.h"
+#include "dnodeEps.h"
+#include "dnodeMInfos.h"
+#include "dnodeCfg.h"
 #include "dnodeCheck.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
@ -33,26 +37,32 @@
 #include "dnodeShell.h"
 #include "dnodeTelemetry.h"

-static int32_t dnodeInitStorage();
-static void dnodeCleanupStorage();
-static void dnodeSetRunStatus(SDnodeRunStatus status);
-static void dnodeCheckDataDirOpenned(char *dir);
 static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+
+static int32_t dnodeInitStorage();
+static void    dnodeCleanupStorage();
+static void    dnodeSetRunStatus(SDnodeRunStatus status);
+static void    dnodeCheckDataDirOpenned(char *dir);
 static int32_t dnodeInitComponents();
-static void dnodeCleanupComponents(int32_t stepId);
-static int dnodeCreateDir(const char *dir);
+static void    dnodeCleanupComponents(int32_t stepId);
+static int     dnodeCreateDir(const char *dir);

 typedef struct {
  const char *const name;
-  int               (*init)();
-  void              (*cleanup)();
+  int32_t (*init)();
+  void (*cleanup)();
 } SDnodeComponent;

 static const SDnodeComponent tsDnodeComponents[] = {
  {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
+  {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
+  {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
+  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
+  {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
+  {"wal",       walInit,             walCleanUp},
  {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
  {"vread",     dnodeInitVnodeRead,  dnodeCleanupVnodeRead},
-  {"vwrite",    dnodeInitVnodeWrite, dnodeCleanupVnodeWrite},
+  {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
  {"mread",     dnodeInitMnodeRead,  dnodeCleanupMnodeRead},
  {"mwrite",    dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
  {"mpeer",     dnodeInitMnodePeer,  dnodeCleanupMnodePeer},  
@ -75,7 +85,9 @@ static int dnodeCreateDir(const char *dir) {

 static void dnodeCleanupComponents(int32_t stepId) {
  for (int32_t i = stepId; i >= 0; i--) {
-    tsDnodeComponents[i].cleanup();
+    if (tsDnodeComponents[i].cleanup) {
+      (*tsDnodeComponents[i].cleanup)();
+    }
  }
 }

@ -112,14 +124,13 @@ int32_t dnodeInitSystem() {
    printf("failed to init log file\n");
  }

-  if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) {
+  if (!taosReadGlobalCfg()) {
    taosPrintGlobalCfg();
    dError("TDengine read global config failed");
    return -1;
  }
-  taosPrintGlobalCfg();

-  dInfo("start to initialize TDengine on %s", tsLocalEp);
+  dInfo("start to initialize TDengine");

  if (dnodeInitComponents() != 0) {
    return -1;
@ -198,7 +209,7 @@ static int32_t dnodeInitStorage() {

  dnodeCheckDataDirOpenned(tsDnodeDir);

-  dInfo("storage directory is initialized");
+  dInfo("dnode storage is initialized at %s", tsDnodeDir);
  return 0;
 }

--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@ -31,12 +31,13 @@
 #include "mnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
 #include "dnodeModule.h"

-#define MPEER_CONTENT_LEN 2000
-
 typedef struct {
  pthread_t thread;
  int32_t   threadIndex;
@ -46,23 +47,13 @@ typedef struct {
  int32_t * vnodeList;
 } SOpenVnodeThread;

-void *          tsDnodeTmr = NULL;
-static void *   tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
+void *            tsDnodeTmr = NULL;
+static void *     tsStatusTimer = NULL;
+static uint32_t   tsRebootTime;
+static taos_qset  tsMgmtQset = NULL;
+static taos_queue tsMgmtQueue = NULL;
+static pthread_t  tsQthread;

-static SRpcEpSet     tsDMnodeEpSet = {0};
-static SDMMnodeInfos tsDMnodeInfos = {0};
-static SDMDnodeCfg   tsDnodeCfg = {0};
-static taos_qset     tsMgmtQset = NULL;
-static taos_queue    tsMgmtQueue = NULL;
-static pthread_t     tsQthread;
-
-static void   dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
-static bool   dnodeReadMnodeInfos();
-static void   dnodeSaveMnodeInfos();
-static void   dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
-static bool   dnodeReadDnodeCfg();
-static void   dnodeSaveDnodeCfg();
 static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
 static void   dnodeSendStatusMsg(void *handle, void *tmrId);
 static void  *dnodeProcessMgmtQueue(void *param);
@ -74,7 +65,7 @@ static int32_t  dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
+static int32_t  dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
 static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);

 int32_t dnodeInitMgmt() {
@ -86,28 +77,8 @@ int32_t dnodeInitMgmt() {
  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;

  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
-  dnodeReadDnodeCfg();
  tsRebootTime = taosGetTimestampSec();

-  if (!dnodeReadMnodeInfos()) {
-    memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet));
-    memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
-
-    tsDMnodeEpSet.numOfEps = 1;
-    taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]);
-    
-    if (strcmp(tsSecond, tsFirst) != 0) {
-      tsDMnodeEpSet.numOfEps = 2;
-      taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]);
-    }
-  } else {
-    tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
-    tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
-    for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-      taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
-    }
-  }
-
  int32_t code = vnodeInitResources();
  if (code != TSDB_CODE_SUCCESS) {
    dnodeCleanupMgmt();
@ -381,7 +352,7 @@ static void dnodeCloseVnodes() {
 }

 static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
  pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
@ -404,7 +375,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
+  SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);

  void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
  if (pVnode != NULL) {
@ -418,7 +389,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
+  SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);

  void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
  if (pVnode != NULL) {
@ -433,14 +404,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
 }

 static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
-  SMDDropVnodeMsg *pDrop = rpcMsg->pCont;
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
  pDrop->vgId = htonl(pDrop->vgId);

  return vnodeDrop(pDrop->vgId);
 }

 static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-//  SMDAlterStreamMsg *pStream = pCont;
+//  SAlterStreamMsg *pStream = pCont;
 //  pStream->uid    = htobe64(pStream->uid);
 //  pStream->stime  = htobe64(pStream->stime);
 //  pStream->vnode  = htonl(pStream->vnode);
@ -453,12 +424,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
 }

 static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
-  SMDCfgDnodeMsg *pCfg = pMsg->pCont;
+  SCfgDnodeMsg *pCfg = pMsg->pCont;
  return taosCfgDynamicOptions(pCfg->config);
 }

 static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
-  SMDCreateMnodeMsg *pCfg = pMsg->pCont;
+  SCreateMnodeMsg *pCfg = pMsg->pCont;
  pCfg->dnodeId = htonl(pCfg->dnodeId);
  if (pCfg->dnodeId != dnodeGetDnodeId()) {
    dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
@ -470,10 +441,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
  }

-  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum);
-  for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) {
-    pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId);
-    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp);
+  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
+  for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
+    pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
+    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
  }

  dnodeStartMnode(&pCfg->mnodes);
@ -481,34 +452,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
  return TSDB_CODE_SUCCESS;
 }

-void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
-  if (pEpSet->numOfEps <= 0) {
-    dError("mnode EP list for peer is changed, but content is invalid, discard it");
-    return;
-  }
-
-  dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse);
-  for (int i = 0; i < pEpSet->numOfEps; ++i) {
-    pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
-    dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
-  }
-
-  tsDMnodeEpSet = *pEpSet;
-}
-
-void dnodeGetMnodeEpSetForPeer(void *epSetRaw) {
-  SRpcEpSet *epSet = epSetRaw;
-  *epSet = tsDMnodeEpSet;
-
-  for (int i=0; i<epSet->numOfEps; ++i) 
-    epSet->port[i] += TSDB_PORT_DNODEDNODE;
-}
-
-void dnodeGetMnodeEpSetForShell(void *epSetRaw) {
-  SRpcEpSet *epSet = epSetRaw;
-  *epSet = tsDMnodeEpSet;
-}
-
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
  if (pMsg->code != TSDB_CODE_SUCCESS) {
    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
@ -516,202 +459,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
    return;
  }

-  SDMStatusRsp *pStatusRsp = pMsg->pCont;
-  SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
-  if (pMnodes->nodeNum <= 0) {
-    dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
+  SStatusRsp *pStatusRsp = pMsg->pCont;
+  SMnodeInfos *minfos = &pStatusRsp->mnodes;
+  dnodeUpdateMInfos(minfos);

-  SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
+  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
  pCfg->dnodeId = htonl(pCfg->dnodeId);
-
-  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
-    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
-    pMnodeInfo->nodeId   = htonl(pMnodeInfo->nodeId);
-  }
+  dnodeUpdateCfg(pCfg);

  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);

-  // will not set mnode in status msg
-  // dnodeProcessModuleStatus(pCfg->moduleStatus);
-  dnodeUpdateDnodeCfg(pCfg);
+  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+  dnodeUpdateEps(pEps);

-  dnodeUpdateMnodeInfos(pMnodes);
  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
 }

-static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) {
-  if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) {
-    dError("invalid mnode infos, num:%d", pMnodes->nodeNum);
-    return false;
-  }
-
-  for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
-    SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
-    if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) {
-      dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp);
-      return false;
-    }
-  }
-
-  return true;
-}
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
-  bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
-  bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
-  if (!(mnodesChanged || mnodesNotInit)) return;
-
-  if (!dnodeCheckMnodeInfos(pMnodes)) return;
-
-  memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
-  dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-  tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
-  tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
-  }
-
-  dnodeSaveMnodeInfos();
-  sdbUpdateAsync();
-}
-
-static bool dnodeReadMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN*2] = {0};
-  
-  sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "r");
-  if (!fp) {
-    dDebug("failed to read mnodeEpSet.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 2000;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read mnodeEpSet.json, content is null");
-    return false;
-  }
-
-  content[len] = 0;
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read mnodeEpSet.json, invalid json format");
-    goto PARSE_OVER;
-  }
-
-  cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
-  if (!inUse || inUse->type != cJSON_Number) {
-    dError("failed to read mnodeEpSet.json, inUse not found");
-    goto PARSE_OVER;
-  }
-  tsDMnodeInfos.inUse = inUse->valueint;
-
-  cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
-  if (!nodeNum || nodeNum->type != cJSON_Number) {
-    dError("failed to read mnodeEpSet.json, nodeNum not found");
-    goto PARSE_OVER;
-  }
-  tsDMnodeInfos.nodeNum = nodeNum->valueint;
-
-  cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
-  if (!nodeInfos || nodeInfos->type != cJSON_Array) {
-    dError("failed to read mnodeEpSet.json, nodeInfos not found");
-    goto PARSE_OVER;
-  }
-
-  int size = cJSON_GetArraySize(nodeInfos);
-  if (size != tsDMnodeInfos.nodeNum) {
-    dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
-    goto PARSE_OVER;
-  }
-
-  for (int i = 0; i < size; ++i) {
-    cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
-    if (nodeInfo == NULL) continue;
-
-    cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
-    if (!nodeId || nodeId->type != cJSON_Number) {
-      dError("failed to read mnodeEpSet.json, nodeId not found");
-      goto PARSE_OVER;
-    }
-    tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
-
-    cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
-    if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
-      dError("failed to read mnodeEpSet.json, nodeName not found");
-      goto PARSE_OVER;
-    }
-    strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
-  }
-
-  ret = true;
-
-  dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
-  }
-
-PARSE_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveMnodeInfos() {
-  char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
-  FILE *fp = fopen(ipFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 2000;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsDMnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum);
-  len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp);
-    if (i < tsDMnodeInfos.nodeNum -1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");  
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");  
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n"); 
-
-  fwrite(content, 1, len, fp);
-  fflush(fp);
-  fclose(fp);
-  free(content);
-  
-  dInfo("save mnode epSet successed");
-}
-
-char *dnodeGetMnodeMasterEp() {
-  return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp;
-}
-
-void* dnodeGetMnodeInfos() {
-  return &tsDMnodeInfos;
-}
-
 static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  if (tsDnodeTmr == NULL) {
    dError("dnode timer is already released");
@ -724,22 +489,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
    return;
  }

-  int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SDMStatusMsg *pStatus = rpcMallocCont(contLen);
+  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SStatusMsg *pStatus = rpcMallocCont(contLen);
  if (pStatus == NULL) {
    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
    dError("failed to malloc status message");
    return;
  }

-  //strcpy(pStatus->dnodeName, tsDnodeName);
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
  pStatus->version          = htonl(tsVersion);
-  pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
  pStatus->lastReboot       = htonl(tsRebootTime);
  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
  pStatus->diskAvailable    = tsAvailDataDirGB;
  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN);
  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);

  // fill cluster cfg parameters
@ -759,7 +523,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
  
  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
  pStatus->openVnodes = htons(pStatus->openVnodes);
  
  SRpcMsg rpcMsg = {
@ -769,110 +533,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
  };

  SRpcEpSet epSet;
-  dnodeGetMnodeEpSetForPeer(&epSet);
+  dnodeGetEpSetForPeer(&epSet);
  dnodeSendMsgToDnode(&epSet, &rpcMsg);
 }

-static bool dnodeReadDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0};
-  
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "r");
-  if (!fp) {
-    dDebug("failed to read dnodeCfg.json, file not exist");
-    return false;
-  }
-
-  bool  ret = false;
-  int   maxLen = 100;
-  char *content = calloc(1, maxLen + 1);
-  int   len = fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    free(content);
-    fclose(fp);
-    dError("failed to read dnodeCfg.json, content is null");
-    return false;
-  }
-
-  content[len] = 0;
-  cJSON* root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read dnodeCfg.json, invalid json format");
-    goto PARSE_CFG_OVER;
-  }
-
-  cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read dnodeCfg.json, dnodeId not found");
-    goto PARSE_CFG_OVER;
-  }
-  tsDnodeCfg.dnodeId = dnodeId->valueint;
-
-  cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read dnodeCfg.json, clusterId not found");
-    goto PARSE_CFG_OVER;
-  }
-  tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
-
-  ret = true;
-
-  dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
-
-PARSE_CFG_OVER:
-  free(content);
-  cJSON_Delete(root);
-  fclose(fp);
-  return ret;
-}
-
-static void dnodeSaveDnodeCfg() {
-  char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
-  FILE *fp = fopen(dnodeCfgFile, "w");
-  if (!fp) return;
-
-  int32_t len = 0;
-  int32_t maxLen = 200;
-  char *  content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  fwrite(content, 1, len, fp);
-  fflush(fp);
-  fclose(fp);
-  free(content);
-  
-  dInfo("save dnodeId successed");
-}
-
-void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
-  if (tsDnodeCfg.dnodeId == 0) {
-    dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId);  
-    tsDnodeCfg.dnodeId = pCfg->dnodeId;
-    tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN);
-    dnodeSaveDnodeCfg();
-  }
-}
-
-int32_t dnodeGetDnodeId() {
-  return tsDnodeCfg.dnodeId;
-}
-
 void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
  SRpcConnInfo connInfo = {0};
  rpcGetConnInfo(rpcMsg->handle, &connInfo);

  SRpcEpSet epSet = {0};
  if (forShell) {
-    dnodeGetMnodeEpSetForShell(&epSet);
+    dnodeGetEpSetForShell(&epSet);
  } else {
-    dnodeGetMnodeEpSetForPeer(&epSet);
+    dnodeGetEpSetForPeer(&epSet);
  }
  
  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@ -114,6 +114,7 @@ int32_t dnodeInitModules() {
    }
  }

+  dInfo("dnode modules is initialized");
  return 0;
 }

@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
  }
 }

-bool dnodeStartMnode(void *pMnodes) {
-  SDMMnodeInfos *mnodes = pMnodes;
+bool dnodeStartMnode(SMnodeInfos *minfos) {
+  SMnodeInfos *mnodes = minfos;

  if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
    dDebug("mnode module is already started, module status:%d", tsModuleStatus);
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@ -28,8 +28,8 @@
 #include "dnodeMgmt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMPeer.h"
+#include "dnodeMInfos.h"

-extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet);
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
 static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
@ -38,10 +38,10 @@ static void *tsDnodeServerRpc = NULL;
 static void *tsDnodeClientRpc = NULL;

 int32_t dnodeInitServer() {
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeDispatchToVnodeWriteQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVnodeWriteQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = dnodeDispatchToVWriteQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVWriteQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVWriteQueue;

  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; 
  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToMgmtQueue; 
@ -72,7 +72,7 @@ int32_t dnodeInitServer() {
    return -1;
  }

-  dInfo("inter-dnodes RPC server is opened");
+  dInfo("dnode inter-dnodes RPC server is initialized");
  return 0;
 }

@ -137,7 +137,7 @@ int32_t dnodeInitClient() {
    return -1;
  }

-  dInfo("inter-dnodes rpc client is opened");
+  dInfo("dnode inter-dnodes rpc client is initialized");
  return 0;
 }

@ -145,13 +145,13 @@ void dnodeCleanupClient() {
  if (tsDnodeClientRpc) {
    rpcClose(tsDnodeClientRpc);
    tsDnodeClientRpc = NULL;
-    dInfo("inter-dnodes rpc client is closed");
+    dInfo("dnode inter-dnodes rpc client is closed");
  }
 }

 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
  if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
-    dnodeUpdateMnodeEpSetForPeer(pEpSet);
+    dnodeUpdateEpSetForPeer(pEpSet);
  }

  if (dnodeProcessRspMsgFp[pMsg->msgType]) {    
@ -173,7 +173,7 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {

 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
  SRpcEpSet epSet = {0};
-  dnodeGetMnodeEpSetForPeer(&epSet);
+  dnodeGetEpSetForPeer(&epSet);
  rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
 }

--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@ -38,10 +38,10 @@ static int32_t tsDnodeQueryReqNum  = 0;
 static int32_t tsDnodeSubmitReqNum = 0;

 int32_t dnodeInitShell() {
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]         = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]         = dnodeDispatchToVWriteQueue;
  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]          = dnodeDispatchToVnodeReadQueue;
  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH]          = dnodeDispatchToVnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
  
  // the following message shall be treated as mnode write
  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
@ -97,7 +97,7 @@ int32_t dnodeInitShell() {
    return -1;
  }

-  dInfo("shell rpc server is opened");
+  dInfo("dnode shell rpc server is initialized");
  return 0;
 }

@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
  int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
  if (code != TSDB_CODE_APP_NOT_READY) return code;

-  SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
+  SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
  tstrncpy(pMsg->user, user, sizeof(pMsg->user));

  SRpcMsg rpcMsg = {0};
  rpcMsg.pCont = pMsg;
-  rpcMsg.contLen = sizeof(SDMAuthMsg);
+  rpcMsg.contLen = sizeof(SAuthMsg);
  rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
  
  dDebug("user:%s, send auth msg to mnodes", user);
@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
  if (rpcRsp.code != 0) {
    dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
  } else {
-    SDMAuthRsp *pRsp = rpcRsp.pCont;
+    SAuthRsp *pRsp = rpcRsp.pCont;
    dDebug("user:%s, auth msg received from mnodes", user);
    memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
    memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
 void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
  dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid);

-  int32_t contLen = sizeof(SDMConfigTableMsg);
-  SDMConfigTableMsg *pMsg = rpcMallocCont(contLen);
+  int32_t contLen = sizeof(SConfigTableMsg);
+  SConfigTableMsg *pMsg = rpcMallocCont(contLen);

  pMsg->dnodeId = htonl(dnodeGetDnodeId());
  pMsg->vgId = htonl(vgId);
--- a/src/dnode/src/dnodeTelemetry.c
+++ b/src/dnode/src/dnodeTelemetry.c
@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) {
    return;
  }
  
-  if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
+  if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
    dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
  } 

@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() {
    dTrace("failed to create telemetry thread, reason:%s", strerror(errno));
  }

+  dInfo("dnode telemetry is initialized");
  return 0;
 }

--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@ -61,7 +61,7 @@ int32_t dnodeInitVnodeRead() {
    pWorker->workerId = i;
  }

-  dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max);
+  dInfo("dnode read is initialized, min worker:%d max worker:%d", readPool.min, readPool.max);
  return 0;
 }

@ -132,7 +132,7 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
  }
 }

-void *dnodeAllocateVnodeRqueue(void *pVnode) {
+void *dnodeAllocVReadQueue(void *pVnode) {
  pthread_mutex_lock(&readPool.mutex);
  taos_queue queue = taosOpenQueue();
  if (queue == NULL) {
@ -167,7 +167,7 @@ void *dnodeAllocateVnodeRqueue(void *pVnode) {
  return queue;
 }

-void dnodeFreeVnodeRqueue(void *rqueue) {
+void dnodeFreeVReadQueue(void *rqueue) {
  taosCloseQueue(rqueue);

  // dynamically adjust the number of threads
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@ -15,74 +15,65 @@

 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
-#include "taoserror.h"
-#include "tutil.h"
+#include "tglobal.h"
 #include "tqueue.h"
-#include "trpc.h"
 #include "tsdb.h"
 #include "twal.h"
-#include "tdataformat.h"
-#include "tglobal.h"
 #include "tsync.h"
 #include "vnode.h"
-#include "dnodeInt.h"
 #include "syncInt.h"
-#include "dnodeVWrite.h"
-#include "dnodeMgmt.h"
+#include "dnodeInt.h"

 typedef struct {
-  taos_qall  qall;
-  taos_qset  qset;      // queue set
-  pthread_t  thread;    // thread
-  int32_t    workerId;  // worker ID
+  taos_qall qall;
+  taos_qset qset;      // queue set
+  int32_t   workerId;  // worker ID
+  pthread_t thread;    // thread
 } SWriteWorker;

 typedef struct {
-  SRspRet  rspRet;
-  int32_t  processedCount;
-  int32_t  code;
-  void    *pCont;
-  int32_t  contLen;
-  SRpcMsg  rpcMsg;
+  SRspRet rspRet;
+  SRpcMsg rpcMsg;
+  int32_t processedCount;
+  int32_t code;
+  int32_t contLen;
+  void *  pCont;
 } SWriteMsg;

 typedef struct {
-  int32_t        max;        // max number of workers
-  int32_t        nextId;     // from 0 to max-1, cyclic
-  SWriteWorker  *writeWorker;
+  int32_t max;     // max number of workers
+  int32_t nextId;  // from 0 to max-1, cyclic
+  SWriteWorker *worker;
  pthread_mutex_t mutex;
 } SWriteWorkerPool;

+static SWriteWorkerPool tsVWriteWP;
 static void *dnodeProcessWriteQueue(void *param);
-static void  dnodeHandleIdleWorker(SWriteWorker *pWorker);

-SWriteWorkerPool wWorkerPool;
+int32_t dnodeInitVWrite() {
+  tsVWriteWP.max = tsNumOfCores;
+  tsVWriteWP.worker = (SWriteWorker *)tcalloc(sizeof(SWriteWorker), tsVWriteWP.max);
+  if (tsVWriteWP.worker == NULL) return -1;
+  pthread_mutex_init(&tsVWriteWP.mutex, NULL);

-int32_t dnodeInitVnodeWrite() {
-  wWorkerPool.max = tsNumOfCores;
-  wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max);
-  if (wWorkerPool.writeWorker == NULL) return -1;
-  pthread_mutex_init(&wWorkerPool.mutex, NULL);
-
-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    wWorkerPool.writeWorker[i].workerId = i;
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    tsVWriteWP.worker[i].workerId = i;
  }

-  dInfo("dnode write is opened, max worker %d", wWorkerPool.max);
+  dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max);
  return 0;
 }

-void dnodeCleanupVnodeWrite() {
-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+void dnodeCleanupVWrite() {
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    SWriteWorker *pWorker = tsVWriteWP.worker + i;
    if (pWorker->thread) {
      taosQsetThreadResume(pWorker->qset);
    }
  }

-  for (int32_t i = 0; i < wWorkerPool.max; ++i) {
-    SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+  for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+    SWriteWorker *pWorker = tsVWriteWP.worker + i;
    if (pWorker->thread) {
      pthread_join(pWorker->thread, NULL);
      taosFreeQall(pWorker->qall);
@ -90,13 +81,13 @@ void dnodeCleanupVnodeWrite() {
    }
  }

-  pthread_mutex_destroy(&wWorkerPool.mutex);
-  free(wWorkerPool.writeWorker);
-  dInfo("dnode write is closed");
+  pthread_mutex_destroy(&tsVWriteWP.mutex);
+  tfree(tsVWriteWP.worker);
+  dInfo("dnode vwrite is closed");
 }

-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
-  char *pCont = (char *)pMsg->pCont;
+void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg) {
+  char *pCont = pMsg->pCont;

  if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
    SMsgDesc *pDesc = (SMsgDesc *)pCont;
@ -111,7 +102,7 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
  taos_queue queue = vnodeAcquireWqueue(pHead->vgId);
  if (queue) {
    // put message into queue
-    SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg));
+    SWriteMsg *pWrite = taosAllocateQitem(sizeof(SWriteMsg));
    pWrite->rpcMsg    = *pMsg;
    pWrite->pCont     = pCont;
    pWrite->contLen   = pHead->contLen;
@ -130,12 +121,12 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
  }
 }

-void *dnodeAllocateVnodeWqueue(void *pVnode) {
-  pthread_mutex_lock(&wWorkerPool.mutex);
-  SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
+void *dnodeAllocVWriteQueue(void *pVnode) {
+  pthread_mutex_lock(&tsVWriteWP.mutex);
+  SWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
  void *queue = taosOpenQueue();
  if (queue == NULL) {
-    pthread_mutex_unlock(&wWorkerPool.mutex);
+    pthread_mutex_unlock(&tsVWriteWP.mutex);
    return NULL;
  }

@ -143,7 +134,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
    pWorker->qset = taosOpenQset();
    if (pWorker->qset == NULL) {
      taosCloseQueue(queue);
-      pthread_mutex_unlock(&wWorkerPool.mutex);
+      pthread_mutex_unlock(&tsVWriteWP.mutex);
      return NULL;
    }

@ -152,7 +143,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
    if (pWorker->qall == NULL) {
      taosCloseQset(pWorker->qset);
      taosCloseQueue(queue);
-      pthread_mutex_unlock(&wWorkerPool.mutex);
+      pthread_mutex_unlock(&tsVWriteWP.mutex);
      return NULL;
    }
    pthread_attr_t thAttr;
@ -160,37 +151,35 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);

    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
-      dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+      dError("failed to create thread to process vwrite queue since %s", strerror(errno));
      taosFreeQall(pWorker->qall);
      taosCloseQset(pWorker->qset);
      taosCloseQueue(queue);
      queue = NULL;
    } else {
-      dDebug("write worker:%d is launched", pWorker->workerId);
-      wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+      dDebug("dnode vwrite worker:%d is launched", pWorker->workerId);
+      tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
    }

    pthread_attr_destroy(&thAttr);
  } else {
    taosAddIntoQset(pWorker->qset, queue, pVnode);
-    wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+    tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
  }

-  pthread_mutex_unlock(&wWorkerPool.mutex);
-  dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue);
+  pthread_mutex_unlock(&tsVWriteWP.mutex);
+  dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue);

  return queue;
 }

-void dnodeFreeVnodeWqueue(void *wqueue) {
+void dnodeFreeVWriteQueue(void *wqueue) {
  taosCloseQueue(wqueue);
-
-  // dynamically adjust the number of threads
 }

-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
-  SWriteMsg *pWrite = (SWriteMsg *)param;
-  if (pWrite == NULL) return;
+void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) {
+  if (param == NULL) return;
+  SWriteMsg *pWrite = param;

  if (code < 0) pWrite->code = code;
  int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
@ -215,44 +204,45 @@ static void *dnodeProcessWriteQueue(void *param) {
  SWriteWorker *pWorker = (SWriteWorker *)param;
  SWriteMsg *   pWrite;
  SWalHead *    pHead;
-  int32_t       numOfMsgs;
-  int           type;
-  void *        pVnode, *item;
  SRspRet *     pRspRet;
+  void *        pVnode;
+  void *        pItem;
+  int32_t       numOfMsgs;
+  int32_t       qtype;

-  dDebug("write worker:%d is running", pWorker->workerId);
+  dDebug("dnode vwrite worker:%d is running", pWorker->workerId);

  while (1) {
    numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
    if (numOfMsgs == 0) {
-      dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset);
+      dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset);
      break;
    }

    for (int32_t i = 0; i < numOfMsgs; ++i) {
      pWrite = NULL;
      pRspRet = NULL;
-      taosGetQitem(pWorker->qall, &type, &item);
-      if (type == TAOS_QTYPE_RPC) {
-        pWrite = (SWriteMsg *)item;
+      taosGetQitem(pWorker->qall, &qtype, &pItem);
+      if (qtype == TAOS_QTYPE_RPC) {
+        pWrite = pItem;
        pRspRet = &pWrite->rspRet;
-        pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
+        pHead = (SWalHead *)((char *)pWrite->pCont - sizeof(SWalHead));
        pHead->msgType = pWrite->rpcMsg.msgType;
        pHead->version = 0;
        pHead->len = pWrite->contLen;
        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
               taosMsg[pWrite->rpcMsg.msgType]);
-      } else if (type == TAOS_QTYPE_CQ) {
-        pHead = (SWalHead *)((char*)item + sizeof(SSyncHead));
+      } else if (qtype == TAOS_QTYPE_CQ) {
+        pHead = (SWalHead *)((char *)pItem + sizeof(SSyncHead));
        dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
               pHead->version);
      } else {
-        pHead = (SWalHead *)item;
+        pHead = pItem;
        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
               pHead->version);
      }

-      int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
+      int32_t code = vnodeProcessWrite(pVnode, qtype, pHead, pRspRet);
      dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
             pHead->version, tstrerror(code));

@ -267,17 +257,17 @@ static void *dnodeProcessWriteQueue(void *param) {
    // browse all items, and process them one by one
    taosResetQitems(pWorker->qall);
    for (int32_t i = 0; i < numOfMsgs; ++i) {
-      taosGetQitem(pWorker->qall, &type, &item);
-      if (type == TAOS_QTYPE_RPC) {
-        pWrite = (SWriteMsg *)item;
-        dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code);
-      } else if (type == TAOS_QTYPE_FWD) {
-        pHead = (SWalHead *)item;
+      taosGetQitem(pWorker->qall, &qtype, &pItem);
+      if (qtype == TAOS_QTYPE_RPC) {
+        pWrite = pItem;
+        dnodeSendRpcVWriteRsp(pVnode, pItem, pWrite->rpcMsg.code);
+      } else if (qtype == TAOS_QTYPE_FWD) {
+        pHead = pItem;
        vnodeConfirmForward(pVnode, pHead->version, 0);
-        taosFreeQitem(item);
+        taosFreeQitem(pItem);
        vnodeRelease(pVnode);
      } else {
-        taosFreeQitem(item);
+        taosFreeQitem(pItem);
        vnodeRelease(pVnode);
      }
    }
@ -285,19 +275,3 @@ static void *dnodeProcessWriteQueue(void *param) {

  return NULL;
 }
-
-UNUSED_FUNC
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
-  int32_t num = taosGetQueueNumber(pWorker->qset);
-
-  if (num > 0) {
-    usleep(30000);
-    sched_yield();
-  } else {
-    taosFreeQall(pWorker->qall);
-    taosCloseQset(pWorker->qset);
-    pWorker->qset = NULL;
-    dDebug("write worker:%d is released", pWorker->workerId);
-    pthread_exit(NULL);
-  }
-}
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@ -21,6 +21,7 @@ extern "C" {
 #endif

 #include "trpc.h"
+#include "taosmsg.h"

 typedef struct {
  int32_t queryReqNum;
@ -38,12 +39,13 @@ SDnodeRunStatus dnodeGetRunStatus();
 SDnodeStatisInfo dnodeGetStatisInfo();

 bool    dnodeIsFirstDeploy();
-char *  dnodeGetMnodeMasterEp();
-void    dnodeGetMnodeEpSetForPeer(void *epSet);
-void    dnodeGetMnodeEpSetForShell(void *epSet);
-void *  dnodeGetMnodeInfos();
+bool    dnodeIsMasterEp(char *ep);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
 int32_t dnodeGetDnodeId();
-bool    dnodeStartMnode(void *pModes);
+void    dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+bool    dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+bool    dnodeStartMnode(SMnodeInfos *minfos);

 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
@ -51,11 +53,11 @@ void  dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
 void  dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);
 void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid);

-void *dnodeAllocateVnodeWqueue(void *pVnode);
-void  dnodeFreeVnodeWqueue(void *queue);
-void *dnodeAllocateVnodeRqueue(void *pVnode);
-void  dnodeFreeVnodeRqueue(void *rqueue);
-void  dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
+void *dnodeAllocVWriteQueue(void *pVnode);
+void  dnodeFreeVWriteQueue(void *wqueue);
+void *dnodeAllocVReadQueue(void *pVnode);
+void  dnodeFreeVReadQueue(void *rqueue);
+void  dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);

 int32_t dnodeAllocateMnodePqueue();
 void    dnodeFreeMnodePqueue();
--- a/src/inc/taos.h
+++ b/src/inc/taos.h
@ -64,7 +64,7 @@ typedef struct taosField {
 #endif

 DLL_EXPORT void  taos_init();
-DLL_EXPORT void  taos_cleanup();
+DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )

+  
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
+
+
 #ifndef TAOS_MESSAGE_C
  TSDB_MSG_TYPE_MAX  // 105
 #endif
@ -291,7 +295,7 @@ typedef struct {
  SSchema schema[];
  // tagVal is padded after schema
  // char    tagVal[];
-} SCMAlterTableMsg;
+} SAlterTableMsg;

 typedef struct {
  SMsgHead  head;
@ -308,12 +312,12 @@ typedef struct {
 } SUpdateTableTagValMsg;

 typedef struct {
-  char clientVersion[TSDB_VERSION_LEN];
-  char msgVersion[TSDB_VERSION_LEN];
-  char db[TSDB_TABLE_FNAME_LEN];
-  char appName[TSDB_APPNAME_LEN];
+  char    clientVersion[TSDB_VERSION_LEN];
+  char    msgVersion[TSDB_VERSION_LEN];
+  char    db[TSDB_TABLE_FNAME_LEN];
+  char    appName[TSDB_APPNAME_LEN];
  int32_t pid;
-} SCMConnectMsg;
+} SConnectMsg;

 typedef struct {
  char      acctId[TSDB_ACCT_LEN];
@ -324,7 +328,7 @@ typedef struct {
  int8_t    reserved2;
  int32_t   connId;
  SRpcEpSet epSet;
-} SCMConnectRsp;
+} SConnectRsp;

 typedef struct {
  int32_t maxUsers;
@ -344,18 +348,18 @@ typedef struct {
  char     user[TSDB_USER_LEN];
  char     pass[TSDB_KEY_LEN];
  SAcctCfg cfg;
-} SCMCreateAcctMsg, SCMAlterAcctMsg;
+} SCreateAcctMsg, SAlterAcctMsg;

 typedef struct {
  char user[TSDB_USER_LEN];
-} SCMDropUserMsg, SCMDropAcctMsg;
+} SDropUserMsg, SDropAcctMsg;

 typedef struct {
  char   user[TSDB_USER_LEN];
  char   pass[TSDB_KEY_LEN];
  int8_t privilege;
  int8_t flag;
-} SCMCreateUserMsg, SCMAlterUserMsg;
+} SCreateUserMsg, SAlterUserMsg;

 typedef struct {
  int32_t  contLen;
@ -370,11 +374,11 @@ typedef struct {
  int32_t  vgId;
  uint64_t uid;
  char    tableId[TSDB_TABLE_FNAME_LEN];
-} SMDDropSTableMsg;
+} SDropSTableMsg;

 typedef struct {
  int32_t vgId;
-} SMDDropVnodeMsg;
+} SDropVnodeMsg;

 typedef struct SColIndex {
  int16_t  colId;      // column id
@ -540,12 +544,12 @@ typedef struct {
  int8_t   replications;
  int8_t   quorum;
  int8_t   ignoreExist;
-} SCMCreateDbMsg, SCMAlterDbMsg;
+} SCreateDbMsg, SAlterDbMsg;

 typedef struct {
  char    db[TSDB_TABLE_FNAME_LEN];
  uint8_t ignoreNotExists;
-} SCMDropDbMsg, SCMUseDbMsg;
+} SDropDbMsg, SUseDbMsg;

 // IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed
 // TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE
@ -560,7 +564,7 @@ typedef struct {
 typedef struct {
  int32_t  vgId;
  int8_t   accessState;
-} SDMVgroupAccess;
+} SVgroupAccess;

 typedef struct {
  int32_t  dnodeId;
@ -568,18 +572,29 @@ typedef struct {
  uint32_t numOfVnodes;
  char     clusterId[TSDB_CLUSTER_ID_LEN];
  char     reserved[16];
-} SDMDnodeCfg;
+} SDnodeCfg;

 typedef struct {
-  int32_t   nodeId;
-  char      nodeEp[TSDB_EP_LEN];
-} SDMMnodeInfo;
+  int32_t  dnodeId;
+  uint16_t dnodePort;
+  char     dnodeFqdn[TSDB_FQDN_LEN];
+} SDnodeEp;

 typedef struct {
-  int8_t       inUse;
-  int8_t       nodeNum;
-  SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
-} SDMMnodeInfos;
+  int32_t  dnodeNum;
+  SDnodeEp dnodeEps[];
+} SDnodeEps;
+
+typedef struct {
+  int32_t mnodeId;
+  char    mnodeEp[TSDB_EP_LEN];
+} SMnodeInfo;
+
+typedef struct {
+  int8_t     inUse;
+  int8_t     mnodeNum;
+  SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA];
+} SMnodeInfos;

 typedef struct {
  int32_t  numOfMnodes;               // tsNumOfMnodes
@ -611,13 +626,13 @@ typedef struct {
  uint8_t     reserve2[15];
  SClusterCfg clusterCfg;
  SVnodeLoad  load[];
-} SDMStatusMsg;
+} SStatusMsg;

 typedef struct {
-  SDMMnodeInfos    mnodes;
-  SDMDnodeCfg      dnodeCfg;
-  SDMVgroupAccess  vgAccess[];
-} SDMStatusRsp;
+  SMnodeInfos   mnodes;
+  SDnodeCfg     dnodeCfg;
+  SVgroupAccess vgAccess[];
+} SStatusRsp;

 typedef struct {
  uint32_t vgId;
@ -640,54 +655,54 @@ typedef struct {
  int8_t   wals;
  int8_t   quorum;
  int8_t   reserved[16];
-} SMDVnodeCfg;
+} SVnodeCfg;

 typedef struct {
  int32_t  nodeId;
  char     nodeEp[TSDB_EP_LEN];
-} SMDVnodeDesc;
+} SVnodeDesc;

 typedef struct {
-  char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
-  SMDVnodeCfg  cfg;
-  SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
-} SMDCreateVnodeMsg, SMDAlterVnodeMsg;
+  char       db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  SVnodeCfg  cfg;
+  SVnodeDesc nodes[TSDB_MAX_REPLICA];
+} SCreateVnodeMsg, SAlterVnodeMsg;

 typedef struct {
  char    tableId[TSDB_TABLE_FNAME_LEN];
  int16_t createFlag;
  char    tags[];
-} SCMTableInfoMsg;
+} STableInfoMsg;

 typedef struct {
  int32_t numOfTables;
  char    tableIds[];
-} SCMMultiTableInfoMsg;
+} SMultiTableInfoMsg;

-typedef struct SCMSTableVgroupMsg {
+typedef struct SSTableVgroupMsg {
  int32_t numOfTables;
-} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg;
+} SSTableVgroupMsg, SSTableVgroupRspMsg;

 typedef struct {
  int32_t       vgId;
  int8_t        numOfEps;
  SEpAddr1      epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupInfo;
+} SVgroupInfo;

 typedef struct {
  int32_t    vgId;
  int8_t     numOfEps;
  SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupMsg;
+} SVgroupMsg;

 typedef struct {
  int32_t numOfVgroups;
-  SCMVgroupInfo vgroups[];
+  SVgroupInfo vgroups[];
 } SVgroupsInfo;

 typedef struct {
  int32_t numOfVgroups;
-  SCMVgroupMsg vgroups[];
+  SVgroupMsg vgroups[];
 } SVgroupsMsg;

 typedef struct STableMetaMsg {
@ -702,7 +717,7 @@ typedef struct STableMetaMsg {
  int16_t       tversion;
  int32_t       tid;
  uint64_t      uid;
-  SCMVgroupMsg  vgroup;
+  SVgroupMsg    vgroup;
  SSchema       schema[];
 } STableMetaMsg;

@ -728,38 +743,38 @@ typedef struct {
  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
  uint16_t payloadLen;
  char     payload[];
-} SCMShowMsg;
+} SShowMsg;

-typedef struct SCMShowRsp {
+typedef struct SShowRsp {
  uint64_t      qhandle;
  STableMetaMsg tableMeta;
-} SCMShowRsp;
+} SShowRsp;

 typedef struct {
-  char     ep[TSDB_EP_LEN];  // end point, hostname:port
-} SCMCreateDnodeMsg, SCMDropDnodeMsg;
+  char ep[TSDB_EP_LEN];  // end point, hostname:port
+} SCreateDnodeMsg, SDropDnodeMsg;

 typedef struct {
  int32_t dnodeId;
  char    dnodeEp[TSDB_EP_LEN];  // end point, hostname:port
-  SDMMnodeInfos mnodes;
-} SMDCreateMnodeMsg;
+  SMnodeInfos mnodes;
+} SCreateMnodeMsg;

 typedef struct {
  int32_t dnodeId;
  int32_t vgId;
  int32_t tid;
-} SDMConfigTableMsg;
+} SConfigTableMsg;

 typedef struct {
  uint32_t dnodeId;
  int32_t  vgId;
-} SDMConfigVnodeMsg;
+} SConfigVnodeMsg;

 typedef struct {
  char ep[TSDB_EP_LEN];  // end point, hostname:port
  char config[64];
-} SMDCfgDnodeMsg, SCMCfgDnodeMsg;
+} SCfgDnodeMsg;

 typedef struct {
  char     sql[TSDB_SHOW_SQL_LEN];
@ -781,13 +796,14 @@ typedef struct {
 } SStreamDesc;

 typedef struct {
+  char     clientVer[TSDB_VERSION_LEN];
  uint32_t connId;
  int32_t  pid;
  int32_t  numOfQueries;
  int32_t  numOfStreams;
  char     appName[TSDB_APPNAME_LEN];
  char     pData[];
-} SCMHeartBeatMsg;
+} SHeartBeatMsg;

 typedef struct {
  uint32_t  queryId;
@ -797,11 +813,11 @@ typedef struct {
  uint32_t  connId;
  int8_t    killConnection;
  SRpcEpSet epSet;
-} SCMHeartBeatRsp;
+} SHeartBeatRsp;

 typedef struct {
  char queryId[TSDB_KILL_MSG_LEN + 1];
-} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg;
+} SKillQueryMsg, SKillStreamMsg, SKillConnMsg;

 typedef struct {
  int32_t  vnode;
@ -810,7 +826,7 @@ typedef struct {
  uint64_t stime;  // stream starting time
  int32_t  status;
  char     tableId[TSDB_TABLE_FNAME_LEN];
-} SMDAlterStreamMsg;
+} SAlterStreamMsg;

 typedef struct {
  char user[TSDB_USER_LEN];
@ -818,7 +834,7 @@ typedef struct {
  char encrypt;
  char secret[TSDB_KEY_LEN];
  char ckey[TSDB_KEY_LEN];
-} SDMAuthMsg, SDMAuthRsp;
+} SAuthMsg, SAuthRsp;

 #pragma pack(pop)

--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@ -68,7 +68,7 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin

 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int      (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index); 
+typedef int32_t  (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); 
 
 // when a forward pkt is received, call this to handle data 
 typedef int      (*FWriteToCache)(void *ahandle, void *pHead, int type);
--- a/src/inc/twal.h
+++ b/src/inc/twal.h
@ -19,43 +19,47 @@
 extern "C" {
 #endif

-#define TAOS_WAL_NOLOG   0
-#define TAOS_WAL_WRITE   1
-#define TAOS_WAL_FSYNC   2
- 
+typedef enum {
+  TAOS_WAL_NOLOG = 0,
+  TAOS_WAL_WRITE = 1,
+  TAOS_WAL_FSYNC = 2
+} EWalType;
+
 typedef struct {
-  int8_t    msgType;
-  int8_t    reserved[3];
-  int32_t   len;
-  uint64_t  version;
-  uint32_t  signature;
-  uint32_t  cksum;
-  char      cont[];
+  int8_t   msgType;
+  int8_t   reserved[3];
+  int32_t  len;
+  uint64_t version;
+  uint32_t signature;
+  uint32_t cksum;
+  char     cont[];
 } SWalHead;

 typedef struct {
-  int8_t    walLevel;  // wal level
-  int32_t   fsyncPeriod; // millisecond
-  int8_t    wals;      // number of WAL files;
-  int8_t    keep;      // keep the wal file when closed
+  int32_t vgId;
+  int32_t fsyncPeriod;  // millisecond
+  int8_t  walLevel;     // wal level
+  int8_t  wals;         // number of WAL files;
+  int8_t  keep;         // keep the wal file when closed
 } SWalCfg;

 typedef void* twalh;  // WAL HANDLE
 typedef int (*FWalWrite)(void *ahandle, void *pHead, int type);

-twalh   walOpen(const char *path, const SWalCfg *pCfg);
-int     walAlter(twalh pWal, const SWalCfg *pCfg);
+int32_t walInit();
+void    walCleanUp();
+
+twalh   walOpen(char *path, SWalCfg *pCfg);
+int32_t walAlter(twalh pWal, SWalCfg *pCfg);
+void    walStop(twalh);
 void    walClose(twalh);
-int     walRenew(twalh);
-int     walWrite(twalh, SWalHead *);
+int32_t walRenew(twalh);
+int32_t walWrite(twalh, SWalHead *);
 void    walFsync(twalh);
-int     walRestore(twalh, void *pVnode, FWalWrite writeFp);
-int     walGetWalFile(twalh, char *name, uint32_t *index);
+int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
+int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
 int64_t walGetVersion(twalh);

-extern int wDebugFlag;
-
-
 #ifdef __cplusplus
 }
 #endif
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@ -43,10 +43,10 @@ typedef struct {

 extern char *vnodeStatus[];

-int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeDrop(int32_t vgId);
 int32_t vnodeOpen(int32_t vgId, char *rootDir);
-int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);

 void*   vnodeAcquire(int32_t vgId);        // add refcount
@ -60,7 +60,7 @@ int32_t vnodeCheckWrite(void *pVnode);
 int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
 void    vnodeBuildStatusMsg(void *param);
 void    vnodeConfirmForward(void *param, uint64_t version, int32_t code);
-void    vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
+void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);

 int32_t vnodeInitResources();
 void    vnodeCleanupResources();
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@ -244,7 +244,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
    }

    *p++ = c;
-    if (c == ';') {
+    if (c == ';' && quote == 0) {
      c = *p;
      *p = 0;
      if (shellRunSingleCommand(con, cmd) < 0) {
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@ -80,7 +80,10 @@ int main(int argc, char* argv[]) {
  shellParseArgument(argc, argv, &args);

  if (args.netTestRole && args.netTestRole[0] != 0) {
-    taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole);
+    taos_init();
+    CmdArguments cmdArgs;
+    memcpy(&cmdArgs, &args, sizeof(SShellArguments));
+    taosNetTest(&cmdArgs);
    exit(0);
  }

--- a/src/mnode/inc/mnodeProfile.h
+++ b/src/mnode/inc/mnodeProfile.h
@ -45,7 +45,7 @@ void    mnodeCleanupProfile();
 SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app);
 SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port);
 void      mnodeReleaseConn(SConnObj *pConn);
-int32_t   mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg);
+int32_t   mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg);

 #ifdef __cplusplus
 }
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@ -41,7 +41,7 @@
 static void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;

-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg);
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg);
 static int32_t mnodeDropDb(SMnodeMsg *newMsg);
 static int32_t mnodeSetDbDropping(SDbObj *pDb);
 static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@ -352,7 +352,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
  return code;
 }

-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
  int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
  if (code != 0) return code;

@ -805,7 +805,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) {
 }

 static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
-  SCMCreateDbMsg *pCreate  = pMsg->rpcMsg.pCont;  
+  SCreateDbMsg *pCreate    = pMsg->rpcMsg.pCont;  
  pCreate->maxTables       = htonl(pCreate->maxTables);
  pCreate->cacheBlockSize  = htonl(pCreate->cacheBlockSize);
  pCreate->totalBlocks     = htonl(pCreate->totalBlocks);
@ -830,7 +830,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
  return code;
 }

-static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
+static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
  SDbCfg  newCfg = pDb->cfg;
  int32_t maxTables      = htonl(pAlter->maxTables);
  int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize);
@ -977,7 +977,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
  return TSDB_CODE_SUCCESS;
 }

-static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
+static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
  SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
  if (terrno != TSDB_CODE_SUCCESS) {
    return terrno;
@ -1009,7 +1009,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
 }

 static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
-  SCMAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
+  SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
  mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle);

  if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db);
@ -1060,7 +1060,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) {
-  SCMDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
+  SDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
  mDebug("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->rpcMsg.handle);

  if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db);
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@ -39,11 +39,15 @@
 #include "mnodeCluster.h"

 int32_t tsAccessSquence = 0;
-static void   *tsDnodeSdb = NULL;
+static void *  tsDnodeSdb = NULL;
 static int32_t tsDnodeUpdateSize = 0;
 extern void *  tsMnodeSdb;
 extern void *  tsVgroupSdb;

+static SDnodeEps*tsDnodeEps;
+static int32_t   tsDnodeEpsSize;
+static pthread_mutex_t tsDnodeEpsMutex;
+
 static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg);
 static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg);
@ -59,6 +63,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
 static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static char*   mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole);
+static void    mnodeUpdateDnodeEps();

 static char* offlineReason[] = {
  "",
@ -95,6 +100,9 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
    pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
  }

+  dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort);
+  mnodeUpdateDnodeEps();
+
  mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
  return TSDB_CODE_SUCCESS;
 }
@ -107,6 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
 #endif  
  mnodeDropMnodeLocal(pDnode->dnodeId);
  balanceAsyncNotify();
+  mnodeUpdateDnodeEps();

  mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
  return TSDB_CODE_SUCCESS;
@ -121,6 +130,7 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
  }
  mnodeDecDnodeRef(pDnode);

+  mnodeUpdateDnodeEps();
  return TSDB_CODE_SUCCESS;
 }

@ -152,12 +162,14 @@ static int32_t mnodeDnodeActionRestored() {
    }
  }

+  mnodeUpdateDnodeEps();
  return TSDB_CODE_SUCCESS;
 }

 int32_t mnodeInitDnodes() {
  SDnodeObj tObj;
  tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+  pthread_mutex_init(&tsDnodeEpsMutex, NULL);

  SSdbTableDesc tableDesc = {
    .tableId      = SDB_TABLE_DNODE,
@ -201,6 +213,9 @@ int32_t mnodeInitDnodes() {

 void mnodeCleanupDnodes() {
  sdbCloseTable(tsDnodeSdb);
+  pthread_mutex_destroy(&tsDnodeEpsMutex);
+  free(tsDnodeEps);
+  tsDnodeEps = NULL;
  tsDnodeSdb = NULL;
 }

@ -299,7 +314,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
    return TSDB_CODE_MND_NO_RIGHTS;
  }
  
-  SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
+  SCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
  if (pCmCfgDnode->ep[0] == 0) {
    tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN);
  }
@ -334,7 +349,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
    mnodeDecDnodeRef(pDnode);
    return code;
  } else {
-    SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
+    SCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg));
    strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
    strcpy(pMdCfgDnode->config, pCmCfgDnode->config);

@ -343,7 +358,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
      .code = 0,
      .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE,
      .pCont = pMdCfgDnode,
-      .contLen = sizeof(SMDCfgDnodeMsg)
+      .contLen = sizeof(SCfgDnodeMsg)
    };

    mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
@ -418,9 +433,54 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
  return 0;
 }

+static int32_t mnodeGetDnodeEpsSize() {
+  pthread_mutex_lock(&tsDnodeEpsMutex);
+  int32_t size = tsDnodeEpsSize;
+  pthread_mutex_unlock(&tsDnodeEpsMutex);
+  return size;
+}
+
+static void mnodeGetDnodeEpsData(SDnodeEps *pEps, int32_t epsSize) {
+  pthread_mutex_lock(&tsDnodeEpsMutex);
+  if (epsSize == tsDnodeEpsSize) {
+    memcpy(pEps, tsDnodeEps, tsDnodeEpsSize);
+  }
+  pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
+static void mnodeUpdateDnodeEps() {
+  pthread_mutex_lock(&tsDnodeEpsMutex);
+
+  int32_t totalDnodes = mnodeGetDnodesNum();
+  tsDnodeEpsSize = sizeof(SDnodeEps) + totalDnodes * sizeof(SDnodeEp);
+  free(tsDnodeEps);
+  tsDnodeEps = calloc(1, tsDnodeEpsSize);
+  tsDnodeEps->dnodeNum = htonl(totalDnodes);
+
+  SDnodeObj *pDnode = NULL;
+  void *     pIter = NULL;
+  int32_t    dnodesNum = 0;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (dnodesNum >= totalDnodes) break;
+
+    SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
+    dnodesNum++;
+    pEp->dnodeId = htonl(pDnode->dnodeId);
+    pEp->dnodePort = htons(pDnode->dnodePort);
+    tstrncpy(pEp->dnodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+  pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
 static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
  SDnodeObj *pDnode     = NULL;
-  SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont;
+  SStatusMsg *pStatus   = pMsg->rpcMsg.pCont;
  pStatus->dnodeId      = htonl(pStatus->dnodeId);
  pStatus->moduleStatus = htonl(pStatus->moduleStatus);
  pStatus->lastReboot   = htonl(pStatus->lastReboot);
@ -477,8 +537,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
  }

  int32_t openVnodes = htons(pStatus->openVnodes);
-  int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
-  SDMStatusRsp *pRsp = rpcMallocCont(contLen);
+  int32_t epsSize = mnodeGetDnodeEpsSize();
+  int32_t vgAccessSize = openVnodes * sizeof(SVgroupAccess);
+  int32_t contLen = sizeof(SStatusRsp) + vgAccessSize + epsSize;
+
+  SStatusRsp *pRsp = rpcMallocCont(contLen);
  if (pRsp == NULL) {
    mnodeDecDnodeRef(pDnode);
    return TSDB_CODE_MND_OUT_OF_MEMORY;
@ -488,8 +551,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
  pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
  pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
  tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN);
-  SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
-
+  SVgroupAccess *pAccess = (SVgroupAccess *)((char *)pRsp + sizeof(SStatusRsp));
+  
  for (int32_t j = 0; j < openVnodes; ++j) {
    SVnodeLoad *pVload = &pStatus->load[j];
    pVload->vgId = htonl(pVload->vgId);
@ -539,6 +602,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
  
  mnodeDecDnodeRef(pDnode);

+  SDnodeEps *pEps = (SDnodeEps *)((char *)pRsp + sizeof(SStatusRsp) + vgAccessSize);
+  mnodeGetDnodeEpsData(pEps, epsSize);
+
  pMsg->rpcRsp.len = contLen;
  pMsg->rpcRsp.rsp =  pRsp;

@ -645,7 +711,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
-  SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
+  SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;

  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
    return TSDB_CODE_MND_NO_RIGHTS;
@ -655,7 +721,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) {
-  SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
+  SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;

  if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
    return TSDB_CODE_MND_NO_RIGHTS;
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@ -38,7 +38,7 @@ static void *        tsMnodeSdb = NULL;
 static int32_t       tsMnodeUpdateSize = 0;
 static SRpcEpSet     tsMnodeEpSetForShell;
 static SRpcEpSet     tsMnodeEpSetForPeer;
-static SDMMnodeInfos tsMnodeInfos;
+static SMnodeInfos   tsMnodeInfos;
 static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);

@ -70,8 +70,9 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {

  pDnode->isMgmt = true;
  mnodeDecDnodeRef(pDnode);
-  
-  mInfo("mnode:%d, fqdn:%s ep:%s port:%d, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
+
+  mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp,
+        pDnode->dnodePort);
  return TSDB_CODE_SUCCESS;
 }

@ -202,7 +203,7 @@ void mnodeUpdateMnodeEpSet() {

  memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet));
  memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet));
-  memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
+  memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos));

  int32_t index = 0;
  void *  pIter = NULL;
@ -221,8 +222,8 @@ void mnodeUpdateMnodeEpSet() {
      tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
      mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index]));

-      tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
-      strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp);
+      tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId);
+      strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp);

      if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
        tsMnodeEpSetForShell.inUse = index;
@ -238,7 +239,7 @@ void mnodeUpdateMnodeEpSet() {
    mnodeDecMnodeRef(pMnode);
  }

-  tsMnodeInfos.nodeNum = index;
+  tsMnodeInfos.mnodeNum = index;
  tsMnodeEpSetForShell.numOfEps = index;
  tsMnodeEpSetForPeer.numOfEps = index;

@ -260,19 +261,19 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
 }

 char* mnodeGetMnodeMasterEp() {
-  return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp;
+  return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp;
 }

 void mnodeGetMnodeInfos(void *mnodeInfos) {
  mnodeMnodeRdLock();
-  *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
+  *(SMnodeInfos *)mnodeInfos = tsMnodeInfos;
  mnodeMnodeUnLock();
 }

 static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
  mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp);

-  SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg));
+  SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg));
  if (pCreate == NULL) {
    return TSDB_CODE_MND_OUT_OF_MEMORY;
  } else {
@ -280,21 +281,21 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
    tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
    pCreate->mnodes = tsMnodeInfos;
    bool found = false;
-    for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) {
-      if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) {
+    for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
+      if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) {
        found = true;
      }
    }
    if (!found) {
-      pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId);
-      tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
-      pCreate->mnodes.nodeNum++;
+      pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeId = htonl(dnodeId);
+      tstrncpy(pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
+      pCreate->mnodes.mnodeNum++;
    }
  }

  SRpcMsg rpcMsg = {0};
  rpcMsg.pCont = pCreate;
-  rpcMsg.contLen = sizeof(SMDCreateMnodeMsg);
+  rpcMsg.contLen = sizeof(SCreateMnodeMsg);
  rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE;

  SRpcMsg   rpcRsp = {0};
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@ -280,7 +280,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
 }

 // not thread safe, need optimized
-int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
+int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) {
  pConn->numOfQueries = htonl(pHBMsg->numOfQueries);
  if (pConn->numOfQueries > 0) {
    if (pConn->pQueries == NULL) {
@ -561,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
  SUserObj *pUser = pMsg->pUser;
  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;

-  SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+  SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
  mInfo("kill query msg is received, queryId:%s", pKill->queryId);

  const char delim = ':';
@ -592,7 +592,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
  SUserObj *pUser = pMsg->pUser;
  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;

-  SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+  SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
  mInfo("kill stream msg is received, streamId:%s", pKill->queryId);

  const char delim = ':';
@ -623,7 +623,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) {
  SUserObj *pUser = pMsg->pUser;
  if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;

-  SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont;
+  SKillConnMsg *pKill = pMsg->rpcMsg.pCont;
  int32_t connId = atoi(pKill->queryId);
  SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t));
  if (pConn == NULL) {
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
 }

 static int32_t sdbInitWal() {
-  SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
+  SWalCfg walCfg = {.vgId = 1, .walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
  char temp[TSDB_FILENAME_LEN];
  sprintf(temp, "%s/wal", tsMnodeDir);
  tsSdbObj.wal = walOpen(temp, &walCfg);
@ -237,8 +237,8 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3
  return 0;
 }

-static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
-  return walGetWalFile(tsSdbObj.wal, name, index);
+static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
+  return walGetWalFile(tsSdbObj.wal, fileName, fileId);
 }

 static void sdbNotifyRole(void *ahandle, int8_t role) {
@ -312,7 +312,7 @@ void sdbUpdateAsync() {
 }

 void sdbUpdateSync(void *pMnodes) {
-  SDMMnodeInfos *mnodes = pMnodes;
+  SMnodeInfos *mnodes = pMnodes;
  if (!mnodeIsRunning()) {
    mDebug("mnode not start yet, update sync config later");
    return;
@ -346,10 +346,10 @@ void sdbUpdateSync(void *pMnodes) {
    syncCfg.replica = index;
    mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
  } else {
-    for (index = 0; index < mnodes->nodeNum; ++index) {
-      SDMMnodeInfo *node = &mnodes->nodeInfos[index];
-      syncCfg.nodeInfo[index].nodeId = node->nodeId;
-      taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
+    for (index = 0; index < mnodes->mnodeNum; ++index) {
+      SMnodeInfo *node = &mnodes->mnodeInfos[index];
+      syncCfg.nodeInfo[index].nodeId = node->mnodeId;
+      taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
      syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
    }
    syncCfg.replica = index;
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@ -110,7 +110,7 @@ static char *mnodeGetShowType(int32_t showType) {
 }

 static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
-  SCMShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
+  SShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
  if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
    return TSDB_CODE_MND_INVALID_MSG_TYPE;
  }
@ -132,8 +132,8 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
    return TSDB_CODE_MND_OUT_OF_MEMORY;
  }

-  int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
-  SCMShowRsp *pShowRsp = rpcMallocCont(size);
+  int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
+  SShowRsp *pShowRsp = rpcMallocCont(size);
  if (pShowRsp == NULL) {
    mnodeReleaseShowObj(pShow, true);
    return TSDB_CODE_MND_OUT_OF_MEMORY;
@ -146,7 +146,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {

  if (code == TSDB_CODE_SUCCESS) {
    pMsg->rpcRsp.rsp = pShowRsp;
-    pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
+    pMsg->rpcRsp.len = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
    mnodeReleaseShowObj(pShow, false);
    return TSDB_CODE_SUCCESS;
  } else {
@ -232,12 +232,17 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
-  SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp));
-  if (pHBRsp == NULL) {
+  SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp));
+  if (pRsp == NULL) {
    return TSDB_CODE_MND_OUT_OF_MEMORY;
  }

-  SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+  SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+  if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) {
+    rpcFreeCont(pRsp);
+    return TSDB_CODE_TSC_INVALID_VERSION;  // todo change the error code
+  }
+
  SRpcConnInfo connInfo = {0};
  rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo);
    
@ -251,40 +256,40 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
  if (pConn == NULL) {
    // do not close existing links, otherwise
    // mError("failed to create connId, close connect");
-    // pHBRsp->killConnection = 1;
+    // pRsp->killConnection = 1;
  } else {
-    pHBRsp->connId = htonl(pConn->connId);
+    pRsp->connId = htonl(pConn->connId);
    mnodeSaveQueryStreamList(pConn, pHBMsg);
    
    if (pConn->killed != 0) {
-      pHBRsp->killConnection = 1;
+      pRsp->killConnection = 1;
    }

    if (pConn->streamId != 0) {
-      pHBRsp->streamId = htonl(pConn->streamId);
+      pRsp->streamId = htonl(pConn->streamId);
      pConn->streamId = 0;
    }

    if (pConn->queryId != 0) {
-      pHBRsp->queryId = htonl(pConn->queryId);
+      pRsp->queryId = htonl(pConn->queryId);
      pConn->queryId = 0;
    }
  }

-  pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
-  pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
-  mnodeGetMnodeEpSetForShell(&pHBRsp->epSet);
+  pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
+  pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
+  mnodeGetMnodeEpSetForShell(&pRsp->epSet);
+
+  pMsg->rpcRsp.rsp = pRsp;
+  pMsg->rpcRsp.len = sizeof(SHeartBeatRsp);

-  pMsg->rpcRsp.rsp = pHBRsp;
-  pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp);
-  
  mnodeReleaseConn(pConn);
  return TSDB_CODE_SUCCESS;
 }

 static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
-  SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
-  SCMConnectRsp *pConnectRsp = NULL;
+  SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
+  SConnectRsp *pConnectRsp = NULL;
  int32_t code = TSDB_CODE_SUCCESS;

  SRpcConnInfo connInfo = {0};
@ -320,7 +325,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
    mnodeDecDbRef(pDb);
  }

-  pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
+  pConnectRsp = rpcMallocCont(sizeof(SConnectRsp));
  if (pConnectRsp == NULL) {
    code = TSDB_CODE_MND_OUT_OF_MEMORY;
    goto connect_over;
@ -349,14 +354,14 @@ connect_over:
  } else {
    mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
    pMsg->rpcRsp.rsp = pConnectRsp;
-    pMsg->rpcRsp.len = sizeof(SCMConnectRsp);
+    pMsg->rpcRsp.len = sizeof(SConnectRsp);
  }

  return code;
 }

 static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) {
-  SCMUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;
+  SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;

  int32_t code = TSDB_CODE_SUCCESS;
  if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db);
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@ -778,7 +778,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
-  SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+  STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
  pInfo->createFlag = htons(pInfo->createFlag);
  mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg,
         pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
@ -915,8 +915,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
      SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
      if (pVgroup == NULL) break;

-      SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
-      pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
+      SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
+      pDrop->contLen = htonl(sizeof(SDropSTableMsg));
      pDrop->vgId = htonl(pVgroup->vgId);
      pDrop->uid = htobe64(pStable->uid);
      mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
@ -924,7 +924,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
      mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
             pVgroup->vgId);
      SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
-      SRpcMsg   rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
+      SRpcMsg   rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
      dnodeSendMsgToDnode(&epSet, &rpcMsg);
      mnodeDecVgroupRef(pVgroup);
    }
@ -1472,31 +1472,31 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
 }

 static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
-  SCMSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
+  SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
  int32_t numOfTable = htonl(pInfo->numOfTables);

  // reserve space
-  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg);
+  int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg);
  for (int32_t i = 0; i < numOfTable; ++i) {
-    char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i;
+    char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
    SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
    if (pTable != NULL && pTable->vgHash != NULL) {
-      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg));
-    } 
-    
+      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg));
+    }
+
    mnodeDecTableRef(pTable);
  }

-  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
+  SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
  if (pRsp == NULL) {
    return TSDB_CODE_MND_OUT_OF_MEMORY;
  }

  pRsp->numOfTables = 0;
-  char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg);
+  char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg);

  for (int32_t i = 0; i < numOfTable; ++i) {
-    char *          stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+    char *          stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
    SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
    if (pTable == NULL) {
      mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName);
@ -1548,7 +1548,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
      pVgroupMsg->numOfVgroups = htonl(vgSize);

      // one table is done, try the next table
-      msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg);
+      msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg);
      pRsp->numOfTables++;
    }
  }
@ -2146,7 +2146,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
 }

 static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
-  SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+  STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
  STagData *pTags = (STagData *)pInfo->tags;
  int32_t tagLen = htonl(pTags->dataLen);
  if (pTags->name[0] == 0) {
@ -2307,7 +2307,7 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
 static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
  return TSDB_CODE_COM_OPS_NOT_SUPPORT;
 #if 0  
-  SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
+  SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
  pCfg->dnodeId = htonl(pCfg->dnodeId);
  pCfg->vgId = htonl(pCfg->vgId);
  pCfg->sid = htonl(pCfg->sid);
@ -2469,7 +2469,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
 }

 static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
-  SCMMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+  SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
  pInfo->numOfTables = htonl(pInfo->numOfTables);

  int32_t totalMallocLen = 4 * 1024 * 1024;  // first malloc 4 MB, subsequent reallocation as twice
@ -2705,7 +2705,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
 }

 static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
-  SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
+  SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
  mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
         pAlter->tableId, pMsg->rpcMsg.handle);

--- a/src/mnode/src/mnodeUser.c
+++ b/src/mnode/src/mnodeUser.c
@ -414,7 +414,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) {
  SUserObj *pOperUser = pMsg->pUser;
  
  if (pOperUser->superAuth) {
-    SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
+    SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
    return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg);
  } else {
    mError("user:%s, no rights to create user", pOperUser->user);
@ -426,7 +426,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
  int32_t code;
  SUserObj *pOperUser = pMsg->pUser;
  
-  SCMAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
+  SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
  SUserObj *pUser = mnodeGetUser(pAlter->user);
  if (pUser == NULL) {
    return TSDB_CODE_MND_INVALID_USER;
@ -514,7 +514,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
  int32_t code;
  SUserObj *pOperUser = pMsg->pUser;

-  SCMDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
+  SDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
  SUserObj *pUser = mnodeGetUser(pDrop->user);
  if (pUser == NULL) {
    return TSDB_CODE_MND_INVALID_USER;
@ -604,11 +604,11 @@ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, cha
 }

 static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) {
-  SDMAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
-  SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp));
+  SAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
+  SAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SAuthRsp));
  
  pMsg->rpcRsp.rsp = pAuthRsp;
-  pMsg->rpcRsp.len = sizeof(SDMAuthRsp);
+  pMsg->rpcRsp.len = sizeof(SAuthRsp);
  
  return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey);
 }
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
  for (int32_t i = 0; i < pShow->maxReplica; ++i) {
    pShow->bytes[cols] = 2;
    pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1);
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
    pSchema[cols].bytes = htons(pShow->bytes[cols]);
    cols++;

    pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
    pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1);
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
    pSchema[cols].bytes = htons(pShow->bytes[cols]);
    cols++;
  }
@ -818,11 +818,11 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
  }
 }

-static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
+static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
  SDbObj *pDb = pVgroup->pDb;
  if (pDb == NULL) return NULL;

-  SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
+  SCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SCreateVnodeMsg));
  if (pVnode == NULL) return NULL;

  strcpy(pVnode->db, pVgroup->dbName);
@ -830,7 +830,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
  //TODO: dynamic alloc tables in tsdb
  maxTables = MAX(10000, tsMaxTablePerVnode);

-  SMDVnodeCfg *pCfg = &pVnode->cfg;
+  SVnodeCfg *pCfg = &pVnode->cfg;
  pCfg->vgId                = htonl(pVgroup->vgId);
  pCfg->cfgVersion          = htonl(pDb->cfgVersion);
  pCfg->cacheBlockSize      = htonl(pDb->cfg.cacheBlockSize);
@ -851,7 +851,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
  pCfg->wals                = 3;
  pCfg->quorum              = pDb->cfg.quorum;
  
-  SMDVnodeDesc *pNodes = pVnode->nodes;
+  SVnodeDesc *pNodes = pVnode->nodes;
  for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
    SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode;
    if (pDnode != NULL) {
@ -886,11 +886,11 @@ SRpcEpSet mnodeGetEpSetFromIp(char *ep) {
 }

 static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) {
-  SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
+  SAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
  SRpcMsg rpcMsg = {
    .ahandle = NULL,
    .pCont   = pAlter,
-    .contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0,
+    .contLen = pAlter ? sizeof(SAlterVnodeMsg) : 0,
    .code    = 0,
    .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE
  };
@ -909,11 +909,11 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) {
 }

 static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) {
-  SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
+  SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
  SRpcMsg rpcMsg = {
    .ahandle = ahandle,
    .pCont   = pCreate,
-    .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0,
+    .contLen = pCreate ? sizeof(SCreateVnodeMsg) : 0,
    .code    = 0,
    .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
  };
@ -982,8 +982,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
  }
 }

-static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
-  SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg));
+static SDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
+  SDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SDropVnodeMsg));
  if (pDrop == NULL) return NULL;

  pDrop->vgId = htonl(vgId);
@ -991,11 +991,11 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
 }

 void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) {
-  SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
+  SDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
  SRpcMsg rpcMsg = {
      .ahandle = ahandle,
      .pCont   = pDrop,
-      .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0,
+      .contLen = pDrop ? sizeof(SDropVnodeMsg) : 0,
      .code    = 0,
      .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE
  };
@ -1044,7 +1044,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
 }

 static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
-  SDMConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
+  SConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
  pCfg->dnodeId = htonl(pCfg->dnodeId);
  pCfg->vgId    = htonl(pCfg->vgId);

--- a/src/os/inc/os.h
+++ b/src/os/inc/os.h
@ -52,9 +52,10 @@ extern "C" {
 #include "osWindows.h"
 #endif

+#include "osDef.h"
+#include "osAlloc.h"
 #include "osAtomic.h"
 #include "osCommon.h"
-#include "osDef.h"
 #include "osDir.h"
 #include "osFile.h"
 #include "osLz4.h"
--- a/src/os/inc/osAlloc.h
+++ b/src/os/inc/osAlloc.h
@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_ALLOC_H
+#define TDENGINE_OS_ALLOC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef TAOS_OS_FUNC_ALLOC
+  #define tmalloc(size) malloc(size)
+  #define tcalloc(nmemb, size) calloc(nmemb, size)
+  #define trealloc(p, size) realloc(p, size)
+  #define tmemalign(alignment, size) malloc(size)
+  #define tfree(p) free(p)
+  #define tmemzero(p, size) memset(p, 0, size)
+#else
+  void *tmalloc(int32_t size);
+  void *tcalloc(int32_t nmemb, int32_t size);
+  void *trealloc(void *p, int32_t size);
+  void *tmemalign(int32_t alignment, int32_t size);
+  void  tfree(void *p);
+  void  tmemzero(void *p, int32_t size);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/src/os/inc/osDarwin.h
+++ b/src/os/inc/osDarwin.h
@ -72,8 +72,6 @@ extern "C" {
 #include <sys/utsname.h>

 #define TAOS_OS_FUNC_FILE_SENDIFLE
-  #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
-  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)

 #define TAOS_OS_FUNC_SEMPHONE
  #define tsem_t dispatch_semaphore_t
--- a/src/os/inc/osFile.h
+++ b/src/os/inc/osFile.h
@ -20,46 +20,52 @@
 extern "C" {
 #endif

-ssize_t taosTReadImp(int fd, void *buf, size_t count);
-ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+#define tread(fd, buf, count) read(fd, buf, count)
+#define twrite(fd, buf, count) write(fd, buf, count)
+#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
+#define tclose(fd)    \
+  {                       \
+    if (FD_VALID(fd)) {    \
+      close(fd);           \
+      fd = FD_INITIALIZER; \
+    }                     \
+  }

-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
-int     taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);

-#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
-  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
-  #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size)
-#endif
+#define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
+#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
+#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
+#define taosClose(x) tclose(x)

-#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
-#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
-#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
+// TAOS_OS_FUNC_FILE_SENDIFLE
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);

 #ifdef TAOS_RANDOM_FILE_FAIL
-  void taosSetRandomFileFailFactor(int factor);
+  void taosSetRandomFileFailFactor(int32_t factor);
  void taosSetRandomFileFailOutput(const char *path);
  #ifdef TAOS_RANDOM_FILE_FAIL_TEST
-    ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
-    ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
-    off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
-    #undef taosTRead
-    #undef taosTWrite
+    int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+    int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+    int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line);
+    #undef taosRead
+    #undef taosWrite
    #undef taosLSeek
-    #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
-    #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+    #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+    #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
    #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
  #endif  
 #endif 

-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
 // TAOS_OS_FUNC_FILE_GETTMPFILEPATH
 void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);

-#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
-  #define taosFtruncate ftruncate
-#endif
-
+// TAOS_OS_FUNC_FILE_FTRUNCATE
+int32_t taosFtruncate(int32_t fd, int64_t length);
 #ifdef __cplusplus
 }
 #endif
--- a/src/os/inc/osSocket.h
+++ b/src/os/inc/osSocket.h
@ -33,21 +33,19 @@ extern "C" {
        x = FD_INITIALIZER;  \
      }                      \
    }
-  typedef int SOCKET;
+  typedef int32_t SOCKET;
 #endif

 #ifndef TAOS_OS_DEF_EPOLL
  #define TAOS_EPOLL_WAIT_TIME -1
 #endif  

-#define taosClose(x) taosCloseSocket(x)
-
 #ifdef TAOS_RANDOM_NETWORK_FAIL
  #ifdef TAOS_RANDOM_NETWORK_FAIL_TEST
-    ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
-    ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
-    ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
-    ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+    int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags);
+    int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+    int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count);
+    int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count);
    #undef taosSend
    #undef taosSendto
    #undef taosReadSocket
@ -60,14 +58,14 @@ extern "C" {
 #endif

 // TAOS_OS_FUNC_SOCKET
-int taosSetNonblocking(SOCKET sock, int on);
-void taosBlockSIGPIPE();
+int32_t taosSetNonblocking(SOCKET sock, int32_t on);
+void    taosBlockSIGPIPE();

 // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen);
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);

 // TAOS_OS_FUNC_SOCKET_INET
-uint32_t taosInetAddr(char *ipAddr);
+uint32_t    taosInetAddr(char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);

 #ifdef __cplusplus
--- a/src/os/inc/osWindows.h
+++ b/src/os/inc/osWindows.h
@ -62,11 +62,8 @@ extern "C" {
 #define TAOS_OS_FUNC_FILE_ISDIR
 #define TAOS_OS_FUNC_FILE_ISLNK
 #define TAOS_OS_FUNC_FILE_SENDIFLE
-  #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
-  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
 #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH
-#define TAOS_OS_FUNC_FILE_FTRUNCATE
-  extern int taosFtruncate(int fd, int64_t length); 
+#define TAOS_OS_FUNC_FILE_FTRUNCATE 

 #define TAOS_OS_FUNC_MATH
  #define SWAP(a, b, c)      \
@ -139,7 +136,6 @@ typedef int (*__compar_fn_t)(const void *, const void *);
 #define in_addr_t unsigned long
 #define socklen_t int
 #define htobe64 htonll
-#define twrite write
 #define getpid _getpid

 struct tm *localtime_r(const time_t *timep, struct tm *result);
--- a/src/os/src/darwin/darwinFile.c
+++ b/src/os/src/darwin/darwinFile.c
@ -19,21 +19,19 @@

 #define _SEND_FILE_STEP_ 1000

-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
  fseek(in_file, (int32_t)(*offset), 0);
-  int writeLen = 0;
-  uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
+  int     writeLen = 0;
+  uint8_t buffer[_SEND_FILE_STEP_] = {0};

  for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
    if (rlen <= 0) {
      return writeLen;
-    }
-    else if (rlen < _SEND_FILE_STEP_) {
+    } else if (rlen < _SEND_FILE_STEP_) {
      fwrite(buffer, 1, rlen, out_file);
      return (int)(writeLen + rlen);
-    }
-    else {
+    } else {
      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
      writeLen += _SEND_FILE_STEP_;
    }
@ -44,8 +42,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
    size_t rlen = fread(buffer, 1, remain, in_file);
    if (rlen <= 0) {
      return writeLen;
-    }
-    else {
+    } else {
      fwrite(buffer, 1, remain, out_file);
      writeLen += remain;
    }
@ -54,7 +51,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
  return writeLen;
 }

-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
-  uError("not implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+  uError("taosSendFile not implemented yet");
  return -1;
 }
--- a/src/os/src/detail/osAlloc.c
+++ b/src/os/src/detail/osAlloc.c
@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "osAlloc.h"
+
+#define TSDB_HAVE_MEMALIGN
+#ifdef TAOS_OS_FUNC_ALLOC
+
+void *tmalloc(int32_t size) {
+  void *p = malloc(size);
+  if (p == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    uError("failed to malloc memory, size:%d reason:%s", size, strerror(errno));
+  }
+
+  return p;
+}
+
+void *tcalloc(int32_t nmemb, int32_t size) {
+  void *p = calloc(nmemb, size);
+  if (p == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    uError("failed to calloc memory, nmemb:%d size:%d reason:%s", nmemb, size, strerror(errno));
+  }
+
+  return p;
+}
+
+void *trealloc(void *p, int32_t size) {
+  p = realloc(p, size);
+  if (p == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    uError("failed to realloc memory, size:%d reason:%s", size, strerror(errno));
+  }
+
+  return p;
+}
+
+void tfree(void *p) { free(p); }
+
+void tmemzero(void *p, int32_t size) { memset(p, 0, size); }
+
+#ifdef TSDB_HAVE_MEMALIGN
+
+void *tmemalign(int32_t alignment, int32_t size) {
+  void *p;
+
+  int err = posix_memalign(&p, alignment, size);
+  if (err) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    uError("failed to memalign memory, alignment:%d size:%d reason:%s", alignment, size, strerror(err));
+    p = NULL;
+  }
+
+  return p;
+}
+
+#else
+
+void *tmemalign(int32_t alignment, int32_t size) { return tmalloc(size); }
+
+#endif
+#endif
--- a/src/os/src/detail/osFail.c
+++ b/src/os/src/detail/osFail.c
@ -20,7 +20,7 @@

 #ifdef TAOS_RANDOM_NETWORK_FAIL

-ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) {
  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
    errno = ECONNRESET;
    return -1;
@ -29,8 +29,8 @@ ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
  return send(sockfd, buf, len, flags);
 }

-ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
-                           socklen_t addrlen) {
+int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags,
+                             const struct sockaddr *dest_addr, socklen_t addrlen) {
  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
    errno = ECONNRESET;
    return -1;
@ -39,7 +39,7 @@ ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags,
  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
 }

-ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) {
  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
    errno = ECONNRESET;
    return -1;
@ -48,7 +48,7 @@ ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
  return read(fd, buf, count);
 }

-ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) {
  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
    errno = EINTR;
    return -1;
@ -61,10 +61,10 @@ ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {

 #ifdef TAOS_RANDOM_FILE_FAIL

-static int random_file_fail_factor = 20;
+static int32_t random_file_fail_factor = 20;
 static FILE *fpRandomFileFailOutput = NULL;

-void taosSetRandomFileFailFactor(int factor) { 
+void taosSetRandomFileFailFactor(int32_t factor) { 
  random_file_fail_factor = factor;
 }

@ -77,7 +77,7 @@ static void close_random_file_fail_output() {
  }
 }

-static void random_file_fail_output_sig(int sig) {
+static void random_file_fail_output_sig(int32_t sig) {
  fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);

  struct sigaction act = {0};
@ -105,7 +105,7 @@ void taosSetRandomFileFailOutput(const char *path) {
  sigaction(SIGILL, &act, NULL);
 }

-ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
  if (random_file_fail_factor > 0) {
    if (rand() % random_file_fail_factor == 0) {
      errno = EIO;
@ -113,10 +113,10 @@ ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file
    }
  }

-  return taosTReadImp(fd, buf, count);
+  return taosReadImp(fd, buf, count);
 }

-ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
  if (random_file_fail_factor > 0) {
    if (rand() % random_file_fail_factor == 0) {
      errno = EIO;
@ -124,10 +124,10 @@ ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *fil
    }
  }

-  return taosTWriteImp(fd, buf, count);
+  return taosWriteImp(fd, buf, count);
 }

-off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
+int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) {
  if (random_file_fail_factor > 0) {
    if (rand() % random_file_fail_factor == 0) {
      errno = EIO;
@ -135,7 +135,7 @@ off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, ui
    }
  }

-  return lseek(fd, offset, whence);
+  return taosLSeekImp(fd, offset, whence);
 }

 #endif //TAOS_RANDOM_FILE_FAIL
--- a/src/os/src/detail/osFile.c
+++ b/src/os/src/detail/osFile.c
@ -15,16 +15,22 @@

 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "tglobal.h"

 #ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH
+
 void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
  const char *tdengineTmpFileNamePrefix = "tdengine-";

  char  tmpPath[PATH_MAX];
-  char *tmpDir = "/tmp/";
+  int32_t len = strlen(tsTempDir);
+  memcpy(tmpPath, tsTempDir, len);

-  strcpy(tmpPath, tmpDir);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (tmpPath[len - 1] != '/') {
+      tmpPath[len++] = '/';
+  }
+
+  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
    strcat(tmpPath, fileNamePrefix);
    strcat(tmpPath, "-%d-%s");
@ -34,10 +40,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
  taosRandStr(rand, tListLen(rand) - 1);
  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
 }
+
 #endif

-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) {
  int32_t ts = taosGetTimestampSec();

  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
@ -46,12 +52,13 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
  if (delimiterPos == NULL) return -1;

  int32_t fileNameLen = 0;
-  if (suffix)
+  if (suffix) {
    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
-  else
+  } else {
    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
+  }

-  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+  int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1);
  if (*dstPath == NULL) {
    *dstPath = calloc(1, len + 1);
    if (*dstPath == NULL) return -1;
@ -64,9 +71,9 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
  return rename(fullPath, *dstPath);
 }

-ssize_t taosTReadImp(int fd, void *buf, size_t count) {
-  size_t  leftbytes = count;
-  ssize_t readbytes;
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count) {
+  int64_t leftbytes = count;
+  int64_t readbytes;
  char *  tbuf = (char *)buf;

  while (leftbytes > 0) {
@ -78,19 +85,19 @@ ssize_t taosTReadImp(int fd, void *buf, size_t count) {
        return -1;
      }
    } else if (readbytes == 0) {
-      return (ssize_t)(count - leftbytes);
+      return (int64_t)(count - leftbytes);
    }

    leftbytes -= readbytes;
    tbuf += readbytes;
  }

-  return (ssize_t)count;
+  return count;
 }

-ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
-  size_t  nleft = n;
-  ssize_t nwritten = 0;
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
+  int64_t nleft = n;
+  int64_t nwritten = 0;
  char *  tbuf = (char *)buf;

  while (nleft > 0) {
@ -105,13 +112,18 @@ ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
    tbuf += nwritten;
  }

-  return (ssize_t)n;
+  return n;
+}
+
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
+  return (int64_t)tlseek(fd, (long)offset, whence);
 }

 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
-  size_t  leftbytes = size;
-  ssize_t sentbytes;
+
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) {
+  int64_t leftbytes = size;
+  int64_t sentbytes;

  while (leftbytes > 0) {
    /*
@ -126,7 +138,7 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
        return -1;
      }
    } else if (sentbytes == 0) {
-      return (ssize_t)(size - leftbytes);
+      return (int64_t)(size - leftbytes);
    }

    leftbytes -= sentbytes;
@ -134,4 +146,17 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {

  return size;
 }
+
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
+  return taosSendFile(fileno(outfile), fileno(infile), offset, size);
+}
+
+#endif
+
+#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
+
+int32_t taosFtruncate(int32_t fd, int64_t length) {
+  return ftruncate(fd, length);
+}
+
 #endif
--- a/src/os/src/detail/osSocket.c
+++ b/src/os/src/detail/osSocket.c
@ -19,8 +19,8 @@

 #ifndef TAOS_OS_FUNC_SOCKET

-int taosSetNonblocking(SOCKET sock, int on) {
-  int flags = 0;
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
+  int32_t flags = 0;
  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
    return 1;
@ -43,7 +43,7 @@ void taosBlockSIGPIPE() {
  sigset_t signal_mask;
  sigemptyset(&signal_mask);
  sigaddset(&signal_mask, SIGPIPE);
-  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+  int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
  if (rc != 0) {
    uError("failed to block SIGPIPE");
  }
@ -53,7 +53,7 @@ void taosBlockSIGPIPE() {

 #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT

-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
 }

--- a/src/os/src/windows/wEnv.c
+++ b/src/os/src/windows/wEnv.c
@ -46,5 +46,16 @@ void osInit() {
  strcpy(tsDnodeDir, "");
  strcpy(tsMnodeDir, "");  
  strcpy(tsOsName, "Windows");
+
+  const char *tmpDir = getenv("tmp");
+  if (tmpDir != NULL) {
+    tmpDir = getenv("temp");
+  }
+  if (tmpDir != NULL) {
+    strcpy(tsTempDir, tmpDir);
+  } else {
+    strcpy(tsTempDir, "C:\\Windows\\Temp");
+  }
+  
  taosWinSocketInit();
 }
--- a/src/os/src/windows/wFile.c
+++ b/src/os/src/windows/wFile.c
@ -16,17 +16,20 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tulog.h"
+#include "tglobal.h"

 void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
  const char* tdengineTmpFileNamePrefix = "tdengine-";
-    char tmpPath[PATH_MAX];
+  char tmpPath[PATH_MAX];

-  char *tmpDir = getenv("tmp");
-  if (tmpDir == NULL) {
-    tmpDir = "";
+  int32_t len = (int32_t)strlen(tsTempDir);
+  memcpy(tmpPath, tsTempDir, len);
+
+  if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
+      tmpPath[len++] = '\\';
  }
-  
-  strcpy(tmpPath, tmpDir);
+
+  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
  strcat(tmpPath, tdengineTmpFileNamePrefix);
  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
    strcat(tmpPath, fileNamePrefix);
@ -40,19 +43,19 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {

 #define _SEND_FILE_STEP_ 1000

-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
  fseek(in_file, (int32_t)(*offset), 0);
-  int writeLen = 0;
+  int64_t writeLen = 0;
  uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
  
-  for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
    if (rlen <= 0) {
      return writeLen;
    }
    else if (rlen < _SEND_FILE_STEP_) {
      fwrite(buffer, 1, rlen, out_file);
-      return (int)(writeLen + rlen);
+      return (int64_t)(writeLen + rlen);
    }
    else {
      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
@ -60,7 +63,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
    }
  }

-  int remain = count - writeLen;
+  int64_t remain = count - writeLen;
  if (remain > 0) {
    size_t rlen = fread(buffer, 1, remain, in_file);
    if (rlen <= 0) {
@ -75,12 +78,12 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
  return writeLen;
 }

-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
-  uError("taosTSendFileImp no implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+  uError("taosSendFile no implemented yet");
  return 0;
 }

-int taosFtruncate(int fd, int64_t length) {
+int32_t taosFtruncate(int32_t fd, int64_t length) {
  uError("taosFtruncate no implemented yet");
  return 0;
 }
--- a/src/os/src/windows/wSocket.c
+++ b/src/os/src/windows/wSocket.c
@ -34,7 +34,7 @@ void taosWinSocketInit() {
  }
 }

-int taosSetNonblocking(SOCKET sock, int on) {
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
  u_long mode;
  if (on) {
    mode = 1;
@ -48,7 +48,7 @@ int taosSetNonblocking(SOCKET sock, int on) {

 void taosBlockSIGPIPE() {}

-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
  if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
    return 0;
  }
@ -72,7 +72,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op

 uint32_t taosInetAddr(char *ipAddr) {
  uint32_t value;
-  int ret = inet_pton(AF_INET, ipAddr, &value);
+  int32_t ret = inet_pton(AF_INET, ipAddr, &value);
  if (ret <= 0) {
    return INADDR_NONE;
  } else {
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@ -40,6 +40,19 @@ typedef struct SGroupResInfo {
  int32_t  rowId;
 } SGroupResInfo;

+typedef struct SResultRowPool {
+  int32_t elemSize;
+  int32_t blockSize;
+  int32_t numOfElemPerBlock;
+
+  struct {
+    int32_t blockIndex;
+    int32_t pos;
+  } position;
+
+  SArray* pData;    // SArray<void*>
+} SResultRowPool;
+
 typedef struct SSqlGroupbyExpr {
  int16_t tableIndex;
  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
@ -48,14 +61,14 @@ typedef struct SSqlGroupbyExpr {
  int16_t orderType;   // order by type: asc/desc
 } SSqlGroupbyExpr;

-typedef struct SWindowResult {
+typedef struct SResultRow {
  int32_t       pageId;      // pageId & rowId is the position of current result in disk-based output buffer
  int32_t       rowId:15;
  bool          closed:1;    // this result status: closed or opened
  uint16_t      numOfRows;   // number of rows of current time window
-  SResultInfo*  resultInfo;  // For each result column, there is a resultInfo
+  SResultRowCellInfo*  pCellInfo;  // For each result column, there is a resultInfo
  union {STimeWindow win; char* key;};  // start key of current time window
-} SWindowResult;
+} SResultRow;

 /**
 * If the number of generated results is greater than this value,
@ -69,16 +82,14 @@ typedef struct SResultRec {
 } SResultRec;

 typedef struct SWindowResInfo {
-  SWindowResult* pResult;    // result list
-  SHashObj*      hashList;   // hash list for quick access
-  int16_t        type;       // data type for hash key
+  SResultRow**   pResult;    // result list
+  int16_t        type:8;     // data type for hash key
+  int32_t        size:24;    // number of result set
+  int32_t        threshold;  // threshold to halt query and return the generated results.
  int32_t        capacity;   // max capacity
  int32_t        curIndex;   // current start active index
-  int32_t        size;       // number of result set
  int64_t        startTime;  // start time of the first time window for sliding query
  int64_t        prevSKey;   // previous (not completed) sliding window start key
-  int64_t        threshold;  // threshold to halt query and return the generated results.
-  int64_t        interval;   // time window interval
 } SWindowResInfo;

 typedef struct SColumnFilterElem {
@ -94,7 +105,7 @@ typedef struct SSingleColumnFilterInfo {
  SColumnFilterElem* pFilters;
 } SSingleColumnFilterInfo;

-typedef struct STableQueryInfo {  // todo merge with the STableQueryInfo struct
+typedef struct STableQueryInfo {
  TSKEY       lastKey;
  int32_t     groupIndex;     // group id in table list
  int16_t     queryRangeSet;  // denote if the query range is set, only available for interval query
@ -122,7 +133,9 @@ typedef struct SQueryCostInfo {
  uint32_t discardBlocks;
  uint64_t elapsedTime;
  uint64_t firstStageMergeTime;
-  uint64_t internalSupSize;
+  uint64_t winInfoSize;
+  uint64_t tableInfoSize;
+  uint64_t hashSize;
  uint64_t numOfTimeWindows;
 } SQueryCostInfo;

@ -155,11 +168,11 @@ typedef struct SQuery {

 typedef struct SQueryRuntimeEnv {
  jmp_buf              env;
-  SResultInfo*         resultInfo;       // todo refactor to merge with SWindowResInfo
+  SResultRow*          pResultRow;       // todo refactor to merge with SWindowResInfo
  SQuery*              pQuery;
  SQLFunctionCtx*      pCtx;
  int32_t              numOfRowsPerPage;
-  int16_t              offset[TSDB_MAX_COLUMNS];
+  uint16_t             offset[TSDB_MAX_COLUMNS];
  uint16_t             scanFlag;         // denotes reversed scan of data or not
  SFillInfo*           pFillInfo;
  SWindowResInfo       windowResInfo;
@ -175,6 +188,11 @@ typedef struct SQueryRuntimeEnv {
  int32_t              interBufSize;     // intermediate buffer sizse
  int32_t              prevGroupId;      // previous executed group id
  SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
+  SHashObj*            pResultRowHashTable; // quick locate the window object for each result
+  char*                keyBuf;           // window key buffer
+  SResultRowPool*      pool;             // window result object pool
+
+  int32_t*             rowCellInfoOffset;// offset value for each row result cell info
 } SQueryRuntimeEnv;

 enum {
--- a/src/query/inc/qFill.h
+++ b/src/query/inc/qFill.h
@ -71,7 +71,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_

 void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);

-void* taosDestoryFillInfo(SFillInfo *pFillInfo);
+void* taosDestroyFillInfo(SFillInfo *pFillInfo);

 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);

--- a/src/query/inc/qSqlparser.h
+++ b/src/query/inc/qSqlparser.h
@ -223,13 +223,6 @@ typedef struct tSQLExprList {
  tSQLExprItem *a;      /* One entry for each expression */
 } tSQLExprList;

-typedef struct tSQLExprListList {
-  int32_t        nList;  /* Number of expressions on the list */
-  int32_t        nAlloc; /* Number of entries allocated below */
-  tSQLExprList **a;      /* one entry for each row */
-} tSQLExprListList;
-
-
 /**
 *
 * @param yyp      The parser
--- a/src/query/inc/qTsbuf.h
+++ b/src/query/inc/qTsbuf.h
@ -135,6 +135,12 @@ int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf);

 void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId);

+int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks);
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
+
+bool tsBufIsValidElem(STSElem* pElem);
+
 #ifdef __cplusplus
 }
 #endif
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@ -15,13 +15,22 @@
 #ifndef TDENGINE_QUERYUTIL_H
 #define TDENGINE_QUERYUTIL_H

+#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
+  do {                                               \
+    assert(sizeof(_uid) == sizeof(uint64_t));        \
+    *(uint64_t *)(_k) = (_uid);                      \
+    memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
+  } while (0)
+
+#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
+
 int32_t getOutputInterResultBufSize(SQuery* pQuery);

-void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
-void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
+void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow);
+void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src);
+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);

-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
-                          int32_t threshold, int16_t type);
+int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);

 void    cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
 void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
@ -33,9 +42,9 @@ void    closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
 void    closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
 void    removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);

-static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
  assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
-  return &pWindowResInfo->pResult[slot];
+  return pWindowResInfo->pResult[slot];
 }

 #define curTimeWindowIndex(_winres)        ((_winres)->curIndex)
@ -43,9 +52,9 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf

 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);

-int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
+int32_t initResultRow(SResultRow *pResultRow);

-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult,
    tFilePage* page) {
  assert(pResult != NULL && pRuntimeEnv != NULL);

@ -62,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
 __filter_func_t *getRangeFilterFuncArray(int32_t type);
 __filter_func_t *getValueFilterFuncArray(int32_t type);

+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv);
+
+SResultRowPool* initResultRowPool(size_t size);
+SResultRow* getNewResultRow(SResultRowPool* p);
+int64_t getResultRowPoolMemSize(SResultRowPool* p);
+void* destroyResultRowPool(SResultRowPool* p);
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
+int32_t getNumOfUsedResultRows(SResultRowPool* p);
+
+
 #endif  // TDENGINE_QUERYUTIL_H
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail {
  int8_t primaryCol;
 } SInterpInfoDetail;

-typedef struct SResultInfo {
+typedef struct SResultRowCellInfo {
  int8_t   hasResult;       // result generated, not NULL value
-  bool     initialized;   // output buffer has been initialized
-  bool     complete;      // query has completed
-  bool     superTableQ;   // is super table query
-  uint32_t bufLen;       // buffer size
-  uint64_t numOfRes;        // num of output result in current buffer
-  void*    interResultBuf;  // output result buffer
-} SResultInfo;
+  bool     initialized;     // output buffer has been initialized
+  bool     complete;        // query has completed
+  uint16_t numOfRes;        // num of output result in current buffer
+} SResultRowCellInfo;
+
+#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))

 struct SQLFunctionCtx;

@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx {
  int16_t      inputBytes;
  
  int16_t      outputType;
-  int16_t      outputBytes;  // size of results, determined by function and input column data type
-  bool         hasNull;      // null value exist in current block
+  int16_t      outputBytes;   // size of results, determined by function and input column data type
+  int32_t      interBufBytes; // internal buffer size
+  bool         hasNull;       // null value exist in current block
  bool         requireNull;  // require null in some function
+  bool         stableQuery;
  int16_t      functionId;   // function id
  void *       aInputElemBuf;
  char *       aOutputBuf;            // final result output buffer, point to sdata->data
@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx {
  void *       ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
  SQLPreAggVal preAggVals;
  tVariant     tag;
-  SResultInfo *resultInfo;
+
+  SResultRowCellInfo *resultInfo;

  SExtTagsInfo tagInfo;
 } SQLFunctionCtx;
@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
    (_r)->initialized = false; \
  } while (0)

-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);
+//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf);

-static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) {
+static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
  pResInfo->initialized = true;  // the this struct has been initialized flag
  
  pResInfo->complete = false;
  pResInfo->hasResult = false;
  pResInfo->numOfRes = 0;
  
-  memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen);
+  memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen);
 }

 #ifdef __cplusplus
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
--- a/src/query/src/qFill.c
+++ b/src/query/src/qFill.c
@ -91,7 +91,7 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
  pFillInfo->numOfTotal   = 0;
 }

-void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
+void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
  if (pFillInfo == NULL) {
    return NULL;
  }
--- a/src/query/src/qResultbuf.c
+++ b/src/query/src/qResultbuf.c
@ -407,14 +407,14 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
  }

  if (pResultBuf->file != NULL) {
-    qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes",
-        pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize,
-        pResultBuf->fileSize);
+    qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f",
+        pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
+        pResultBuf->fileSize/1024.0);

    fclose(pResultBuf->file);
  } else {
-    qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
-           pResultBuf->totalBufSize);
+    qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle,
+           pResultBuf->totalBufSize/1024.0);
  }

  unlink(pResultBuf->path);
--- a/src/query/src/qTokenizer.c
+++ b/src/query/src/qTokenizer.c
@ -251,16 +251,16 @@ static const char isIdChar[] = {
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
 };

-static void* KeywordHashTable = NULL;
+static void* keywordHashTable = NULL;

 static void doInitKeywordsTable(void) {
  int numOfEntries = tListLen(keywordTable);
  
-  KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
+  keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
  for (int32_t i = 0; i < numOfEntries; i++) {
    keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name);
    void* ptr = &keywordTable[i];
-    taosHashPut(KeywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
+    taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
  }
 }

@ -282,7 +282,7 @@ int tSQLKeywordCode(const char* z, int n) {
    }
  }

-  SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n);
+  SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
  return (pKey != NULL)? (*pKey)->type:TK_ID;
 }

@ -660,5 +660,8 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
 bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }

 void taosCleanupKeywordsTable() {
-  taosHashCleanup(KeywordHashTable);
+  void* m = keywordHashTable;
+  if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) {
+    taosHashCleanup(m);
+  }
 }
--- a/src/query/src/qTsbuf.c
+++ b/src/query/src/qTsbuf.c
@ -156,7 +156,8 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
  } else {
 //    tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
  }
-  
+
+  tVariantDestroy(&pTSBuf->block.tag);
  free(pTSBuf);
  return NULL;
 }
@ -218,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) {
  }
 }

+static int32_t getTagAreaLength(tVariant* pa) {
+  int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType);
+  if (pa->nType != TSDB_DATA_TYPE_NULL) {
+    t += pa->nLen;
+  }
+
+  return t;
+}
+
 static void writeDataToDisk(STSBuf* pTSBuf) {
  if (pTSBuf->tsData.len == 0) {
    return;
@ -243,19 +253,27 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
   */
  int32_t metaLen = 0;
  metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
-  metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);

+  int32_t trueLen = pBlock->tag.nLen;
  if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
    metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
-    metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f);
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
+  } else {
+    trueLen = 0;
+    metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
  }

  fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
  fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
-  
+
+  metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+  assert(metaLen == getTagAreaLength(&pBlock->tag));
+
  int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
  pTSBuf->fileSize += blockSize;
  
@ -284,23 +302,28 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) {

 STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
  STSBlock* pBlock = &pTSBuf->block;
-  
+
  // clear the memory buffer
-  void* tmp = pBlock->payload;
-  memset(pBlock, 0, sizeof(STSBlock));
-  pBlock->payload = tmp;
-  
+  pBlock->compLen   = 0;
+  pBlock->padding   = 0;
+  pBlock->numOfElem = 0;
+
+  int32_t offset = -1;
+
  if (order == TSDB_ORDER_DESC) {
    /*
     * set the right position for the reversed traverse, the reversed traverse is started from
     * the end of each comp data block
     */
-    int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR);
-    size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+    int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
+    int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
+    size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
+    sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
    UNUSED(sz); 
-    
+
    pBlock->compLen = pBlock->padding;
-    int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+
+    offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
    ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
    UNUSED(ret);
  }
@ -319,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {

    sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
-    sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f);
+    sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
  }

  sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
@ -327,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
  sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
  UNUSED(sz);
  sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
-  UNUSED(sz);
-  
+
  if (decomp) {
    pTSBuf->tsData.len =
        tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
@ -337,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
  
  // read the comp length at the length of comp block
  sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+  assert(pBlock->padding == pBlock->compLen);
+
+  int32_t n = 0;
+  sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+  if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
+    assert(n == 0);
+  } else {
+    assert(n == pBlock->tag.nLen);
+  }
+
  UNUSED(sz);
  
  // for backwards traverse, set the start position at the end of previous block
  if (order == TSDB_ORDER_DESC) {
-    int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
    int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
    UNUSED(r);
  }
@ -446,7 +477,7 @@ void tsBufFlush(STSBuf* pTSBuf) {
  fsync(fileno(pTSBuf->f));
 }

-static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) {
+static int32_t tsBufFindVnodeById(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) {
  int32_t j = -1;
  for (int32_t i = 0; i < numOfVnodes; ++i) {
    if (pVnodeInfoEx[i].info.vnode == vnodeId) {
@ -478,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
  if (pTSBuf->cur.order == TSDB_ORDER_DESC) {
    STSBlock* pBlock = &pTSBuf->block;
    int32_t   compBlockSize =
-        pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+        pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
    int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
    UNUSED(ret);
  }
@ -506,7 +537,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo
    }
    
    if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) {
-      return i;
+      return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1));
    }
  }
  
@ -575,7 +606,7 @@ static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockIn
 }

 STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) {
-  int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+  int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
  if (j == -1) {
    return NULL;
  }
@ -772,7 +803,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
  int64_t offset = getDataStartOffset();
  int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;

-  ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
+  int64_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
  
  if (rc == -1) {
 //    tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno));
@ -839,7 +870,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) {
    return elem;
  }
  
-  int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+  int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
  if (j == -1) {
    return elem;
  }
@ -992,4 +1023,47 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) {
  for(int32_t i = 0; i < size; ++i) {
    (*vnodeId)[i] = pTSBuf->pData[i].info.vnode;
  }
-}
+}
+
+int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeIndex, void* buf, int32_t* len, int32_t* numOfBlocks) {
+  assert(vnodeIndex >= 0 && vnodeIndex < pTSBuf->numOfVnodes);
+  STSVnodeBlockInfo *pBlockInfo = &pTSBuf->pData[vnodeIndex].info;
+
+  *len = 0;
+  *numOfBlocks = 0;
+
+  if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+//    qError("%p: fseek failed: %s", pSql, tstrerror(code));
+    return code;
+  }
+
+  size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+  if (s != pBlockInfo->compLen) {
+    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+//    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+    return code;
+  }
+
+  *len = pBlockInfo->compLen;
+  *numOfBlocks = pBlockInfo->numOfBlocks;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) {
+  STSElem el = {.vnode = -1};
+
+  for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) {
+    el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.vnode, pTag);
+    if (el.vnode == pTSBuf->pData[i].info.vnode) {
+      return el;
+    }
+  }
+
+  return el;
+}
+
+bool tsBufIsValidElem(STSElem* pElem) {
+  return pElem->vnode >= 0;
+}
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@ -14,8 +14,9 @@
 */

 #include "os.h"
-#include "hash.h"
 #include "taosmsg.h"
+#include "hash.h"
+
 #include "qExecutor.h"
 #include "qUtil.h"

@ -26,74 +27,36 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
    size += pQuery->pSelectExpr[i].interBytes;
  }

-  assert(size > 0);
+  assert(size >= 0);
  return size;
 }

-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
-                          int32_t threshold, int16_t type) {
+int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
  pWindowResInfo->capacity = size;
  pWindowResInfo->threshold = threshold;
  
  pWindowResInfo->type = type;
-  _hash_fn_t fn = taosGetDefaultHashFunction(type);
-  pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false);
-  if (pWindowResInfo->hashList == NULL) {
-    return TSDB_CODE_QRY_OUT_OF_MEMORY;
-  }
-  
  pWindowResInfo->curIndex = -1;
  pWindowResInfo->size     = 0;
  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;

-  SQueryCostInfo* pSummary = &pRuntimeEnv->summary;
-
-  // use the pointer arraylist
-  pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
+  pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
  if (pWindowResInfo->pResult == NULL) {
    return TSDB_CODE_QRY_OUT_OF_MEMORY;
  }

-  pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval;
-
-  pSummary->internalSupSize += sizeof(SWindowResult) * threshold;
-  pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity;
-  pSummary->numOfTimeWindows = threshold;
-
-  for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
-    int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-  }
-  
  return TSDB_CODE_SUCCESS;
 }

-void destroyTimeWindowRes(SWindowResult *pWindowRes) {
-  if (pWindowRes == NULL) {
-    return;
-  }
-
-  free(pWindowRes->resultInfo);
-}
-
 void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
  if (pWindowResInfo == NULL) {
    return;
  }
  if (pWindowResInfo->capacity == 0) {
-    assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL);
+    assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL);
    return;
  }
  
-  if (pWindowResInfo->pResult != NULL) {
-    for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
-      destroyTimeWindowRes(&pWindowResInfo->pResult[i]);
-    }
-  }
-  
-  taosHashCleanup(pWindowResInfo->hashList);
  taosTFree(pWindowResInfo->pResult);
 }

@ -103,17 +66,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
  }
  
  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-    SWindowResult *pWindowRes = &pWindowResInfo->pResult[i];
-    clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+    SResultRow *pWindowRes = pWindowResInfo->pResult[i];
+    clearResultRow(pRuntimeEnv, pWindowRes);
  }
  
  pWindowResInfo->curIndex = -1;
-  taosHashCleanup(pWindowResInfo->hashList);
  pWindowResInfo->size = 0;
  
-  _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type);
-  pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false);
-  
  pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
@ -127,13 +86,13 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
  int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
  assert(num >= 0 && num <= numOfClosed);

-  int16_t type = pWindowResInfo->type;
-
-  char *key = NULL;
-  int16_t bytes = -1;
+  int16_t  type = pWindowResInfo->type;
+  STableId* id  = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
+  char    *key  = NULL;
+  int16_t  bytes = -1;

  for (int32_t i = 0; i < num; ++i) {
-    SWindowResult *pResult = &pWindowResInfo->pResult[i];
+    SResultRow *pResult = pWindowResInfo->pResult[i];
    if (pResult->closed) {  // remove the window slot from hash table

      // todo refactor
@ -145,7 +104,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
        bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
      }

-      taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes);
+      SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+      taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
    } else {
      break;
    }
@ -155,19 +115,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
  
  // clear all the closed windows from the window list
  for (int32_t k = 0; k < remain; ++k) {
-    copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]);
+    copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]);
  }
  
  // move the unclosed window in the front of the window list
  for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
-    SWindowResult *pWindowRes = &pWindowResInfo->pResult[k];
-    clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+    SResultRow *pWindowRes = pWindowResInfo->pResult[k];
+    clearResultRow(pRuntimeEnv, pWindowRes);
  }
  
  pWindowResInfo->size = remain;

  for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
-    SWindowResult *pResult = &pWindowResInfo->pResult[k];
+    SResultRow *pResult = pWindowResInfo->pResult[k];

    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
      key = varDataVal(pResult->key);
@ -177,12 +137,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
      bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
    }

-    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes);
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+    int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
    assert(p != NULL); 

    int32_t  v = (*p - num);
    assert(v >= 0 && v <= pWindowResInfo->size);
-    taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t));
+
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+    taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
  }
  
  pWindowResInfo->curIndex = -1;
@ -200,7 +163,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {

 int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
  int32_t i = 0;
-  while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) {
+  while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
    ++i;
  }
  
@ -211,11 +174,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
  assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
  
  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-    if (pWindowResInfo->pResult[i].closed) {
+    if (pWindowResInfo->pResult[i]->closed) {
      continue;
    }
    
-    pWindowResInfo->pResult[i].closed = true;
+    pWindowResInfo->pResult[i]->closed = true;
  }
 }

@ -231,19 +194,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
  }

  // get the result order
-  int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1;
+  int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
  if (order != resultOrder) {
    return;
  }

  int32_t i = 0;
  if (order == QUERY_ASC_FORWARD_STEP) {
-    TSKEY ekey = pWindowResInfo->pResult[i].win.ekey;
+    TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
    while (i < pWindowResInfo->size && (ekey < lastKey)) {
      ++i;
    }
  } else if (order == QUERY_DESC_FORWARD_STEP) {
-    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) {
+    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
      ++i;
    }
  }
@ -254,30 +217,33 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
 }

 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
-  return (getWindowResult(pWindowResInfo, slot)->closed == true);
+  return (getResultRow(pWindowResInfo, slot)->closed == true);
 }

 void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
-  getWindowResult(pWindowResInfo, slot)->closed = true;
+  getResultRow(pWindowResInfo, slot)->closed = true;
 }

-void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) {
+void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
  if (pWindowRes == NULL) {
    return;
  }

-  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
+  // the result does not put into the SDiskbasedResultBuf, ignore it.
+  if (pWindowRes->pageId >= 0) {
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);

-  for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
-    SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
-    
-    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
-    size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
-    memset(s, 0, size);
-    
-    RESET_RESULT_INFO(pResultInfo);
+    for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
+      SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i];
+
+      char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
+      size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
+      memset(s, 0, size);
+
+      RESET_RESULT_INFO(pResultInfo);
+    }
  }
-  
+
  pWindowRes->numOfRows = 0;
  pWindowRes->pageId = -1;
  pWindowRes->rowId = -1;
@ -290,7 +256,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
 * since the attribute of "Pos" is bound to each window result when the window result is created in the
 * disk-based result buffer.
 */
-void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
+void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) {
  dst->numOfRows = src->numOfRows;
  dst->win   = src->win;
  dst->closed = src->closed;
@ -298,25 +264,105 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
  int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;
  
  for (int32_t i = 0; i < nOutputCols; ++i) {
-    SResultInfo *pDst = &dst->resultInfo[i];
-    SResultInfo *pSrc = &src->resultInfo[i];
+    SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i);
+    SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i);
    
-    char *buf = pDst->interResultBuf;
-    memcpy(pDst, pSrc, sizeof(SResultInfo));
-    pDst->interResultBuf = buf;  // restore the allocated buffer
+//    char *buf = pDst->interResultBuf;
+    memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes);
+//    pDst->interResultBuf = buf;  // restore the allocated buffer
    
    // copy the result info struct
-    memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
+//    memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes);
    
    // copy the output buffer data from src to dst, the position info keep unchanged
    tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId);
    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);

    tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId);
-    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
+    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage);
    size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
    
    memcpy(dstBuf, srcBuf, s);
  }
 }

+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) {
+  assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput);
+  return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]);
+}
+
+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) {
+  return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow);
+}
+
+SResultRowPool* initResultRowPool(size_t size) {
+  SResultRowPool* p = calloc(1, sizeof(SResultRowPool));
+  if (p == NULL) {
+    return NULL;
+  }
+
+  p->numOfElemPerBlock = 128;
+
+  p->elemSize = (int32_t) size;
+  p->blockSize = p->numOfElemPerBlock * p->elemSize;
+  p->position.pos = 0;
+
+  p->pData = taosArrayInit(8, POINTER_BYTES);
+  return p;
+}
+
+SResultRow* getNewResultRow(SResultRowPool* p) {
+  if (p == NULL) {
+    return NULL;
+  }
+
+  void* ptr = NULL;
+  if (p->position.pos == 0) {
+    ptr = calloc(1, p->blockSize);
+    taosArrayPush(p->pData, &ptr);
+
+  } else {
+    size_t last = taosArrayGetSize(p->pData);
+
+    void** pBlock = taosArrayGet(p->pData, last - 1);
+    ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos;
+  }
+
+  p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
+  initResultRow(ptr);
+
+  return ptr;
+}
+
+int64_t getResultRowPoolMemSize(SResultRowPool* p) {
+  if (p == NULL) {
+    return 0;
+  }
+
+  return taosArrayGetSize(p->pData) * p->blockSize;
+}
+
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p) {
+  return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock;
+}
+
+int32_t getNumOfUsedResultRows(SResultRowPool* p) {
+  return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos;
+}
+
+void* destroyResultRowPool(SResultRowPool* p) {
+  if (p == NULL) {
+    return NULL;
+  }
+
+  size_t size = taosArrayGetSize(p->pData);
+  for(int32_t i = 0; i < size; ++i) {
+    void** ptr = taosArrayGet(p->pData, i);
+    taosTFree(*ptr);
+  }
+
+  taosArrayDestroy(p->pData);
+
+  taosTFree(p);
+  return NULL;
+}
--- a/src/query/tests/tsBufTest.cpp
+++ b/src/query/tests/tsBufTest.cpp
@ -472,13 +472,20 @@ void mergeIdenticalVnodeBufferTest() {
  tsBufFlush(pTSBuf2);

  tsBufMerge(pTSBuf1, pTSBuf2);
-  EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
+  EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
  EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);

  tsBufResetPos(pTSBuf1);
+
+  int32_t count = 0;
  while (tsBufNextPos(pTSBuf1)) {
    STSElem elem = tsBufGetElem(pTSBuf1);
-    EXPECT_EQ(elem.vnode, 12);
+
+    if (count++ < numOfTags * num) {
+      EXPECT_EQ(elem.vnode, 12);
+    } else {
+      EXPECT_EQ(elem.vnode, 77);
+    }

    printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts);
  }
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@ -341,7 +341,7 @@ void *rpcMallocCont(int contLen) {
    tError("failed to malloc msg, size:%d", size);
    return NULL;
  } else {
-    tTrace("malloc mem: %p", start);
+    tTrace("malloc mem:%p size:%d", start, size);
  }

  return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
@ -557,10 +557,7 @@ void rpcCancelRequest(void *handle) {
  int code = taosAcquireRef(tsRpcRefId, pContext);
  if (code < 0) return;

-  if (pContext->pConn) {
-    tDebug("%s, app tries to cancel request", pContext->pConn->info);
-    rpcCloseConn(pContext->pConn);
-  }
+  rpcCloseConn(pContext->pConn);

  taosReleaseRef(tsRpcRefId, pContext);
 }
@ -655,6 +652,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {

 static void rpcCloseConn(void *thandle) {
  SRpcConn *pConn = (SRpcConn *)thandle;
+  if (pConn == NULL) return;

  rpcLockConn(pConn);

@ -1026,6 +1024,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
  if (pConn->outType) {
    SRpcReqContext *pContext = pConn->pContext;
    pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+    pContext->pConn = NULL;
    pConn->pReqMsg = NULL;
    taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
  }
@ -1076,6 +1075,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
        tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
      } 
    } else { // msg is passed to app only parsing is ok 
+
+      if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
+        rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS);
+        rpcFreeMsg(pRecv->msg); 
+        return pConn;
+      }
+      
      rpcProcessIncomingMsg(pConn, pHead, pContext);
    }
  }
@ -1128,6 +1134,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
    // it's a response
    rpcMsg.handle = pContext;
    rpcMsg.ahandle = pContext->ahandle;
+    pContext->pConn = NULL;

    // for UDP, port may be changed by server, the port in epSet shall be used for cache
    if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
@ -1363,6 +1370,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
      tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
      if (pConn->pContext) {
        pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+        pConn->pContext->pConn = NULL;
        pConn->pReqMsg = NULL;
        taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl);
        rpcReleaseConn(pConn);
@ -1471,7 +1479,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
      pNewHead->msgLen = rpcMsgLenFromCont(origLen);
      rpcFreeMsg(pHead); // free the compressed message buffer
      pHead = pNewHead; 
-      tTrace("decomp malloc mem: %p", temp);
+      tTrace("decomp malloc mem:%p", temp);
    } else {
      tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
    }
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@ -437,12 +437,13 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
  }

  msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
-  buffer = malloc(msgLen + tsRpcOverhead);
+  int32_t size = msgLen + tsRpcOverhead;
+  buffer = malloc(size);
  if (NULL == buffer) {
    tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
    return -1;
  } else {
-    tTrace("TCP malloc mem: %p", buffer);
+    tTrace("TCP malloc mem:%p size:%d", buffer, size);
  }

  msg = buffer + tsRpcOverhead;
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@ -209,12 +209,13 @@ static void *taosRecvUdpData(void *param) {
      continue;
    }

-    char *tmsg = malloc(dataLen + tsRpcOverhead);
+    int32_t size = dataLen + tsRpcOverhead;
+    char *tmsg = malloc(size);
    if (NULL == tmsg) {
      tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen);
      continue;
    } else {
-      tTrace("UDP malloc mem: %p", tmsg);
+      tTrace("UDP malloc mem:%p size:%d", tmsg, size);
    }

    tmsg += tsRpcOverhead;  // overhead for SRpcReqContext
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@ -20,6 +20,7 @@
 #include "tlog.h"
 #include "tutil.h"
 #include "ttimer.h"
+#include "tref.h"
 #include "tsocket.h"
 #include "tglobal.h"
 #include "taoserror.h"
@ -43,6 +44,7 @@ char tsNodeFqdn[TSDB_FQDN_LEN];
 static ttpool_h tsTcpPool;
 static void *   syncTmrCtrl = NULL;
 static void *   vgIdHash;
+static int      tsSyncRefId = -1;

 // local functions
 static void  syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
@ -54,13 +56,13 @@ static int   syncProcessPeerMsg(void *param, void *buffer);
 static void  syncProcessIncommingConnection(int connFd, uint32_t sourceIp); 
 static void  syncRemovePeer(SSyncPeer *pPeer);
 static void  syncAddArbitrator(SSyncNode *pNode);
-static void  syncAddNodeRef(SSyncNode *pNode);
-static void  syncDecNodeRef(SSyncNode *pNode);
+static void  syncFreeNode(void *);
 static void  syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
 static void  syncMonitorFwdInfos(void *param, void *tmrId);
 static void  syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
 static void  syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); 
 static void  syncRestartPeer(SSyncPeer *pPeer);
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtyp);
 static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);

 char* syncRole[] = {
@ -106,6 +108,12 @@ int32_t syncInit() {
    return -1;
  }

+  tsSyncRefId = taosOpenRef(200, syncFreeNode);
+  if (tsSyncRefId < 0) {
+    syncCleanUp();
+    return -1;
+  }
+
  tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn));
  sInfo("sync module initialized successfully");

@ -128,6 +136,9 @@ void syncCleanUp() {
    vgIdHash = NULL;
  }

+  taosCloseRef(tsSyncRefId);
+  tsSyncRefId = -1;
+
  sInfo("sync module is cleaned up");
 }

@ -159,6 +170,12 @@ void *syncStart(const SSyncInfo *pInfo) {
  pNode->quorum = pCfg->quorum;
  if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;

+  int ret = taosAddRef(tsSyncRefId, pNode);
+  if (ret < 0) {
+    syncFreeNode(pNode);
+    return NULL;
+  }
+
  for (int i = 0; i < pCfg->replica; ++i) {
    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
    pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
@ -167,8 +184,6 @@ void *syncStart(const SSyncInfo *pInfo) {
    }
  }

-  syncAddNodeRef(pNode);
-
  if (pNode->selfIndex < 0) {
    sInfo("vgId:%d, this node is not configured", pNode->vgId);
    terrno = TSDB_CODE_SYN_INVALID_CONFIG;
@ -210,7 +225,9 @@ void syncStop(void *param) {
  SSyncNode *pNode = param;
  SSyncPeer *pPeer;

-  if (pNode == NULL) return;
+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return;
+
  sInfo("vgId:%d, cleanup sync", pNode->vgId);

  pthread_mutex_lock(&(pNode->mutex));
@ -228,14 +245,17 @@ void syncStop(void *param) {

  pthread_mutex_unlock(&(pNode->mutex));

-  syncDecNodeRef(pNode);
+  taosReleaseRef(tsSyncRefId, pNode);
+  taosRemoveRef(tsSyncRefId, pNode);
 }

 int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
  SSyncNode *pNode = param;
  int        i, j;

-  if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG;
+
  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
        pNode->replica);

@ -298,105 +318,63 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
        syncRole[nodeRole]);
  syncBroadcastStatus(pNode);

+  taosReleaseRef(tsSyncRefId, pNode);
+
  return 0;
 }

 int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
  SSyncNode *pNode = param;
-  SSyncPeer *pPeer;
-  SSyncHead *pSyncHead;
-  SWalHead * pWalHead = data;
-  int        fwdLen;
-  int        code = 0;

-  if (pNode == NULL) return 0;
+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return 0; 

-  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
-    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
-           pWalHead->version, nodeVersion);
-    for (int i = 0; i < pNode->replica; ++i) {
-      pPeer = pNode->peerInfo[i];
-      syncRestartConnection(pPeer);
-    }
-    return TSDB_CODE_SYN_INVALID_VERSION;
-  }
+  int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);

-  // always update version
-  nodeVersion = pWalHead->version;
-  sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
-         qtype, pWalHead->version);
-
-  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
-
-  // only pkt from RPC or CQ can be forwarded
-  if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
-
-  // a hacker way to improve the performance
-  pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
-  pSyncHead->type = TAOS_SMSG_FORWARD;
-  pSyncHead->pversion = 0;
-  pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
-  fwdLen = pSyncHead->len + sizeof(SSyncHead);  // include the WAL and SYNC head
-
-  pthread_mutex_lock(&(pNode->mutex));
-
-  for (int i = 0; i < pNode->replica; ++i) {
-    pPeer = pNode->peerInfo[i];
-    if (pPeer == NULL || pPeer->peerFd < 0) continue;
-    if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
-
-    if (pNode->quorum > 1 && code == 0) {
-      syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
-      code = 1;
-    }
-
-    int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
-    if (retLen == fwdLen) {
-      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
-    } else {
-      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
-      syncRestartConnection(pPeer);
-    }
-  }
-
-  pthread_mutex_unlock(&(pNode->mutex));
+  taosReleaseRef(tsSyncRefId, pNode);

  return code;
 }

 void syncConfirmForward(void *param, uint64_t version, int32_t code) {
  SSyncNode *pNode = param;
-  if (pNode == NULL) return;
-  if (pNode->quorum <= 1) return;
+
+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return;

  SSyncPeer *pPeer = pNode->pMaster;
-  if (pPeer == NULL) return;
+  if (pPeer && pNode->quorum > 1) {
+    char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};

-  char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+    SSyncHead *pHead = (SSyncHead *)msg;
+    pHead->type = TAOS_SMSG_FORWARD_RSP;
+    pHead->len = sizeof(SFwdRsp);

-  SSyncHead *pHead = (SSyncHead *)msg;
-  pHead->type = TAOS_SMSG_FORWARD_RSP;
-  pHead->len = sizeof(SFwdRsp);
+    SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+    pFwdRsp->version = version;
+    pFwdRsp->code = code;

-  SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
-  pFwdRsp->version = version;
-  pFwdRsp->code = code;
+    int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+    int retLen = write(pPeer->peerFd, msg, msgLen);

-  int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
-  int retLen = write(pPeer->peerFd, msg, msgLen);
-
-  if (retLen == msgLen) {
-    sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
-  } else {
-    sDebug("%s, failed to send forward ack, restart", pPeer->id);
-    syncRestartConnection(pPeer);
+    if (retLen == msgLen) {
+      sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
+    } else {
+      sDebug("%s, failed to send forward ack, restart", pPeer->id);
+      syncRestartConnection(pPeer);
+    }
  }
+
+  taosReleaseRef(tsSyncRefId, pNode);
 }

 void syncRecover(void *param) {
  SSyncNode *pNode = param;
  SSyncPeer *pPeer;

+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return;
+
  // to do: add a few lines to check if recover is OK
  // if take this node to unsync state, the whole system may not work

@ -414,17 +392,24 @@ void syncRecover(void *param) {
  }

  pthread_mutex_unlock(&(pNode->mutex));
+
+  taosReleaseRef(tsSyncRefId, pNode);
 }

 int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
  SSyncNode *pNode = param;

+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if (ret < 0) return -1;
+
  pNodesRole->selfIndex = pNode->selfIndex;
  for (int i = 0; i < pNode->replica; ++i) {
    pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
    pNodesRole->role[i] = pNode->peerInfo[i]->role;
  }

+  taosReleaseRef(tsSyncRefId, pNode);
+
  return 0;
 }

@ -457,22 +442,20 @@ static void syncAddArbitrator(SSyncNode *pNode) {
  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
 }

-static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); }
+static void syncFreeNode(void *param) {
+  SSyncNode *pNode = param;

-static void syncDecNodeRef(SSyncNode *pNode) {
-  if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
-    pthread_mutex_destroy(&pNode->mutex);
-    taosTFree(pNode->pRecv);
-    taosTFree(pNode->pSyncFwds);
-    taosTFree(pNode);
-  }
+  pthread_mutex_destroy(&pNode->mutex);
+  taosTFree(pNode->pRecv);
+  taosTFree(pNode->pSyncFwds);
+  taosTFree(pNode);
 }

 void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }

 int syncDecPeerRef(SSyncPeer *pPeer) {
  if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
-    syncDecNodeRef(pPeer->pSyncNode);
+    taosReleaseRef(tsSyncRefId, pPeer->pSyncNode);

    sDebug("%s, resource is freed", pPeer->id);
    taosTFree(pPeer->watchFd);
@ -529,7 +512,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
    taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
  }

-  syncAddNodeRef(pNode);
+  taosAcquireRef(tsSyncRefId, pNode);  
  return pPeer;
 }

@ -1122,7 +1105,7 @@ static void syncProcessBrokenLink(void *param) {
  SSyncPeer *pPeer = param;
  SSyncNode *pNode = pPeer->pSyncNode;

-  syncAddNodeRef(pNode);
+  if (taosAcquireRef(tsSyncRefId, pNode) < 0) return;
  pthread_mutex_lock(&(pNode->mutex));

  sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@ -1133,7 +1116,7 @@ static void syncProcessBrokenLink(void *param) {
  }

  pthread_mutex_unlock(&(pNode->mutex));
-  syncDecNodeRef(pNode);
+  taosReleaseRef(tsSyncRefId, pNode);
 }

 static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@ -1202,22 +1185,90 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code

 static void syncMonitorFwdInfos(void *param, void *tmrId) {
  SSyncNode *pNode = param;
+
+  int ret = taosAcquireRef(tsSyncRefId, pNode);
+  if ( ret < 0) return;
+
  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
-  if (pSyncFwds == NULL) return;

-  uint64_t   time = taosGetTimestampMs();
+  if (pSyncFwds) {;
+    uint64_t   time = taosGetTimestampMs();

-  if (pSyncFwds->fwds > 0) {
-    pthread_mutex_lock(&(pNode->mutex));
-    for (int i = 0; i < pSyncFwds->fwds; ++i) {
-      SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
-      if (time - pFwdInfo->time < 2000) break;
-      syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+    if (pSyncFwds->fwds > 0) {
+      pthread_mutex_lock(&(pNode->mutex));
+      for (int i = 0; i < pSyncFwds->fwds; ++i) {
+        SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
+        if (time - pFwdInfo->time < 2000) break;
+        syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+      }
+
+      syncRemoveConfirmedFwdInfo(pNode);
+      pthread_mutex_unlock(&(pNode->mutex));
    }

-    syncRemoveConfirmedFwdInfo(pNode);
-    pthread_mutex_unlock(&(pNode->mutex));
+    pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
  }

-  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+  taosReleaseRef(tsSyncRefId, pNode);
 }
+
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) {
+  SSyncPeer *pPeer;
+  SSyncHead *pSyncHead;
+  SWalHead * pWalHead = data;
+  int        fwdLen;
+  int32_t    code = 0;
+
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
+    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
+           pWalHead->version, nodeVersion);
+    for (int i = 0; i < pNode->replica; ++i) {
+      pPeer = pNode->peerInfo[i];
+      syncRestartConnection(pPeer);
+    }
+    return TSDB_CODE_SYN_INVALID_VERSION;
+  }
+
+  // always update version
+  nodeVersion = pWalHead->version;
+  sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
+         qtype, pWalHead->version);
+
+  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
+
+  // only pkt from RPC or CQ can be forwarded
+  if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+  // a hacker way to improve the performance
+  pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
+  pSyncHead->type = TAOS_SMSG_FORWARD;
+  pSyncHead->pversion = 0;
+  pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
+  fwdLen = pSyncHead->len + sizeof(SSyncHead);  // include the WAL and SYNC head
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer == NULL || pPeer->peerFd < 0) continue;
+    if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
+
+    if (pNode->quorum > 1 && code == 0) {
+      syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+      code = 1;
+    }
+
+    int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+    if (retLen == fwdLen) {
+      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+    } else {
+      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
--- a/src/sync/src/syncRestore.c
+++ b/src/sync/src/syncRestore.c
@ -56,6 +56,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
  int        code = -1;
  char       name[TSDB_FILENAME_LEN * 2] = {0};
  uint32_t   pindex = 0;    // index in last restore
+  bool       fileChanged = false;

  *fversion = 0;
  sinfo.index = 0;
@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
    close(dfd);
    if (ret < 0) break;

+    fileChanged = true;
    sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
  }

-  if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+  if (code == 0 && fileChanged) {
    // data file is changed, code shall be set to 1
    *fversion = minfo.fversion;
    code = 1;
--- a/Show More
+++ b/Show More