Merge branch 'develop' into namedtype

.lgtm.yml

@@ -0,0 +1,402 @@
+##########################################################################################
+# Customize file classifications.                                                        #
+# Results from files under any classifier will be excluded from LGTM                     #
+# statistics.                                                                            #
+##########################################################################################
+
+##########################################################################################
+# Use the `path_classifiers` block to define changes to the default classification of    #
+# files.                                                                                 #
+##########################################################################################
+
+path_classifiers:
+        # docs:
+    # Identify the top-level file called `generate_javadoc.py` as documentation-related.
+  test:
+    # Override LGTM's default classification of test files by excluding all files.
+    - exclude: /
+    # Classify all files in the top-level directories tests/ and testsuites/ as test code.
+    - tests
+      # - testsuites
+    # Classify all files with suffix `.test` as test code.
+    #        Note: use only forward slash / as a path separator.
+    #        Use ** to indicate an arbitrary parent path.
+    #        Use * to indicate any sequence of characters excluding /.
+    #        Always enclose the expression in double quotes if it includes *.
+    # - "**/*.test"
+    # Refine the classifications above by excluding files in test/util/.
+    # - exclude: test/util
+  # The default behavior is to tag all files created during the
+  # build as `generated`. Results are hidden for generated code. You can tag 
+  # further files as being generated by adding them to the `generated` section.
+  generated:
+    # Exclude all `*.c` files under the `ui/` directory from classification as 
+    # generated code.
+    # - exclude: ui/**/*.c
+    # By default, all files not checked into the repository are considered to be
+    # 'generated'.
+  # The default behavior is to tag library code as `library`. Results are hidden 
+  # for library code. You can tag further files as being library code by adding them
+  # to the `library` section.
+  library:
+    - exclude: deps/
+  # The default behavior is to tag template files as `template`. Results are hidden 
+  # for template files. You can tag further files as being template files by adding 
+  # them to the `template` section.
+  template:
+          #- exclude: path/to/template/code/**/*.c
+  # Define your own category, for example: 'some_custom_category'.
+  some_custom_category:
+    # Classify all files in the top-level directory tools/ (or the top-level file 
+    # called tools).
+    # - tools
+
+#########################################################################################
+# Use the `queries` block to change the default display of query results.               #
+#########################################################################################
+
+          # queries:
+  # Start by hiding the results of all queries.
+  #  - exclude: "*"
+  # Then include all queries tagged 'security' and 'correctness', and with a severity of
+  # 'error'.
+  #  - include:           
+  #      tags:
+  #        - "security"
+  #        - "correctness"           
+  #    severity: "error"
+  # Specifically hide the results of two queries.
+  # - exclude: cpp/use-of-goto
+    # - exclude: java/equals-on-unrelated-types
+  # Refine by including the `java/command-line-injection` query.
+  # - include: java/command-line-injection
+
+#########################################################################################
+# Define changes to the default code extraction process.                                #
+# Each block configures the extraction of a single language, and modifies actions in a  #
+# named step. Every named step includes automatic default actions,                      #
+# except for the 'prepare' step. The steps are performed in the following sequence:     #
+# prepare                                                                               #
+# after_prepare                                                                         #
+# configure (C/C++ only)                                                                #
+# python_setup (Python only)                                                            #
+# before_index                                                                          #
+# index                                                                                 #
+##########################################################################################
+
+#########################################################################################
+# Environment variables available to the steps:                                         #
+#########################################################################################
+
+# LGTM_SRC
+#   The root of the source tree.
+# LGTM_WORKSPACE
+#   An existing (initially empty) folder outside the source tree.
+#   Used for temporary download and setup commands.
+
+#########################################################################################
+# Use the extraction block to define changes to the default code extraction process     #
+# for one or more languages. The settings for each language are defined in a child      #
+# block, with one or more steps.                                                        #
+#########################################################################################
+
+extraction: 
+  # Define settings for C/C++ analysis
+  #####################################
+  cpp:
+    # The `prepare` step exists for customization on LGTM.com only.
+    prepare:
+    #   # The `packages` section is valid for LGTM.com only. It names Ubuntu packages to
+    #   # be installed.
+        packages:
+          - cmake
+    # Add an `after-prepare` step if you need to run commands after the prepare step.
+    # Each command should be listed on a separate line.
+    # This step is useful for C/C++ analysis where you want to prepare the environment
+    # for the `configure` step without changing the default behavior for that step. 
+    #    after_prepare:
+            #- export GNU_MAKE=make
+            #- export GIT=true
+    # The `configure` step generates build configuration files which the `index` step
+    # then uses to build the codebase.
+    configure:
+      command:
+        - mkdir build
+        - cd build
+        - cmake ..
+              # - ./prepare_deps
+    # Optional step. You should add a `before_index` step if you need to run commands
+    # before the `index` step.
+    # before_index:
+            # - export BOOST_DIR=$LGTM_SRC/boost
+            # - export GTEST_DIR=$LGTM_SRC/googletest
+            # - export HUNSPELL_DIR=$LGTM_SRC/hunspell
+            # - export CRYPTOPP_DIR=$LGTM_SRC/cryptopp
+    # The `index` step builds the code and extracts information during the build
+    # process.
+    index:
+      # Override the autobuild process by specifying a list of custom build commands
+      # to use instead. 
+      build_command:
+        - cd build
+        - make
+              # - $GNU_MAKE -j2 -s
+      # Specify that all project or solution files should be used for extraction.
+      # Default: false.
+      # all_solutions: true
+      # Specify a list of one or more project or solution files for extraction.
+      # Default: LGTM chooses the file closest to the root of the repository (this may
+      # fail if there are multiple candidates).
+      # solution: 
+      #  - myProject.sln
+      # Specify MSBuild settings
+      # msbuild:
+        # Specify a list of additional arguments to MSBuild. Default: empty.
+        # arguments: /p:Platform=x64 /p:Configuration=Release
+        # Specify the MSBuild configuration to use, for example, debug or release.
+        # Default: read from the solution file or files.
+        # configuration:
+        # Specify the platform to target, for example: x86, x64, or Any CPU.
+        # Default: read from the solution file or files.
+        # platform:
+        # Specify the MSBuild target. Default: rebuild.
+        # target:
+      # Specify whether or not to perform a NuGet restore for extraction. Default: true.
+      # nuget_restore: false
+      # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom
+      # build commands (build_command). For example:
+      #    10 for Visual Studio 2010
+      #    12 for Visual Studio 2012
+      #    14 for Visual Studio 2015
+      #    15 for Visual Studio 2017
+      # Default: read from project files.
+      # vstools_version: 10
+
+  # Define settings for C# analysis
+  ##################################
+  #  csharp:
+    # The `prepare` step exists for customization on LGTM.com only.
+    # prepare:
+    #   packages:
+    #     - example_package
+    # Add an `after-prepare` step if you need to run commands after the `prepare` step.
+    # Each command should be listed on a separate line.
+    #  after_prepare:
+    #  - export PATH=$LGTM_WORKSPACE/tools:$PATH
+    # The `index` step builds the code and extracts information during the build
+    # process.
+    #index:
+      # Specify that all project or solution files should be used for extraction.
+      # Default: false.
+      #  all_solutions: true
+      # Specify a list of one or more project or solution files for extraction.
+      # Default: LGTM chooses the file closest to the root of the repository (this may
+      # fail if there are multiple candidates).
+      # solution: 
+      #  - myProject.sln
+      # Override the autobuild process by specifying a list of custom build commands
+      # to use instead. 
+      # build_command:
+      #  - ./example-compile-all.sh
+      # By default, LGTM analyzes the code by building it. You can override this,
+      # and tell LGTM not to build the code. Beware that this can lead
+      # to less accurate results.
+      # buildless: true
+      # Specify .NET Core settings.
+      # dotnet:
+        # Specify additional arguments to `dotnet build`.
+        # Default: empty.
+        # arguments: "example_arg"
+        # Specify the version of .NET Core SDK to use.
+        # Default: The version installed on the build machine.
+        # version: 2.1
+      # Specify MSBuild settings.
+      # msbuild:
+        # Specify a list of additional arguments to MSBuild. Default: empty.
+        # arguments: /P:WarningLevel=2
+        # Specify the MSBuild configuration to use, for example, debug or release.
+        # Default: read from the solution file or files.
+        # configuration: release
+        # Specify the platform to target, for example: x86, x64, or Any CPU.
+        # Default: read from the solution file or files.
+        # platform: x86
+        # Specify the MSBuild target. Default: rebuild.
+        # target: notest
+      # Specify whether or not to perform a NuGet restore for extraction. Default: true.
+      # nuget_restore: false
+      # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom
+      # build commands (build_command). For example:
+      #    10 for Visual Studio 2010
+      #    12 for Visual Studio 2012
+      #    14 for Visual Studio 2015
+      #    15 for Visual Studio 2017
+      # Default: read from project files
+      # vstools_version: 10
+      # Specify additional options for the extractor,
+      # for example --fast to perform a faster extraction that produces a smaller 
+      # database.
+      # extractor: "--fast"
+
+  # Define settings for Go analysis
+  ##################################
+  #  go:
+    # The `prepare` step exists for customization on LGTM.com only.
+    # prepare:
+    #   packages:
+    #     - example_package
+    # Add an `after-prepare` step if you need to run commands after the `prepare` step.
+    # Each command should be listed on a separate line.
+    # after_prepare:
+    #  - export PATH=$LGTM_WORKSPACE/tools:$PATH
+    # The `index` step builds the code and extracts information during the build
+    # process.
+    # index:
+      # Override the autobuild process by specifying a list of custom build commands
+      # to use instead.
+      # build_command:
+              #  - ./compile-all.sh
+
+  # Define settings for Java analysis
+  ####################################
+  #  java:
+    # The `prepare` step exists for customization on LGTM.com only.
+    # prepare:
+    #   packages:
+    #     - example_package
+    # Add an `after-prepare` step if you need to run commands after the prepare step.
+    # Each command should be listed on a separate line.
+    # after_prepare:
+    # - export PATH=$LGTM_WORKSPACE/tools:$PATH
+    # The `index` step extracts information from the files in the codebase.
+    # index:
+      # Specify Gradle settings.
+      # gradle:
+        # Specify the required Gradle version.
+        # Default: determined automatically.
+        # version: 4.4
+      # Override the autobuild process by specifying a list of custom build commands
+      # to use instead. 
+      # build_command: ./compile-all.sh
+      # Specify the Java version required to build the project.
+      # java_version: 11
+      # Specify whether to extract Java .properties files
+      # Default: false
+      # properties_files: true
+      # Specify Maven settings.
+      # maven:
+        # Specify the path (absolute or relative) of a Maven settings file to use.
+        # Default: Maven uses a settings file in the default location, if it exists.
+        # settings_file: /opt/share/settings.xml
+        # Specify the path of a Maven toolchains file.
+        # Default: Maven uses a toolchains file in the default location, if it exists.
+        # toolchains_file: /opt/share/toolchains.xml
+        # Specify the required Maven version.
+        # Default: the Maven version is determined automatically, where feasible.
+        # version: 3.5.2
+      # Specify how XML files should be extracted:
+      #   all = extract all XML files.
+      #   default = only extract XML files named `AndroidManifest.xml`, `pom.xml`, and `web.xml`.
+      #   disabled = do not extract any XML files.
+      # xml_mode: all
+
+  # Define settings for JavaScript analysis
+  ##########################################
+  # javascript:
+    # The `prepare` step exists for customization on LGTM.com only.
+    # prepare:
+    #   packages:
+    #     - example_package
+    # Add an `after-prepare` step if you need to run commands after the prepare step.
+    # Each command should be listed on a separate line.
+    # after_prepare:
+    # - export PATH=$LGTM_WORKSPACE/tools:$PATH
+    # The `index` step extracts information from the files in the codebase.
+    # index:
+      # Specify a list of files and folders to extract.
+      # Default: The project root directory.
+      # include:
+              # - src/js
+      # Specify a list of files and folders to exclude from extraction.
+      # exclude:
+              # - thirdparty/lib
+      # You can add additional file types for LGTM to extract, by mapping file 
+      # extensions (including the leading dot) to file types. The usual 
+      # include/exclude patterns apply, so, for example, `.jsm` files under
+      # `thirdparty/lib` will not be extracted.
+      # filetypes:
+      # ".jsm": "js"
+      # ".tmpl": "html"
+      # Specify a list of glob patterns to include/exclude files from extraction; this 
+      # is applied on top of the include/exclude paths from above; patterns are 
+      # processed in the same way as for path classifiers above.
+      # Default: include all files with known extensions (such as .js, .ts and .html),
+      # but exclude files ending in `-min.js` or `.min.js` and folders named `node_modules`
+      # or `bower_components`
+      # filters:
+        # exclude any *.ts files anywhere.
+        # - exclude: "**/*.ts"
+        # but include *.ts files under src/js/typescript.
+        # - include: "src/js/typescript/**/*.ts"
+      # Specify how TypeScript files should be extracted:
+      #   none = exclude all TypeScript files.
+      #   basic = extract syntactic information from TypeScript files.
+      #   full = extract syntactic and type information from TypeScript files.
+      # Default: full.
+      # typescript: basic
+      # By default, LGTM doesn't extract any XML files. You can override this by 
+      # using the `xml_mode` property and setting it to `all`.
+      # xml_mode: all
+
+  # Define settings for Python analysis
+  ######################################
+  #  python:
+  #    # The `prepare` step exists for customization on LGTM.com only.
+  #    # prepare:
+  #    #   # The `packages` section is valid for LGTM.com only. It names packages to
+  #    #   # be installed.
+  #    #   packages: libpng-dev
+  #    # This step is useful for Python analysis where you want to prepare the 
+  #    # environment for the `python_setup` step without changing the default behavior 
+  #    # for that step.
+  #    after_prepare:
+  #      - export PATH=$LGTM_WORKSPACE/tools:$PATH
+  #      # This sets up the Python interpreter and virtual environment, ready for the 
+  #      # `index` step to extract the codebase.
+  #    python_setup:
+  #      # Specify packages that should NOT be installed despite being mentioned in the
+  #      # requirements.txt file.
+  #      # Default: no package marked for exclusion.
+  #      exclude_requirements:
+  #        - pywin32
+  #      # Specify a list of pip packages to install.
+  #      # If any of these packages cannot be installed, the extraction will fail.
+  #      requirements:
+  #        - Pillow
+  #      # Specify a list of requirements text files to use to set up the environment,
+  #      # or false for none. Default: any requirements.txt, test-requirements.txt,
+  #      # and similarly named files identified in the codebase are used.
+  #      requirements_files:
+  #        - required-packages.txt
+  #      # Specify a setup.py file to use to set up the environment, or false for none.
+  #      # Default: any setup.py files identified in the codebase are used in preference
+  #      # to any requirements text files.
+  #      setup_py: new-setup.py
+  #      # Override the version of the Python interpreter used for setup and extraction
+  #      # Default: Python 3.
+  #      version: 2
+  #    # Optional step. You should add a `before_index` step if you need to run commands
+  #    # before the `index` step.
+  #    before_index:
+  #      - antlr4 -Dlanguage=Python3 Grammar.g4
+  #    # The `index` step extracts information from the files in the codebase.
+  #    index:
+  #      # Specify a list of files and folders to exclude from extraction.
+  #      # Default: Git submodules and Subversion externals.
+  #      exclude:
+  #        - legacy-implementation
+  #        - thirdparty/libs
+  #      filters:
+  #       - exclude: "**/documentation/examples/snippets/*.py"
+  #       - include: "**/documentation/examples/test_application/*"
+  #      include:
+  #        - example/to/include

.travis.yml

@@ -54,6 +54,7 @@ matrix:
 
           py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev
           pip3 install psutil
+          pip3 install guppy3
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests

Jenkinsfile

@@ -1,65 +1,119 @@
 pipeline {
-  agent any
+  agent none
+  environment{
+      WK = '/var/lib/jenkins/workspace/TDinternal'
+      WKC= '/var/lib/jenkins/workspace/TDinternal/community'
+  }
   stages {
-    stage('build TDengine') {
+      stage('Parallel test stage') {
-      steps {
-        sh '''cd ${WORKSPACE}
-export TZ=Asia/Harbin
-date
-rm -rf ${WORKSPACE}/debug
-mkdir debug
-cd debug
-cmake .. > /dev/null
-make > /dev/null
-cd ${WORKSPACE}/debug'''
-      }
-    }
-
-    stage('test_tsim') {
       parallel {
-        stage('test') {
+        stage('pytest') {
+          agent{label 'master'}
           steps {
-            sh '''cd ${WORKSPACE}/tests
+            sh '''
-#./test-all.sh smoke
+            date
-sudo ./test-all.sh full'''
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests
+            #./test-all.sh smoke
+            ./test-all.sh pytest
+            date'''
+          }
+        }
+        stage('test_b1') {
+          agent{label '184'}
+          steps {
+            sh '''
+            date
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests
+            #./test-all.sh smoke
+            ./test-all.sh b1
+            date'''
           }
         }
 
         stage('test_crash_gen') {
+          agent{label "185"}
           steps {
-            sh '''cd ${WORKSPACE}/tests/pytest
+            sh '''
-sudo ./crash_gen.sh -a -p -t 4 -s 2000'''
+            
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests/pytest
+            ./crash_gen.sh -a -p -t 4 -s 2000
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
           }
         }
 
         stage('test_valgrind') {
+          agent{label "186"}
           steps {
-            sh '''cd ${WORKSPACE}/tests/pytest
+            sh '''
-sudo ./valgrind-test.sh 2>&1 > mem-error-out.log
+            date
-grep \'start to execute\\|ERROR SUMMARY\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-mem-error-out.log
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests/pytest
+            ./valgrind-test.sh 2>&1 > mem-error-out.log
+            ./handle_val_log.sh
           
-for memError in `grep \'ERROR SUMMARY\' uniq-mem-error-out.log | awk \'{print $4}\'`
+            date
-do
+            cd ${WKC}/tests
-  if [ -n "$memError" ]; then
+            ./test-all.sh b3
-    if [ "$memError" -gt 12 ]; then
+            date'''
-      echo -e "${RED} ## Memory errors number valgrind reports is $memError.\\
-               More than our threshold! ## ${NC}"
-      travis_terminate $memError
-    fi
-  fi
-done
-
-grep \'start to execute\\|definitely lost:\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-definitely-lost-out.log
-for defiMemError in `grep \'definitely lost:\' uniq-definitely-lost-out.log | awk \'{print $7}\'`
-do
-  if [ -n "$defiMemError" ]; then
-    if [ "$defiMemError" -gt 13 ]; then
-      echo -e "${RED} ## Memory errors number valgrind reports \\
-               Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
-      travis_terminate $defiMemError
-    fi
-  fi
-done'''
           }
         }
 
@@ -67,4 +121,5 @@ done'''
     }
 
   }
+  
 }

README.md

@@ -83,12 +83,18 @@ sudo dnf install -y maven
 
 ## Get the source codes
 
-- github:
+First of all, you may clone the source codes from github:
 ```bash
 git clone https://github.com/taosdata/TDengine.git
 cd TDengine
 ```
 
+The connectors for go & grafana have been moved to separated repositories,
+so you should run this command in the TDengine directory to install them:
+```bash
+git submodule update --init --recursive
+```
+
 ## Build TDengine
 
 ### On Linux platform

cmake/env.inc

@@ -45,7 +45,7 @@ IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
 ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
   MESSAGE(STATUS "Build Release Version")
 ELSE ()
-  IF (TD_WINDOWS_64)
+  IF (TD_WINDOWS)
     SET(CMAKE_BUILD_TYPE "Release")
     MESSAGE(STATUS "Build Release Version in Windows as default")
   ELSE ()

cmake/install.inc

@@ -16,6 +16,7 @@ ELSEIF (TD_WINDOWS)
   INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
   INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
   INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
+  INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) 

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.4.0")
+  SET(TD_VER_NUMBER "2.0.5.1")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

deps/zlib-1.2.11/inc/zconf.h

@@ -472,7 +472,7 @@ typedef uint64_t  z_crc_t;
 #endif
 #ifndef Z_SOLO
 #if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else

deps/zlib-1.2.11/src/zconf.h

@@ -472,7 +472,7 @@ typedef uLong FAR uLongf;
 #endif
 #ifndef Z_SOLO
 #  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
-#if (_WIN64)
+#if defined(_WIN64) || defined(_WIN32)
 #include <io.h>
 #include <process.h>
 #else

documentation20/webdocs/markdowndocs/Evaluation-ch.md

@@ -11,7 +11,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的
 * __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 
 * __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
 * __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
-* __零运维成本、零学习成本__：安装、集群一秒搞定，无需分库分表，实时备份。标准SQL，支持JDBC, RESTful, 支持Python/Java/C/C++/Go, 与MySQL相似，零学习成本。
+* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。
 
 采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
 

documentation20/webdocs/markdowndocs/Getting Started-ch.md

@@ -30,13 +30,13 @@ TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版
 
 - TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M)
 
-目前，TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which`命令来检测系统中是否存在`systemd`:
+目前，TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which systemctl`命令来检测系统中是否存在`systemd`包:
 
 ```cmd
-which systemd
+which systemctl
 ```
 
-如果系统中不存在`systemd`命令，请考虑[通过源码安装](#通过源码安装)TDengine。
+如果系统中不存在`systemd`包，请考虑[通过源码安装](#通过源码安装)TDengine。
 
 具体的安装过程，请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>。
 

documentation20/webdocs/markdowndocs/Getting Started.md

@@ -16,13 +16,13 @@ Three different packages are provided, please pick up the one you like.
 <li><a id='tdengine-deb' style='color:var(--b2)'>TDengine DEB package (1.7M)</a></li>
 <li><a id='tdengine-tar' style='color:var(--b2)'>TDengine Tarball (3.0M)</a></li>
 </ul>
-For the time being, TDengine only supports installation on Linux systems using [`systemd`](https://en.wikipedia.org/wiki/Systemd) as the service manager. To check if your system has *systemd*, use the _which_ command.
+For the time being, TDengine only supports installation on Linux systems using [`systemd`](https://en.wikipedia.org/wiki/Systemd) as the service manager. To check if your system has *systemd* package, use the _which systemctl_ command.
 
 ```cmd
-which systemd
+which systemctl
 ```
 
-If the `systemd` command is not found, please [install from source code](#Install-from-Source). 
+If the `systemd` package is not found, please [install from source code](#Install-from-Source). 
 
 ### Running TDengine
 

documentation20/webdocs/markdowndocs/Queries-ch.md

@@ -29,23 +29,9 @@ Query OK, 2 row(s) in set (0.001100s)
 具体的查询语法请看<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a>。
 
 ## 多表聚合查询
+物联网场景中，往往同一个类型的数据采集点有多个。TDengine采用超级表(STable)的概念来描述某一个类型的数据采集点，一张普通的表来描述一个具体的数据采集点。同时TDengine使用标签来描述数据采集点的静态属性，一个具体的数据采集点有具体的标签值。通过指定标签的过滤条件，TDengine提供了一高效的方法将超级表(某一类型的数据采集点)所属的子表进行聚合查询。对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样。  
 
-TDengine对每个数据采集点单独建表，但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是包含多张表的表集合，集合里每张表的模式（schema）完全一致，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。
+**示例1**：在TAOS Shell，查找北京所有智能电表采集的电压平均值，并按照location分组
-
-应用可通过指定标签的过滤条件，对一个STable下的全部或部分表进行聚合或统计操作，这样大大简化应用的开发。其具体流程如下图所示：
-
-<center> <img src="../assets/stable.png"> </center>
-
-<center> 多表聚合查询原理图  </center>
-
-1：应用将一个查询条件发往系统；2: taosc将超级表的名字发往 Meta Node（管理节点)；3：管理节点将超级表所拥有的 vnode 列表发回 taosc；4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点；5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc；6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。
-
-由于TDengine在vnode内将标签数据与时序数据分离存储，通过先在内存里过滤标签数据，将需要扫描的数据集大幅减少，大幅提升聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。
-
-对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看 TAOS SQL。
-
-比如：在TAOS Shell，查找所有智能电表采集的电压平均值，并按照location分组
-
 ```mysql
 taos> SELECT AVG(voltage) FROM meters GROUP BY location;
        avg(voltage)        |            location            |
@@ -55,6 +41,18 @@ taos> SELECT AVG(voltage) FROM meters GROUP BY location;
 Query OK, 2 row(s) in set (0.002136s)
 ```
 
+**示例2**：在TAOS shell, 查找groupId为2的所有智能电表过去24小时的记录条数，电流的最大值
+
+```mysql
+taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h;
+     cunt(*)  |    max(current)  |
+==================================
+            5 |             13.4 |
+Query OK, 1 row(s) in set (0.002136s)
+```
+
+TDengine仅容许对属于同一个超级表的表之间进行聚合查询，不同超级表之间的聚合查询不支持。在<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a>一章，查询类操作都会注明是否支持超级表。
+
 ## 降采样查询、插值
 
 物联网场景里，经常需要通过降采样（down sampling）将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如，将智能电表 d1001 采集的电流值每10秒钟求和
@@ -66,9 +64,9 @@ taos> SELECT sum(current) FROM d1001 INTERVAL(10s);
  2018-10-03 14:38:10.000 |              24.900000572 |
 Query OK, 2 row(s) in set (0.000883s)
 ```
-降采样操作也适用于超级表，比如：将所有智能电表采集的电流值每秒钟求和
+降采样操作也适用于超级表，比如：将北京所有智能电表采集的电流值每秒钟求和
 ```mysql
-taos> SELECT SUM(current) FROM meters INTERVAL(1s);
+taos> SELECT SUM(current) FROM meters where location like "Beijing%" INTERVAL(1s);
            ts            |       sum(current)        |
 ======================================================
  2018-10-03 14:38:04.000 |              10.199999809 |

documentation20/webdocs/markdowndocs/TAOS SQL-ch.md

@@ -125,6 +125,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
     1) 表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；
     2) 表名最大长度为193；
     3) 表的每行长度不能超过16k个字符;
+    4) 子表名只能由字母、数字和下划线组成，且不能以数字开头
     5) 使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节；
 
 - **删除数据表**

documentation20/webdocs/markdowndocs/administrator-ch.md

@@ -82,8 +82,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 
 下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节，而且这些参数的缺省配置都是工作的，一般无需设置。**注意：配置修改后，需要重启*taosd*服务才能生效。**
 
-- firstEp: taosd启动时，主动连接的集群中第一个dnode的end point, 默认值为localhost:6030。
+- firstEp: taosd启动时，主动连接的集群中首个dnode的end point, 默认值为localhost:6030。
-- secondEp: taosd启动时，如果first连接不上，尝试连接集群中第二个dnode的end point, 默认值为空。
 - fqdn：数据节点的FQDN，缺省为操作系统配置的第一个hostname。如果习惯IP地址访问，可设置为该节点的IP地址。
 - serverPort：taosd启动后，对外服务的端口号，默认值为6030。
 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求, 默认值为6041。
@@ -156,76 +155,80 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同
 客户端配置参数
 
 - firstEp: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。
-- secondEp: taos启动时，如果first连接不上，尝试连接集群中第二个taosd实例的end point, 缺省值为空。
 - locale
 
-    > 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
+    默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
     
-TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是，编码正确性是客户端来保证。因此，如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。
+    TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是，编码正确性是客户端来保证。因此，如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。
 
-客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。
+    客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。
 
-在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。
+    在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如：zh_CN.UTF-8，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。
 
 - charset
 
-    > 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
+    默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
     
-如果配置文件中不设置charset，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，则中断启动过程。
+    如果配置文件中不设置charset，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，则中断启动过程。
 
-在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如：
+    在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如：
-```
+    ```
     locale zh_CN.UTF-8
-```
+    ```
-在Windows系统中，无法从locale获取系统当前编码。如果无法从配置文件中读取字符串编码信息，taos默认设置为字符编码为CP936。其等效在配置文件中添加如下配置：
+    在Windows系统中，无法从locale获取系统当前编码。如果无法从配置文件中读取字符串编码信息，taos默认设置为字符编码为CP936。其等效在配置文件中添加如下配置：
-```
+    ```
     charset CP936
-```
+    ```
-如果需要调整字符编码，请查阅当前操作系统使用的编码，并在配置文件中正确设置。
+    如果需要调整字符编码，请查阅当前操作系统使用的编码，并在配置文件中正确设置。
 
-在Linux系统中，如果用户同时设置了locale和字符集编码charset，并且locale和charset的不一致，后设置的值将覆盖前面设置的值。
+    在Linux系统中，如果用户同时设置了locale和字符集编码charset，并且locale和charset的不一致，后设置的值将覆盖前面设置的值。
-```
+    ```
     locale zh_CN.UTF-8
     charset GBK
-```
+    ```
-则charset的有效值是GBK。
+    则charset的有效值是GBK。
-```
+    ```
     charset GBK
     locale zh_CN.UTF-8
-```
+    ```
-charset的有效值是UTF-8。
+    charset的有效值是UTF-8。
 
-日志的配置参数，与server 的配置参数完全一样。
+    日志的配置参数，与server 的配置参数完全一样。
 
 - timezone
 
     默认值：从系统中动态获取当前的时区设置
 
-客户端运行系统所在的时区。为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。
+    客户端运行系统所在的时区。为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。
 
-在Linux系统中，客户端会自动读取系统设置的时区信息。用户也可以采用多种方式在配置文件设置时区。例如：
+    在Linux系统中，客户端会自动读取系统设置的时区信息。用户也可以采用多种方式在配置文件设置时区。例如：
-```
+    ```
     timezone UTC-8
     timezone GMT-8
     timezone Asia/Shanghai
-```
+    ```
-均是合法的设置东八区时区的格式。
+    均是合法的设置东八区时区的格式。
 
-时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词now的解析）产生影响。例如：
+    时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词now的解析）产生影响。例如：
-```
+    ```
     SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08';
-```
+    ```
-在东八区，SQL语句等效于
+    在东八区，SQL语句等效于
-```
+    ```
     SELECT count(*) FROM table_name WHERE TS<1554955268000;
-```
+    ```
-在UTC时区，SQL语句等效于
+    在UTC时区，SQL语句等效于
-```
+    ```
     SELECT count(*) FROM table_name WHERE TS<1554984068000;
-```
+    ```
-为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
+    为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
+
+    启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。
+   
+- maxBinaryDisplayWidth
+
+    Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。
    
-启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。
 
 ## 用户管理
 
@@ -378,7 +381,7 @@ KILL STREAM <stream-id>;
 
 ## 系统监控
 
-TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
+TDengine启动后，会自动创建一个监测数据库log，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在log库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
 
@@ -408,5 +411,4 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 
 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
 
-##
 

documentation20/webdocs/markdowndocs/architecture-ch.md

@@ -65,24 +65,24 @@ TDengine 的设计是基于单个硬件、软件系统不可靠，基于任何
 TDengine 分布式架构的逻辑结构图如下：
 <center> <img src="../assets/structure.png"> </center>
 <center> 图 1 TDengine架构示意图  </center>
-一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine客户端(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
+一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
 
-**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。
+**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。
 
 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。
 
-**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的SCHEMA、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的VGroup ID在系统内唯一标识，由管理节点创建并管理。
+**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的schema、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的VGroup ID在系统内唯一标识，由管理节点创建并管理。
 
 **管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
 
 **虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的ID，VGroup ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。
 
-**TAOSC:** taosc是TDengine给应用提供的驱动程序(driver)，负责处理应用与集群的接口交互，内嵌于JDBC、ODBC driver中，或者C、Python、Go语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的数据节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, ODBC, C/C++接口而言，这个模块是在应用所处的物理节点上运行，但消耗的资源很小。同时，为支持全分布式的RESTful接口，taosc在TDengine集群的每个dnode上都有一运行实例。
+**TAOSC:** taosc是TDengine给应用提供的驱动程序(driver)，负责处理应用与集群的接口交互，提供C/C++语言原生接口，内嵌于JDBC、C#、Python、Go、Node.js语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的数据节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, C/C++/C#/Python/Go/Node.js接口而言，这个模块是在应用所处的物理节点上运行。同时，为支持全分布式的RESTful接口，taosc在TDengine集群的每个dnode上都有一运行实例。
 
 ### 节点之间的通讯
-**通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。
+**通讯方式：**TDengine系统的各个数据节点之间，以及应用驱动与各数据节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。
 
-**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。
+**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。
 
 **端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP连接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。
 
@@ -153,6 +153,7 @@ TDengine除vnode分片之外，还对时序数据按照时间段进行分区。
 当新的数据节点被添加进集群，因为新的计算和存储被添加进来，系统也将自动启动负载均衡流程。
 
 负载均衡过程无需任何人工干预，应用也无需重启，将自动连接新的节点，完全透明。
+**提示：负载均衡由参数balance控制，决定开启/关闭自动负载均衡。**
 
 ## 数据写入与复制流程
 如果一个数据库有N个副本，那一个虚拟节点组就有N个虚拟节点，但是只有一个是Master，其他都是slave。当应用将新的记录写入系统时，只有Master vnode能接受写的请求。如果slave vnode收到写的请求，系统将通知taosc需要重新定向。
@@ -192,7 +193,8 @@ Master Vnode遵循下面的写入流程：
 
 理论上，只要是异步复制，就无法保证100%不丢失。但是这个窗口极小，mater与slave要同时发生故障，而且发生在刚给应用确认写入成功之后。
 
-注：异地容灾、IDC无中断迁移，仅仅企业版支持
+注：异地容灾、IDC无中断迁移，仅仅企业版支持。
+**提示：该功能暂未提供**
 
 ### 主从选择
 Vnode会保持一个数据版本号(Version)，对内存数据进行持久化存储时，对该版本号也进行持久化存储。每个数据更新操作，无论是采集的时序数据还是元数据，这个版本号将增一。
@@ -259,6 +261,7 @@ dataDir /mnt/disk6/taos 2
 挂载的盘也可以是非本地的网络盘，只要系统能访问即可。
 
 注：多级存储功能仅企业版支持
+**提示：该功能暂未提供**
 
 ## 数据查询
 TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。
@@ -289,11 +292,18 @@ select count(*) from d1001 interval(1h) fill(prev);
 针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。
 
 ### 多表聚合查询
-多表聚合查询与单表查询的整体流程相同，但是存在如下的差异：
+TDengine对每个数据采集点单独建表，但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是包含多张表的表集合，集合里每张表的模式（schema）完全一致，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。 应用可通过指定标签的过滤条件，对一个STable下的全部或部分表进行聚合或统计操作，这样大大简化应用的开发。其具体流程如下图所示：
+<center> <img src="../assets/multi_tables.png">  </center>
 
-- 由于多表可能分布在不同的节点(dnode)，因此多表的聚合查询需要首先获得表所在的全部数据节点的信息，并且同时向相关的dnode发出查询请求。
+<center> 图 5 多表聚合查询原理图  </center>
-- 每个vnode的计算获得的中间结果(partial results)需要进行第二阶段的聚合才能形成最终结果，第二阶段的聚合过程在客户端完成。
+1：应用将一个查询条件发往系统；
-- 由于表标签信息存储在vnode中，因此针对标签信息的查询也需要vnode完成。客户端将标签的过滤表达式封装在查询请求结构体中发送给vnode，由vnode的查询执行线程从中抽取出标签查询条件，然后执行查询。标签查询与过滤是在针对表的查询之前完成。标签查询完成以后，将符合条件的表纳入到接下来的查询处理流程中。
+2: taosc将超级表的名字发往 Meta Node（管理节点)；
+3：管理节点将超级表所拥有的 vnode 列表发回 taosc；
+4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点；
+5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc；
+6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。 
+
+由于TDengine在vnode内将标签数据与时序数据分离存储，通过在内存里过滤标签数据，先找到需要参与聚合操作的表的集合，将需要扫描的数据集大幅减少，大幅提升聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。 对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看 TAOS SQL。
 
 ### 预计算
 为有效提升查询处理的性能，针对物联网数据的不可更改的特点，在数据块头部记录该数据块中存储数据的统计信息：包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据，直接使用预计算结果，完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小，对于磁盘IO为瓶颈的查询处理，使用预计算结果可以极大地减小读取IO压力，加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN（block range index）有异曲同工之妙。

documentation20/webdocs/markdowndocs/cluster-ch.md

@@ -8,7 +8,7 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预
 
 ## 准备工作
 
-**第零步**：如果没有部署DNS服务，请规划集群所有物理节点的FQDN，然后按照《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》里的步骤，将所有集群物理节点的IP与FQDN的对应关系添加好。
+**第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS，请联系网络管理员在DNS上做好相关配置】
 
 **第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) 
 **注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）；
@@ -16,9 +16,9 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预
 
 **第二步**：建议关闭所有物理节点的防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口；
 
-**第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**。安装时，提示输入是否要加入一个已经存在的TDengine集群时，第一个物理节点直接回车创建新集群，后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030)；
+**第三步**：在所有物理节点安装TDengine，且版本必须是一致的，**但不要启动taosd**。安装时，提示输入是否要加入一个已经存在的TDengine集群时，第一个物理节点直接回车创建新集群，后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030)；
 
-**第四步**：检查所有数据节点，以及应用所在物理节点的网络设置：
+**第四步**：检查所有数据节点，以及应用程序所在物理节点的网络设置：
 
 1. 每个物理节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查)；
 2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts)，或DNS的配置。如果无法ping通，是无法组成集群的；

documentation20/webdocs/markdowndocs/connector-java-ch.md

@@ -72,38 +72,34 @@ maven 项目中使用如下 pom.xml 配置即可：
 
 ### 获取连接
 
-如下所示配置即可获取 TDengine Connection：
+#### 通过JdbcUrl获取连接
+
+通过指定的jdbcUrl获取连接，如下所示：
 ```java
 Class.forName("com.taosdata.jdbc.TSDBDriver");
-String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata";
+String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
 Connection conn = DriverManager.getConnection(jdbcUrl);
 ```
-> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。
+以上示例，建立了到hostname为taosdemo.com，端口为6030（TDengine的默认端口），数据库名为test的连接。这个url中指定用户名（user）为root，密码（password）为taosdata。
 
 TDengine 的 JDBC URL 规范格式为：
-`jdbc:TAOS://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
+`jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
-
+url中的配置参数如下：
-其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下：
-
 * user：登录 TDengine 用户名，默认值 root。
 * password：用户登录密码，默认值 taosdata。
-* charset：客户端使用的字符集，默认值为系统字符集。
 * cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* charset：客户端使用的字符集，默认值为系统字符集。
 * locale：客户端语言环境，默认值系统当前 locale。
 * timezone：客户端使用的时区，默认值为系统当前时区。
 
-以上参数可以在 3 处配置，`优先级由高到低`分别如下：
+#### 使用JdbcUrl和Properties获取连接
-1. JDBC URL 参数
+
-    如上所述，可以在 JDBC URL 的参数中指定。
+除了通过指定的jdbcUrl获取连接，还可以使用Properties指定建立连接时的参数，如下所示：
-2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
 ```java
 public Connection getConn() throws Exception{
   Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata";
+  String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
   Properties connProps = new Properties();
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
   connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
@@ -111,16 +107,39 @@ public Connection getConn() throws Exception{
   return conn;
 }
 ```
+以上示例，建立一个到hostname为taosdemo.com，端口为6030，数据库名为test的连接。这个连接在url中指定了用户名(user)为root，密码（password）为taosdata，并在connProps中指定了使用的字符集、语言环境、时区等信息。
 
-3. 客户端配置文件 taos.cfg
+properties中的配置参数如下：
+* TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。
+* TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。
+* TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
+* TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
+* TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。
+* TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。
 
-    linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
+#### 使用客户端配置文件建立连接
-```properties
+当使用JDBC连接TDengine集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的firstEp、secondEp参数。
-# client default username
+如下所示：
-# defaultUser           root
+1. 在java中不指定hostname和port
+```java
+public Connection getConn() throws Exception{
+  Class.forName("com.taosdata.jdbc.TSDBDriver");
+  String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
+  Properties connProps = new Properties();
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+  return conn;
+}
+```
+2. 在配置文件中指定firstEp和secondEp
+```
+# first fully qualified domain name (FQDN) for TDengine system
+firstEp               cluster_node1:6030
 
-# client default password
+# second fully qualified domain name (FQDN) for TDengine system, for cluster only
-# defaultPass           taosdata
+secondEp              cluster_node2:6030
 
 # default system charset
 # charset               UTF-8
@@ -128,6 +147,19 @@ public Connection getConn() throws Exception{
 # system locale
 # locale                en_US.UTF-8
 ```
+
+以上示例，jdbc会使用客户端的配置文件，建立到hostname为cluster_node1，端口为6030，数据库名为test的连接。当集群中firstEp节点失效时，JDBC会尝试使用secondEp连接集群。
+TDengine中，只要保证firstEp和secondEp中一个节点有效，就可以正常建立到集群的连接。
+
+> 注意：这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
+
+#### 配置参数的优先级
+通过以上3种方式获取连接，如果配置参数在url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下：
+1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。
+2. Properties connProps
+3. 客户端配置文件 taos.cfg
+例如：在url中指定了password为taosdata，在Properties中指定了password为taosdemo，那么，JDBC会使用url中的password建立连接。
+
 > 更多详细配置请参考[客户端配置][13]
 
 ### 创建数据库和表

documentation20/webdocs/markdowndocs/insert-ch.md

@@ -22,7 +22,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6,
 
 **Tips:** 
 
-- 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为8M）。
+- 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为1M）。
 - TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。
 - 对同一张表，如果新插入记录的时间戳已经存在，新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。
 - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。

packaging/cfg/taos.cfg

@@ -6,10 +6,7 @@
 ########################################################
 
 # first fully qualified domain name (FQDN) for TDengine system
-# firstEp               hostname1:6030
+# firstEp                   hostname:6030
-
-# second fully qualified domain name (FQDN) for TDengine system, for cluster only
-# secondEp              cluster_hostname2:6030
 
 # local fully qualified domain name (FQDN)
 # fqdn                      hostname
@@ -29,12 +26,18 @@
 # number of threads per CPU core
 # numOfThreadsPerCore       1.0
 
+# the proportion of total threads responsible for query
+# ratioOfQueryThreads       0.5
+
 # number of management nodes in the system
 # numOfMnodes               3
 
 # enable/disable backuping vnode directory when removing dnode
 # vnodeBak                  1
 
+# if report installation / use information
+# telemetryReporting        1
+
 # enable/disable load balancing
 # balance                   1
 
@@ -89,9 +92,6 @@
 # max number of tables per vnode
 # maxTablesPerVnode         1000000
 
-# step size of increasing table number in a vnode
-# tableIncStepPerVnode      1000
-
 # cache block size (Mbyte)
 # cache                     16
 
@@ -110,6 +110,9 @@
 # maximum rows of records in file block
 # maxRows               4096
 
+# the number of acknowledgments required for successful data writing
+# quorum                1     
+
 # enable/disable compression
 # comp                  2
 
@@ -122,15 +125,6 @@
 # number of replications, for cluster only 
 # replica               1
 
-# mqtt hostname  
-# mqttHostName          test.mosquitto.org
-
-# mqtt port  
-# mqttPort              1883
-
-# mqtt topic
-# mqttTopic             /test
-
 # the compressed rpc message, option:
 #  -1 (no compression)
 #   0 (all message compressed),
@@ -167,12 +161,12 @@
 # stop writing data when the disk size of the log folder is less than this value
 # minimalDataDirGB      0.1
 
+# One mnode is equal to the number of vnode consumed
+# mnodeEqualVnodeNum    4
+
 # enbale/disable http service
 # http                  1
 
-# enable/disable muqq service
-# mqtt                  0
-
 # enable/disable system monitor 
 # monitor               1
 
@@ -189,11 +183,12 @@
 # max number of rows per log filters
 # numOfLogLines         10000000
 
+# enable/disable async log
+# asyncLog              1
+
 # time of keeping log files, days
 # logKeepDays           0
 
-# enable/disable async log
-# asyncLog              1
 
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
@@ -231,18 +226,12 @@
 # debug flag for JNI
 # jniDebugflag          131
 
-# debug flag for ODBC
-# odbcDebugflag         131
-
 # debug flag for storage
 # uDebugflag            131
 
 # debug flag for http server
 # httpDebugFlag         131
 
-# debug flag for mqtt
-# mqttDebugFlag         131
-
 # debug flag for monitor
 # monitorDebugFlag      131
 
@@ -255,6 +244,9 @@
 # debug flag for http server
 # tsdbDebugFlag         131
 
+# debug flag for continue query
+# cqDebugFlag           131
+
 # enable/disable recording the SQL in taos client
 # tscEnableRecordSql    0
 

packaging/docker/Dockerfile

@@ -1,10 +1,12 @@
-FROM centos:7
+FROM ubuntu:16
 
 WORKDIR /root
 
+ARG version
+RUN echo $version
 COPY tdengine.tar.gz /root/
 RUN tar -zxf tdengine.tar.gz
-WORKDIR /root/TDengine-server/
+WORKDIR /root/TDengine-server-$version/
 RUN sh install.sh -e no
 
 

packaging/docker/dockerbuild.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -x
-docker build --rm -f "Dockerfile" -t tdengine/tdengine:$1 "."
+docker build --rm -f "Dockerfile" -t tdengine/tdengine:$1 "." --build-arg version=$1
 docker login -u tdengine -p $2  #replace the docker registry username and password
 docker push tdengine/tdengine:$1

packaging/tools/install.sh

@@ -272,6 +272,29 @@ function install_config() {
             break
         fi
     done	
+
+    # user email 
+    #EMAIL_PATTERN='^[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+$'
+    #EMAIL_PATTERN='^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$'
+    #EMAIL_PATTERN="^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$"
+    echo
+    echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: "
+    read emailAddr
+    while true; do
+        if [ ! -z "$emailAddr" ]; then
+            # check the format of the emailAddr
+            #if [[ "$emailAddr" =~ $EMAIL_PATTERN ]]; then
+                # Write the email address to temp file                    
+                email_file="${install_main_dir}/email" 
+                ${csudo} bash -c "echo $emailAddr > ${email_file}"
+                break         
+            #else
+            #    read -p "Please enter the correct email address: " emailAddr   
+            #fi
+        else
+            break
+        fi
+    done	
 }
 
 

snap/snapcraft.yaml

@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: 'RELEASE_VERSION'
+version: '2.0.5.1'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.RELEASE_VERSION
+      - usr/lib/libtaos.so.2.0.5.1
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/balance/src/balance.c

@@ -126,6 +126,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
 
   balanceAccquireDnodeList();
 
+  mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
+         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     for (; dnode < tsBalanceDnodeListSize; ++dnode) {
       SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
@@ -135,17 +137,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
         pVnodeGid->pDnode = pDnode;
         dnode++;
         vnodes++;
+        mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i);
         break;
+      } else {
+        mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
+               mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
+               pDnode->alternativeRole);
       }
     }
   }
 
   if (vnodes != pVgroup->numOfVnodes) {
-    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
-           pVgroup->numOfVnodes, vnodes);
     balanceReleaseDnodeList();
     balanceUnLock();
 
+    mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
+
+    void *     pIter = NULL;
+    SDnodeObj *pDnode = NULL;
+    while (1) {
+      pIter = mnodeGetNextDnode(pIter, &pDnode);
+      if (pDnode == NULL) break;
+      mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
+             pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
+      mnodeDecDnodeRef(pDnode);
+    }
+    sdbFreeIter(pIter);
+
     if (mnodeGetOnlineDnodesNum() == 0) {
       return TSDB_CODE_MND_NOT_READY;
     } else {
@@ -553,7 +571,8 @@ static void balanceCheckDnodeAccess() {
       if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
         pDnode->status = TAOS_DN_STATUS_OFFLINE;
         pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
-        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
+              pDnode->lastAccess);
         balanceSetVgroupOffline(pDnode);
       }
     }
@@ -937,6 +956,7 @@ static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows,
     mnodeDecDnodeRef(pDnode);
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
@@ -957,11 +977,16 @@ static void balanceMonitorDnodeModule() {
       continue;
     }
 
-    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, add mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
+    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
-    mnodeAddMnode(pDnode->dnodeId);
+    mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true);
 
+#if 0
+    // Only create one mnode each time
+    return;
+#else
     numOfMnodes = mnodeGetMnodesNum();
     if (numOfMnodes >= tsNumOfMnodes) return;
+#endif
   }
 }
 

src/client/inc/tscLocalMerge.h

@@ -72,17 +72,10 @@ typedef struct SLocalReducer {
   bool                   orderPrjOnSTable;   // projection query on stable
 } SLocalReducer;
 
-typedef struct SSubqueryState {
-  int32_t  numOfRemain;         // the number of remain unfinished subquery
-  int32_t  numOfTotal;          // the number of total sub-queries
-  uint64_t numOfRetrievedRows;  // total number of points in this query
-} SSubqueryState;
-
 typedef struct SRetrieveSupport {
   tExtMemBuffer **  pExtMemBuffer;     // for build loser tree
   tOrderDescriptor *pOrderDescriptor;
   SColumnModel *    pFinalColModel;    // colModel for final result
-  SSubqueryState *  pState;
   int32_t           subqueryIndex;     // index of current vnode in vnode list
   SSqlObj *         pParentSql;
   tFilePage *       localBuffer;       // temp buffer, there is a buffer for each vnode to

src/client/inc/tscSubquery.h

@@ -28,7 +28,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql);
 void tscSetupOutputColumnIndex(SSqlObj* pSql);
 void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
 
-SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index);
+SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index);
 
 void tscHandleMasterJoinQuery(SSqlObj* pSql);
 

src/client/inc/tscUtil.h

@@ -66,7 +66,6 @@ typedef struct STidTags {
 #pragma pack(pop)
 
 typedef struct SJoinSupporter {
-  SSubqueryState* pState;
   SSqlObj*        pObj;           // parent SqlObj
   int32_t         subqueryIndex;  // index of sub query
   SInterval       interval;
@@ -110,7 +109,6 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
 
 void*   tscDestroyBlockArrayList(SArray* pDataBlockList);
 int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
-void    tscFreeUnusedDataBlocks(SArray* pDataBlockList);
 int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList);
 int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
                                 int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
@@ -207,8 +205,6 @@ void tscTagCondRelease(STagCond* pCond);
 
 void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
 
-void tscSetFreeHeatBeat(STscObj* pObj);
-bool tscShouldFreeHeartBeat(SSqlObj* pHb);
 bool tscShouldBeFreed(SSqlObj* pSql);
 
 STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex);

src/client/inc/tsclient.h

@@ -80,6 +80,8 @@ enum {
   DATA_FROM_DATA_FILE = 2,
 };
 
+typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
+
 typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
@@ -226,7 +228,7 @@ typedef struct STableDataBlocks {
 typedef struct SQueryInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
   uint32_t         type;          // query/insert type
-  // TODO refactor
+
   STimeWindow      window;        // query time window
   SInterval        interval;
 
@@ -334,6 +336,12 @@ typedef struct STscObj {
   T_REF_DECLARE()
 } STscObj;
 
+typedef struct SSubqueryState {
+  int32_t          numOfRemain;         // the number of remain unfinished subquery
+  int32_t          numOfSub;            // the number of total sub-queries
+  uint64_t         numOfRetrievedRows;  // total number of points in this query
+} SSubqueryState;
+
 typedef struct SSqlObj {
   void            *signature;
   pthread_t        owner;        // owner of sql object, by which it is executed
@@ -355,10 +363,11 @@ typedef struct SSqlObj {
   tsem_t           rspSem;
   SSqlCmd          cmd;
   SSqlRes          res;
-  uint16_t         numOfSubs;
-  struct SSqlObj **pSubs;
-  struct SSqlObj * prev, *next;
 
+  SSubqueryState   subState;
+  struct SSqlObj **pSubs;
+
+  struct SSqlObj  *prev, *next;
   struct SSqlObj **self;
 } SSqlObj;
 
@@ -433,19 +442,21 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql);
  * @param pObj
  */
 void tscFreeSqlObj(SSqlObj *pSql);
-
+void tscFreeRegisteredSqlObj(void *pSql);
-void tscFreeSqlObjInCache(void *pSql);
 
 void tscCloseTscObj(STscObj *pObj);
 
+// todo move to taos? or create a new file: taos_internal.h
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
-                     void *param, TAOS **taos);
+                     void *param, void **taos);
-void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res);
 
-void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
+void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
+
+
+void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
 
 void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
-void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
 bool tscIsUpdateQuery(SSqlObj* pSql);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
@@ -510,8 +521,6 @@ extern SRpcCorEpSet tscMgmtEpSet;
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 
-typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
-
 int32_t tscCompareTidTags(const void* p1, const void* p2);
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
 

src/client/src/tscAsync.c

@@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
 static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
 
-void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
+void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) {
   SSqlCmd* pCmd = &pSql->cmd;
 
   pSql->signature = pSql;
@@ -361,15 +361,6 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
   (*pSql->fetchFp)(pSql->param, pSql, pRes->tsrow);
 }
 
-void tscProcessAsyncRes(SSchedMsg *pMsg) {
-  SSqlObj *pSql = (SSqlObj *)pMsg->ahandle;
-  SSqlRes *pRes = &pSql->res;
-  assert(pSql->fp != NULL && pSql->fetchFp != NULL);
-
-  pSql->fp = pSql->fetchFp;
-  (*pSql->fp)(pSql->param, pSql, pRes->code);
-}
-
 // this function will be executed by queue task threads, so the terrno is not valid
 static void tscProcessAsyncError(SSchedMsg *pMsg) {
   void (*fp)() = pMsg->ahandle;
@@ -393,22 +384,15 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
   if (pSql == NULL || pSql->signature != pSql) {
     tscDebug("%p SqlObj is freed, not add into queue async res", pSql);
     return;
-  } else {
-    tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
   }
 
-  SSchedMsg schedMsg = { 0 };
+  tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
-  schedMsg.fp = tscProcessAsyncRes;
-  schedMsg.ahandle = pSql;
-  schedMsg.thandle = (void *)1;
-  schedMsg.msg = NULL;
-  taosScheduleTask(tscQhandle, &schedMsg);
-}
 
-void tscProcessAsyncFree(SSchedMsg *pMsg) {
+  SSqlRes *pRes = &pSql->res;
-  SSqlObj *pSql = (SSqlObj *)pMsg->ahandle;
+  assert(pSql->fp != NULL && pSql->fetchFp != NULL);
-  tscDebug("%p sql is freed", pSql);
+
-  taos_free_result(pSql);
+  pSql->fp = pSql->fetchFp;
+  (*pSql->fp)(pSql->param, pSql, pRes->code);
 }
 
 int tscSendMsgToServer(SSqlObj *pSql);

src/client/src/tscFunctionImpl.c

@@ -2445,8 +2445,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
   // in the first round, get the min-max value of all involved data
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = pResInfo->interResultBuf;
-  pInfo->minval = DBL_MAX;
+  SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
-  pInfo->maxval = -DBL_MAX;
+  SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
   pInfo->numOfElems = 0;
 
   return true;
@@ -2461,12 +2461,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
   // the first stage, only acquire the min/max value
   if (pInfo->stage == 0) {
     if (pCtx->preAggVals.isSet) {
-      if (pInfo->minval > pCtx->preAggVals.statis.min) {
+      if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
-        pInfo->minval = (double)pCtx->preAggVals.statis.min;
+        SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
       }
 
-      if (pInfo->maxval < pCtx->preAggVals.statis.max) {
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
-        pInfo->maxval = (double)pCtx->preAggVals.statis.max;
+        SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
       }
 
       pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
@@ -2500,12 +2500,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
             break;
         }
 
-        if (v < pInfo->minval) {
+        if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
-          pInfo->minval = v;
+          SET_DOUBLE_VAL(&pInfo->minval, v);
         }
 
-        if (v > pInfo->maxval) {
+        if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
-          pInfo->maxval = v;
+          SET_DOUBLE_VAL(&pInfo->maxval, v);
         }
 
         pInfo->numOfElems += 1;
@@ -2564,12 +2564,12 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
         break;
     }
 
-    if (v < pInfo->minval) {
+    if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
-      pInfo->minval = v;
+      SET_DOUBLE_VAL(&pInfo->minval, v);
     }
 
-    if (v > pInfo->maxval) {
+    if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
-      pInfo->maxval = v;
+      SET_DOUBLE_VAL(&pInfo->maxval, v);
     }
 
     pInfo->numOfElems += 1;
@@ -2609,7 +2609,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
     }
 
     pInfo->stage += 1;
-    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
   } else {
     pResInfo->complete = true;
   }

src/client/src/tscLocalMerge.c

@@ -554,27 +554,48 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
     numOfGroupByCols++;
   }
 
-  int32_t *orderIdx = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t));
+  int32_t *orderColIndexList = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t));
-  if (orderIdx == NULL) {
+  if (orderColIndexList == NULL) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
   if (numOfGroupByCols > 0) {
+
+    if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
       int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
 
-    // tags value locate at the last columns
+      // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns
       for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
-      orderIdx[i] = startCols++;
+        orderColIndexList[i] = startCols++;
       }
 
       if (pQueryInfo->interval.interval != 0) {
         // the first column is the timestamp, handles queries like "interval(10m) group by tags"
-      orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX;
+        orderColIndexList[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; //TODO ???
+      }
+    } else {
+      /*
+       * 1. the orderby ts asc/desc projection query for the super table
+       * 2. interval query without groupby clause
+       */
+      if (pQueryInfo->interval.interval != 0) {
+        orderColIndexList[0] = PRIMARYKEY_TIMESTAMP_COL_INDEX;
+      } else {
+        size_t size = tscSqlExprNumOfExprs(pQueryInfo);
+        for (int32_t i = 0; i < size; ++i) {
+          SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
+          if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+            orderColIndexList[0] = i;
+          }
         }
       }
 
-  *pOrderDesc = tOrderDesCreate(orderIdx, numOfGroupByCols, pModel, pQueryInfo->order.order);
+      assert(pQueryInfo->order.orderColId == PRIMARYKEY_TIMESTAMP_COL_INDEX);
-  taosTFree(orderIdx);
+    }
+  }
+
+  *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
+  taosTFree(orderColIndexList);
 
   if (*pOrderDesc == NULL) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -588,7 +609,6 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
 
   // disable merge procedure for column projection query
   int16_t functionId = pReducer->pCtx[0].functionId;
-  assert(functionId != TSDB_FUNC_ARITHM);
   if (pReducer->orderPrjOnSTable) {
     return true;
   }
@@ -606,7 +626,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
     return true;
   }
 
-  if (orderInfo->pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+  if (orderInfo->colIndex[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
     /*
      * super table interval query
      * if the order columns is the primary timestamp, all result data belongs to one group
@@ -620,7 +640,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
   }
 
   // only one row exists
-  int32_t index = orderInfo->pData[0];
+  int32_t index = orderInfo->colIndex[0];
   int32_t offset = (pOrderDesc->pColumnModel)->pFields[index].offset;
 
   int32_t ret = memcmp(pPrev + offset, tmpBuffer->data + offset, pOrderDesc->pColumnModel->rowSize - offset);
@@ -639,7 +659,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->numOfSubs);
+  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->subState.numOfSub);
   if (*pMemBuffer == NULL) {
     tscError("%p failed to allocate memory", pSql);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -661,7 +681,6 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
     pSchema[i].bytes = pExpr->resBytes;
     pSchema[i].type = (int8_t)pExpr->resType;
-
     rlen += pExpr->resBytes;
   }
 
@@ -701,12 +720,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
     int16_t type = -1;
     int16_t bytes = 0;
 
-    //    if ((pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) ||
-    //        (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) ||
-    //        pExpr->functionId == TSDB_FUNC_LAST_ROW) {
     // the final result size and type in the same as query on single table.
     // so here, set the flag to be false;
-
     int32_t functionId = pExpr->functionId;
     if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) {
       type = pModel->pFields[i].field.type;
@@ -742,6 +757,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
                                int32_t numOfVnodes) {
   destroyColumnModel(pFinalModel);
   tOrderDescDestroy(pDesc);
+
   for (int32_t i = 0; i < numOfVnodes; ++i) {
     pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
   }

src/client/src/tscProfile.c

@@ -153,10 +153,12 @@ void tscKillQuery(STscObj *pObj, uint32_t killId) {
 
   pthread_mutex_unlock(&pObj->mutex);
 
-  if (pSql == NULL) return;
+  if (pSql == NULL) {
-
+    tscError("failed to kill query, id:%d, it may have completed/terminated", killId);
+  } else {
     tscDebug("%p query is killed, queryId:%d", pSql, killId);
     taos_stop_query(pSql);
+  }
 }
 
 void tscAddIntoStreamList(SSqlStream *pStream) {
@@ -244,6 +246,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
     pQdesc->stime = htobe64(pSql->stime);
     pQdesc->queryId = htonl(pSql->queryId);
     pQdesc->useconds = htobe64(pSql->res.useconds);
+    pQdesc->qHandle = htobe64(pSql->res.qhandle);
 
     pHeartbeat->numOfQueries++;
     pQdesc++;

src/client/src/tscSQLParser.c

@@ -626,6 +626,11 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
     return TSDB_CODE_SUCCESS;
   }
 
+  // orderby column not set yet, set it to be the primary timestamp column
+  if (pQueryInfo->order.orderColId == INT32_MIN) {
+    pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
+  }
+
   // interval is not null
   SStrToken* t = &pQuerySql->interval;
   if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) {
@@ -1200,6 +1205,10 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr
   return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
 }
 
+static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
+  SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+  tscColumnListInsert(pQueryInfo->colList, &tsCol);
+}
 static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
   const char* msg1 = "invalid column name, or illegal column type";
   const char* msg2 = "invalid arithmetic expression in select clause";
@@ -1275,6 +1284,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
     addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
 
     insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
+    // add ts column
+    tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
 
     tbufCloseWriter(&bw);
     taosArrayDestroy(colList);
@@ -1317,10 +1328,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
-static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
-  SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  tscColumnListInsert(pQueryInfo->colList, &tsCol);
-}
 
 static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
   SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex);
@@ -1345,6 +1352,32 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
   insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr);
 }
 
+static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) {
+  // primary timestamp column has been added already
+  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
+  for (int32_t i = 0; i < size; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      return;
+    }
+  }
+
+  SColumnIndex index = {0};
+
+  // set the constant column value always attached to first table.
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX);
+
+  // add the timestamp column into the output columns
+  int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
+  tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
+
+  SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols);
+  pSupInfo->visible = false;
+
+  pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
+}
+
 int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) {
   assert(pSelection != NULL && pCmd != NULL);
 
@@ -1398,20 +1431,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
   // there is only one user-defined column in the final result field, add the timestamp column.
   size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
   if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) {
-    SColumnIndex index = {0};
+    addPrimaryTsColIntoResult(pQueryInfo);
-
-    // set the constant column value always attached to first table.
-    STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0);
-    SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX);
-
-    // add the timestamp column into the output columns
-    int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
-    tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
-
-    SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols);
-    pSupInfo->visible = false;
-
-    pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
   }
 
   if (!functionCompatibleCheck(pQueryInfo, joinQuery)) {
@@ -1603,8 +1623,8 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName,
+static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
-                                       int32_t resColIdx, SColumnIndex* pColIndex) {
+    char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
   const char* msg1 = "not support column types";
 
   int16_t type = 0;
@@ -1650,8 +1670,13 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
   SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
   tscColumnListInsert(pQueryInfo->colList, &index);
 
+  // if it is not in the final result, do not add it
   SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
+  if (finalResult) {
     insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr);
+  } else {
+    tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0]));
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1926,7 +1951,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
             for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
               index.columnIndex = j;
-              if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index) != 0) {
+              if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) {
                 return TSDB_CODE_TSC_INVALID_SQL;
               }
             }
@@ -1943,7 +1968,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) {
               return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
             }
-            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index) != 0) {
+
+            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
             }
 
@@ -1980,7 +2006,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
           for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) {
             SColumnIndex index = {.tableIndex = j, .columnIndex = i};
-            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index) != 0) {
+            if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
             }
 
@@ -4363,14 +4389,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
 
 static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
   /* set default timestamp order information for all queries */
-  pQueryInfo->order.order = TSDB_ORDER_ASC;
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  if (isTopBottomQuery(pQueryInfo)) {
   pQueryInfo->order.order = TSDB_ORDER_ASC;
+  if (isTopBottomQuery(pQueryInfo)) {
     pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
-  } else {
+  } else { // in case of select tbname from super_table, the defualt order column can not be the primary ts column
-    pQueryInfo->order.orderColId = -1;
+    pQueryInfo->order.orderColId = INT32_MIN;
   }
 
   /* for super table query, set default ascending order for group output */
@@ -4474,6 +4499,11 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
       } else {
         pQueryInfo->order.order = pSortorder->a[0].sortOrder;
         pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
+
+        // orderby ts query on super table
+        if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+          addPrimaryTsColIntoResult(pQueryInfo);
+        }
       }
     }
 
@@ -6269,6 +6299,11 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
+  // set order by info
+  if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_TSC_INVALID_SQL;
+  }
+
   // set interval value
   if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -6279,11 +6314,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     }
   }
 
-  // set order by info
-  if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
-    return TSDB_CODE_TSC_INVALID_SQL;
-  }
-
   // user does not specified the query time window, twa is not allowed in such case.
   if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
        (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {

src/client/src/tscServer.c

@@ -24,7 +24,6 @@
 #include "tschemautil.h"
 #include "tsclient.h"
 #include "ttimer.h"
-#include "tutil.h"
 #include "tlockfree.h"
 
 SRpcCorEpSet  tscMgmtEpSet;
@@ -198,16 +197,20 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
     return;
   }
 
-  if (tscShouldFreeHeartBeat(pHB)) {
+  void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
-    tscDebug("%p free HB object and release connection", pHB);
+  if (p == NULL) {
-    pObj->pHb = 0;
+    tscWarn("%p HB object has been released already", pHB);
-    taos_free_result(pHB);
+    return;
-  } else {
+  }
+
+  assert(*pHB->self == pHB);
+
   int32_t code = tscProcessSql(pHB);
+  taosCacheRelease(tscObjCache, (void**) &p, false);
+
   if (code != TSDB_CODE_SUCCESS) {
     tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
   }
-  }
 }
 
 int tscSendMsgToServer(SSqlObj *pSql) {
@@ -464,35 +467,6 @@ int tscProcessSql(SSqlObj *pSql) {
   return doProcessSql(pSql);
 }
 
-void tscKillSTableQuery(SSqlObj *pSql) {
-  SSqlCmd* pCmd = &pSql->cmd;
-  
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    return;
-  }
-
-  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-
-  for (int i = 0; i < pSql->numOfSubs; ++i) {
-    // NOTE: pSub may have been released already here
-    SSqlObj *pSub = pSql->pSubs[i];
-    if (pSub == NULL) {
-      continue;
-    }
-
-    pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    if (pSub->pRpcCtx != NULL) {
-      rpcCancelRequest(pSub->pRpcCtx);
-      pSub->pRpcCtx = NULL;
-    }
-
-    tscQueueAsyncRes(pSub); // async res? not other functions?
-  }
-
-  tscDebug("%p super table query cancelled", pSql);
-}
-
 int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
   pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
@@ -1451,7 +1425,7 @@ int tscProcessLocalRetrieveRsp(SSqlObj *pSql) {
 
 int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
-  SSqlCmd *pCmd = &pSql->cmd;
+  SSqlCmd* pCmd = &pSql->cmd;
 
   int32_t code = pRes->code;
   if (pRes->code != TSDB_CODE_SUCCESS) {
@@ -1494,6 +1468,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
 
+  // TODO refactor full_name
   char *db;  // ugly code to move the space
   db = strstr(pObj->db, TS_PATH_DELIMITER);
   db = (db == NULL) ? pObj->db : db + 1;
@@ -1501,6 +1476,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
   tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
 
+  pConnect->pid = htonl(taosGetPId());
+  taosGetCurrentAPPName(pConnect->appName, NULL);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1653,6 +1631,10 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
   pHeartbeat->numOfQueries = numOfQueries;
   pHeartbeat->numOfStreams = numOfStreams;
+
+  pHeartbeat->pid = htonl(taosGetPId());
+  taosGetCurrentAPPName(pHeartbeat->appName, NULL);
+
   int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj);
 
   pthread_mutex_unlock(&pObj->mutex);

src/client/src/tscSql.c

@@ -26,7 +26,9 @@
 #include "tsclient.h"
 #include "ttokendef.h"
 #include "tutil.h"
+#include "ttimer.h"
 #include "tscProfile.h"
+#include "ttimer.h"
 
 static bool validImpl(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -259,10 +261,21 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port,
 void taos_close(TAOS *taos) {
   STscObj *pObj = (STscObj *)taos;
 
-  if (pObj == NULL || pObj->signature != pObj)  {
+  if (pObj == NULL) {
+    tscDebug("(null) try to free tscObj and close dnodeConn");
     return;
   }
 
+  tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
+  if (pObj->signature != pObj) {
+    tscDebug("%p already closed or invalid tscObj", pObj);
+    return;
+  }
+
+  // make sure that the close connection can only be executed once.
+  pObj->signature = NULL;
+  taosTmrStopA(&(pObj->pTimer));
+
   SSqlObj* pHb = pObj->pHb;
   if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
     if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode
@@ -298,7 +311,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
   tsem_post(&pSql->rspSem);
 }
 
-TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
+TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
@@ -323,12 +336,20 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
   tsem_init(&pSql->rspSem, 0, 0);
   doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
 
+  if (res != NULL) {
+    *res = pSql;
+  }
+
   tsem_wait(&pSql->rspSem);
   return pSql; 
 }
 
 TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
-  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr));
+  return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL);
+}
+
+TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) {
+  return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res);
 }
 
 int taos_result_precision(TAOS_RES *res) {
@@ -462,6 +483,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
   SSqlRes *pRes = &pSql->res;
   
   if (pRes->qhandle == 0 ||
+      pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED ||
       pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
       pCmd->command == TSDB_SQL_INSERT) {
     return NULL;
@@ -525,7 +547,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
     pRes->numOfClauseTotal = 0;
     pRes->rspType = 0;
 
-    pSql->numOfSubs = 0;
+    pSql->subState.numOfSub = 0;
     taosTFree(pSql->pSubs);
 
     assert(pSql->fp == NULL);
@@ -679,6 +701,45 @@ int* taos_fetch_lengths(TAOS_RES *res) {
 
 char *taos_get_client_info() { return version; }
 
+static void tscKillSTableQuery(SSqlObj *pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    return;
+  }
+
+  // set the master sqlObj flag to cancel query
+  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
+  for (int i = 0; i < pSql->subState.numOfSub; ++i) {
+    // NOTE: pSub may have been released already here
+    SSqlObj *pSub = pSql->pSubs[i];
+    if (pSub == NULL) {
+      continue;
+    }
+
+    void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
+    if (p == NULL) {
+      continue;
+    }
+
+    SSqlObj* pSubObj = (SSqlObj*) (*p);
+    assert(pSubObj->self == (SSqlObj**) p);
+
+    pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+    if (pSubObj->pRpcCtx != NULL) {
+      rpcCancelRequest(pSubObj->pRpcCtx);
+      pSubObj->pRpcCtx = NULL;
+    }
+
+    tscQueueAsyncRes(pSubObj);
+    taosCacheRelease(tscObjCache, (void**) &p, false);
+  }
+
+  tscDebug("%p super table query cancelled", pSql);
+}
+
 void taos_stop_query(TAOS_RES *res) {
   SSqlObj *pSql = (SSqlObj *)res;
   if (pSql == NULL || pSql->signature != pSql) {
@@ -688,21 +749,26 @@ void taos_stop_query(TAOS_RES *res) {
   tscDebug("%p start to cancel query", res);
   SSqlCmd *pCmd = &pSql->cmd;
 
-  // TODO there are multi-thread problem.
-  // It may have been released by the other thread already.
-  // The ref count may fix this problem.
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-
   // set the error code for master pSqlObj firstly
   pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
 
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
     assert(pSql->pRpcCtx == NULL);
     tscKillSTableQuery(pSql);
   } else {
     if (pSql->cmd.command < TSDB_SQL_LOCAL) {
-      assert(pSql->pRpcCtx != NULL);
+      /*
+       * There is multi-thread problem here, since pSql->pRpcCtx may have been
+       * reset and freed in the processMsgFromServer function, and causes the invalid
+       * write problem for rpcCancelRequest.
+       */
+      if (pSql->pRpcCtx != NULL) {
         rpcCancelRequest(pSql->pRpcCtx);
+        pSql->pRpcCtx = NULL;
+      }
+
+      tscQueueAsyncRes(pSql);
     }
   }
 
@@ -831,6 +897,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
   pSql->fp = asyncCallback;
   pSql->fetchFp = asyncCallback;
   pSql->param = pSql;
+
+  registerSqlObj(pSql);
   int code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     tsem_wait(&pSql->rspSem);

src/client/src/tscStream.c

@@ -274,7 +274,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
     taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
     tscFreeSqlResult(pSql);
     taosTFree(pSql->pSubs);
-    pSql->numOfSubs = 0;
+    pSql->subState.numOfSub = 0;
     taosTFree(pTableMetaInfo->vgroupList);
     tscSetNextLaunchTimer(pStream, pSql);
   }

src/client/src/tscSubquery.c

@@ -26,7 +26,6 @@
 #include "tscSubquery.h"
 
 typedef struct SInsertSupporter {
-  SSubqueryState* pState;
   SSqlObj*  pSql;
   int32_t   index;
 } SInsertSupporter;
@@ -167,14 +166,13 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
 }
 
 // todo handle failed to create sub query
-SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index) {
+SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) {
   SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter));
   if (pSupporter == NULL) {
     return NULL;
   }
 
   pSupporter->pObj = pSql;
-  pSupporter->pState = pState;
 
   pSupporter->subqueryIndex = index;
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
@@ -250,7 +248,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
   SJoinSupporter* pSupporter = NULL;
   
   //If the columns are not involved in the final select clause, the corresponding query will not be issued.
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     pSupporter = pSql->pSubs[i]->param;
     if (taosArrayGetSize(pSupporter->exprList) > 0) {
       ++numOfSub;
@@ -260,16 +258,15 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
   assert(numOfSub > 0);
   
   // scan all subquery, if one sub query has only ts, ignore it
-  tscDebug("%p start to launch secondary subqueries, total:%d, only:%d needs to query", pSql, pSql->numOfSubs, numOfSub);
+  tscDebug("%p start to launch secondary subqueries, %d out of %d needs to query", pSql, numOfSub, pSql->subState.numOfSub);
 
   //the subqueries that do not actually launch the secondary query to virtual node is set as completed.
-  SSubqueryState* pState = pSupporter->pState;
+  SSubqueryState* pState = &pSql->subState;
-  pState->numOfTotal = pSql->numOfSubs;
   pState->numOfRemain = numOfSub;
 
   bool success = true;
   
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     SSqlObj *pPrevSub = pSql->pSubs[i];
     pSql->pSubs[i] = NULL;
     
@@ -322,7 +319,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
   
     SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
-    assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
+    assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
   
     tscFieldInfoUpdateOffset(pNewQueryInfo);
   
@@ -373,13 +370,13 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
   if (!success) {
     pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql,
-        pSql->numOfSubs, pSql->res.code);
+        pSql->subState.numOfSub, pSql->res.code);
     freeJoinSubqueryObj(pSql);
     
     return pSql->res.code;
   }
   
-  for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     if (pSql->pSubs[i] == NULL) {
       continue;
     }
@@ -391,17 +388,13 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
 }
 
 void freeJoinSubqueryObj(SSqlObj* pSql) {
-  SSubqueryState* pState = NULL;
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
-
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
     SSqlObj* pSub = pSql->pSubs[i];
     if (pSub == NULL) {
       continue;
     }
     
     SJoinSupporter* p = pSub->param;
-    pState = p->pState;
-
     tscDestroyJoinSupporter(p);
 
     if (pSub->res.code == TSDB_CODE_SUCCESS) {
@@ -409,14 +402,13 @@ void freeJoinSubqueryObj(SSqlObj* pSql) {
     }
   }
 
-  taosTFree(pState);
+  pSql->subState.numOfSub = 0;
-  pSql->numOfSubs = 0;
 }
 
 static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
-  assert(pSupporter->pState->numOfRemain > 0);
+  assert(pSqlObj->subState.numOfRemain > 0);
 
-  if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) <= 0) {
+  if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
     tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
     freeJoinSubqueryObj(pSqlObj);
   }
@@ -680,7 +672,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
 
   // no data exists in next vnode, mark the <tid, tags> query completed
   // only when there is no subquery exits any more, proceeds to get the intersect of the <tid, tags> tuple sets.
-  if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
     return;
   }
 
@@ -716,10 +708,10 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
     STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
     tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);
 
-    pSupporter->pState->numOfTotal = 2;
+    pParentSql->subState.numOfSub = 2;
-    pSupporter->pState->numOfRemain = pSupporter->pState->numOfTotal;
+    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;
 
-    for (int32_t m = 0; m < pParentSql->numOfSubs; ++m) {
+    for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) {
       SSqlObj* sub = pParentSql->pSubs[m];
       issueTSCompQuery(sub, sub->param, pParentSql);
     }
@@ -818,7 +810,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
     return;
   }
 
-  if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
     return;
   }
 
@@ -850,7 +842,6 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
   SJoinSupporter* pSupporter = (SJoinSupporter*)param;
 
   SSqlObj* pParentSql = pSupporter->pObj;
-  SSubqueryState* pState = pSupporter->pState;
 
   SSqlObj* pSql = (SSqlObj*)tres;
   SSqlCmd* pCmd = &pSql->cmd;
@@ -871,6 +862,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
     pRes->numOfTotal += pRes->numOfRows;
   }
 
+  SSubqueryState* pState = &pParentSql->subState;
   if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) {
     STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
     assert(pQueryInfo->numOfTables == 1);
@@ -878,7 +870,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
     // for projection query, need to try next vnode if current vnode is exhausted
     if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) {
       pState->numOfRemain = 1;
-      pState->numOfTotal = 1;
+      pState->numOfSub = 1;
 
       pSql->cmd.command = TSDB_SQL_SELECT;
       pSql->fp = tscJoinQueryCallback;
@@ -888,12 +880,12 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
     }
   }
 
-  if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
+  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
-    tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfTotal);
+    tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pParentSql->subState.numOfRemain, pState->numOfSub);
     return;
   }
 
-  tscDebug("%p all %d secondary subqueries retrieval completed, code:%d", tres, pState->numOfTotal, pParentSql->res.code);
+  tscDebug("%p all %d secondary subqueries retrieval completed, code:%d", tres, pState->numOfSub, pParentSql->res.code);
 
   if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
     freeJoinSubqueryObj(pParentSql);
@@ -901,7 +893,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
   }
 
   // update the records for each subquery in parent sql object.
-  for (int32_t i = 0; i < pParentSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pState->numOfSub; ++i) {
     if (pParentSql->pSubs[i] == NULL) {
       continue;
     }
@@ -917,32 +909,26 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
 static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) {
   int32_t notInvolved = 0;
   SJoinSupporter* pSupporter = NULL;
-  SSubqueryState* pState = NULL;
+  SSubqueryState* pState = &pSql->subState;
   
-  for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     if (pSql->pSubs[i] == NULL) {
       notInvolved++;
     } else {
       pSupporter = (SJoinSupporter*)pSql->pSubs[i]->param;
-      pState = pSupporter->pState;
     }
   }
   
-  assert(pState != NULL);
-  if (pState != NULL) {
-    pState->numOfTotal = pSql->numOfSubs;
   pState->numOfRemain = numOfFetch;
-  }
-  
   return pSupporter;
 }
 
 void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
-  assert(pSql->numOfSubs >= 1);
+  assert(pSql->subState.numOfSub >= 1);
   
   int32_t numOfFetch = 0;
   bool hasData = true;
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     // if the subquery is NULL, it does not involved in the final result generation
     SSqlObj* pSub = pSql->pSubs[i];
     if (pSub == NULL) {
@@ -989,7 +975,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
   tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch);
   SJoinSupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch);
   
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     SSqlObj* pSql1 = pSql->pSubs[i];
     if (pSql1 == NULL) {
       continue;
@@ -1124,7 +1110,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
   }
 
   // wait for the other subqueries response from vnode
-  if (atomic_sub_fetch_32(&pSupporter->pState->numOfRemain, 1) > 0) {
+  if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
     return;
   }
 
@@ -1136,7 +1122,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
    * data instead of returning to its invoker
    */
   if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-    pSupporter->pState->numOfRemain = pSupporter->pState->numOfTotal;  // reset the record value
+    pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;  // reset the record value
 
     pSql->fp = joinRetrieveFinalResCallback;  // continue retrieve data
     pSql->cmd.command = TSDB_SQL_FETCH;
@@ -1165,7 +1151,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
   assert(pSql->res.numOfRows == 0);
 
   if (pSql->pSubs == NULL) {
-    pSql->pSubs = calloc(pSupporter->pState->numOfTotal, POINTER_BYTES);
+    pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES);
     if (pSql->pSubs == NULL) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
@@ -1176,8 +1162,8 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
   
-  pSql->pSubs[pSql->numOfSubs++] = pNew;
+  pSql->pSubs[pSql->subState.numOfRemain++] = pNew;
-  assert(pSql->numOfSubs <= pSupporter->pState->numOfTotal);
+  assert(pSql->subState.numOfRemain <= pSql->subState.numOfSub);
   
   if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) {
     addGroupInfoForSubquery(pSql, pNew, 0, tableIndex);
@@ -1221,7 +1207,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
     
     if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
-      SColumnIndex index = {0};
+      SColumnIndex colIndex = {0};
 
       STagCond* pTagCond = &pSupporter->tagCond;
       assert(pTagCond->joinInfo.hasJoin);
@@ -1234,7 +1220,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
       for(int32_t i = 0; i < numOfTags; ++i) {
         if (pSchema[i].colId == tagColId) {
-          index.columnIndex = i;
+          colIndex.columnIndex = i;
           break;
         }
       }
@@ -1251,18 +1237,18 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
 
       // set get tags query type
       TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
-      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s1, TSDB_COL_TAG);
+      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG);
       size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
   
       tscDebug(
           "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), "
           "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s",
           pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo),
-          numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, index.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name);
+          numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name);
     } else {
       SSchema      colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1};
-      SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
+      SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
-      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL);
+      tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL);
 
       // set the tags value for ts_comp function
       SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0);
@@ -1314,20 +1300,19 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   // todo add test
-  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
+//  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
-  if (pState == NULL) {
+//  if (pState == NULL) {
-    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+//    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    goto _error;
+//    goto _error;
-  }
+//  }
 
-  pState->numOfTotal = pQueryInfo->numOfTables;
+  pSql->subState.numOfSub = pQueryInfo->numOfTables;
-  pState->numOfRemain = pState->numOfTotal;
 
   bool hasEmptySub = false;
 
   tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-    SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i);
+    SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i);
     
     if (pSupporter == NULL) {  // failed to create support struct, abort current query
       tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i);
@@ -1354,10 +1339,10 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
     pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
     (*pSql->fp)(pSql->param, pSql, 0);
   } else {
-    for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+    for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
       SSqlObj* pSub = pSql->pSubs[i];
       if ((code = tscProcessSql(pSub)) != TSDB_CODE_SUCCESS) {
-        pState->numOfRemain = i - 1;  // the already sent reques will continue and do not go to the error process routine
+        pSql->subState.numOfRemain = i - 1;  // the already sent request will continue and do not go to the error process routine
         break;
       }
     }
@@ -1372,8 +1357,8 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
   tscQueueAsyncRes(pSql);
 }
 
-static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState* pState) {
+static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
-  assert(numOfSubs <= pSql->numOfSubs && numOfSubs >= 0 && pState != NULL);
+  assert(numOfSubs <= pSql->subState.numOfSub && numOfSubs >= 0);
   
   for(int32_t i = 0; i < numOfSubs; ++i) {
     SSqlObj* pSub = pSql->pSubs[i];
@@ -1386,8 +1371,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState
     
     taos_free_result(pSub);
   }
-  
-  free(pState);
 }
 
 int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
@@ -1410,9 +1393,10 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   
   SQueryInfo     *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  SSubqueryState *pState = &pSql->subState;
 
-  pSql->numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
+  pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups;
-  assert(pSql->numOfSubs > 0);
+  assert(pState->numOfSub > 0);
   
   int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
   if (ret != 0) {
@@ -1422,28 +1406,24 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     return ret;
   }
   
-  pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES);
+  pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES);
 
-  tscDebug("%p retrieved query data from %d vnode(s)", pSql, pSql->numOfSubs);
+  tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);
-  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
 
-  if (pSql->pSubs == NULL || pState == NULL) {
+  if (pSql->pSubs == NULL) {
-    taosTFree(pState);
     taosTFree(pSql->pSubs);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs);
+    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
 
     tscQueueAsyncRes(pSql);
     return ret;
   }
 
-  pState->numOfTotal = pSql->numOfSubs;
+  pState->numOfRemain = pState->numOfSub;
-  pState->numOfRemain = pSql->numOfSubs;
-
   pRes->code = TSDB_CODE_SUCCESS;
   
   int32_t i = 0;
-  for (; i < pSql->numOfSubs; ++i) {
+  for (; i < pState->numOfSub; ++i) {
     SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport));
     if (trs == NULL) {
       tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
@@ -1452,7 +1432,6 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     
     trs->pExtMemBuffer = pMemoryBuf;
     trs->pOrderDescriptor = pDesc;
-    trs->pState = pState;
 
     trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage));
     if (trs->localBuffer == NULL) {
@@ -1483,22 +1462,22 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     tscDebug("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex);
   }
   
-  if (i < pSql->numOfSubs) {
+  if (i < pState->numOfSub) {
     tscError("%p failed to prepare subquery structure and launch subqueries", pSql);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs);
+    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
-    doCleanupSubqueries(pSql, i, pState);
+    doCleanupSubqueries(pSql, i);
     return pRes->code;   // free all allocated resource
   }
   
   if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) {
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs);
+    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
-    doCleanupSubqueries(pSql, i, pState);
+    doCleanupSubqueries(pSql, i);
     return pRes->code;
   }
   
-  for(int32_t j = 0; j < pSql->numOfSubs; ++j) {
+  for(int32_t j = 0; j < pState->numOfSub; ++j) {
     SSqlObj* pSub = pSql->pSubs[j];
     SRetrieveSupport* pSupport = pSub->param;
     
@@ -1509,16 +1488,20 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
+static void tscFreeRetrieveSup(SSqlObj *pSql) {
-  tscDebug("%p start to free subquery obj", pSql);
+  SRetrieveSupport *trsupport = pSql->param;
 
-  int32_t  index = trsupport->subqueryIndex;
+  void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0);
-  SSqlObj *pParentSql = trsupport->pParentSql;
+  if (p == NULL) {
+    tscDebug("%p retrieve supp already released", pSql);
+    return;
+  }
 
-  assert(pSql == pParentSql->pSubs[index]);
+  tscDebug("%p start to free subquery supp obj:%p", pSql, trsupport);
-//  pParentSql->pSubs[index] = NULL;
+//  int32_t  index = trsupport->subqueryIndex;
-//
+//  SSqlObj *pParentSql = trsupport->pParentSql;
-//  taos_free_result(pSql);
+
+//  assert(pSql == pParentSql->pSubs[index]);
   taosTFree(trsupport->localBuffer);
   taosTFree(trsupport);
 }
@@ -1577,12 +1560,18 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 }
 
 void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
+  // it has been freed already
+  if (pSql->param != trsupport || pSql->param == NULL) {
+    return;
+  }
+
   SSqlObj *pParentSql = trsupport->pParentSql;
   int32_t  subqueryIndex = trsupport->subqueryIndex;
   
   assert(pSql != NULL);
-  SSubqueryState* pState = trsupport->pState;
+
-  assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
+  SSubqueryState* pState = &pParentSql->subState;
+  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
 
   // retrieved in subquery failed. OR query cancelled in retrieve phase.
   if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) {
@@ -1616,22 +1605,21 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   int32_t remain = -1;
   if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
     tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
-        pState->numOfTotal - remain);
+        pState->numOfSub - remain);
 
-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
     return;
   }
   
   // all subqueries are failed
-  tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfTotal,
+  tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfSub,
       tstrerror(pParentSql->res.code));
 
   // release allocated resource
   tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel,
-                            pState->numOfTotal);
+                            pState->numOfSub);
   
-  taosTFree(trsupport->pState);
+  tscFreeRetrieveSup(pSql);
-  tscFreeSubSqlObj(trsupport, pSql);
 
   // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
@@ -1650,7 +1638,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   SSqlObj *         pParentSql = trsupport->pParentSql;
   tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
   
-  SSubqueryState* pState = trsupport->pState;
+  SSubqueryState* pState = &pParentSql->subState;
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
@@ -1687,11 +1675,11 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   }
   
   int32_t remain = -1;
-  if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) {
+  if ((remain = atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1)) > 0) {
     tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex,
-        pState->numOfTotal - remain);
+        pState->numOfSub - remain);
 
-    tscFreeSubSqlObj(trsupport, pSql);
+    tscFreeRetrieveSup(pSql);
     return;
   }
   
@@ -1699,21 +1687,19 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
   
   tscDebug("%p retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", pParentSql,
-           pState->numOfTotal, pState->numOfRetrievedRows);
+           pState->numOfSub, pState->numOfRetrievedRows);
   
   SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
   tscClearInterpInfo(pPQueryInfo);
   
-  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pParentSql);
+  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, pParentSql);
   tscDebug("%p build loser tree completed", pParentSql);
   
   pParentSql->res.precision = pSql->res.precision;
   pParentSql->res.numOfRows = 0;
   pParentSql->res.row = 0;
   
-  // only free once
+  tscFreeRetrieveSup(pSql);
-  taosTFree(trsupport->pState);
-  tscFreeSubSqlObj(trsupport, pSql);
 
   // set the command flag must be after the semaphore been correctly set.
   pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE;
@@ -1725,16 +1711,23 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
 }
 
 static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
+  SSqlObj *pSql = (SSqlObj *)tres;
+  assert(pSql != NULL);
+
+  // this query has been freed already
   SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
+  if (pSql->param == NULL || param == NULL) {
+    tscDebug("%p already freed in dnodecallback", pSql);
+    assert(pSql->res.code == TSDB_CODE_TSC_QUERY_CANCELLED);
+    return;
+  }
+
   tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
   int32_t           idx   = trsupport->subqueryIndex;
   SSqlObj *         pParentSql = trsupport->pParentSql;
 
-  assert(tres != NULL);
+  SSubqueryState* pState = &pParentSql->subState;
-  SSqlObj *pSql = (SSqlObj *)tres;
+  assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
-
-  SSubqueryState* pState = trsupport->pState;
-  assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
   
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
   SCMVgroupInfo  *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
@@ -1751,6 +1744,10 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
   if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {
     assert(numOfRows == taos_errno(pSql));
 
+    if (numOfRows == TSDB_CODE_TSC_QUERY_CANCELLED) {
+      trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
+    }
+
     if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) {
       tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry);
 
@@ -1822,7 +1819,7 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo
     SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
 
     pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY;
-    assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs);
+    assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->subState.numOfSub);
     
     // launch subquery for each vnode, so the subquery index equals to the vgroupIndex.
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index);
@@ -1893,7 +1890,6 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
 static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) {
   SInsertSupporter *pSupporter = (SInsertSupporter *)param;
   SSqlObj* pParentObj = pSupporter->pSql;
-  SSubqueryState* pState = pSupporter->pState;
 
   // record the total inserted rows
   if (numOfRows > 0) {
@@ -1908,15 +1904,13 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
   }
 
   taosTFree(pSupporter);
-  if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
+
+  if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
     return;
   }
   
   tscDebug("%p Async insertion completed, total inserted:%" PRId64, pParentObj, pParentObj->res.numOfRows);
 
-  // release data block data
-  taosTFree(pState);
-
   // restore user defined fp
   pParentObj->fp = pParentObj->fetchFp;
 
@@ -1937,7 +1931,7 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
   SSqlRes* pRes = &pSql->res;
 
   SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param;
-  assert(pSupporter->index < pSupporter->pState->numOfTotal);
+  assert(pSupporter->index < pSupporter->pSql->subState.numOfSub);
 
   STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
   int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
@@ -1954,33 +1948,29 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
-  pSql->numOfSubs = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks);
+  pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks);
-  assert(pSql->numOfSubs > 0);
+  assert(pSql->subState.numOfSub > 0);
 
   pRes->code = TSDB_CODE_SUCCESS;
 
   // the number of already initialized subqueries
   int32_t numOfSub = 0;
 
-  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
+  pSql->subState.numOfRemain = pSql->subState.numOfSub;
-  pState->numOfTotal = pSql->numOfSubs;
+  pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES);
-  pState->numOfRemain = pSql->numOfSubs;
-
-  pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES);
   if (pSql->pSubs == NULL) {
     goto _error;
   }
 
-  tscDebug("%p submit data to %d vnode(s)", pSql, pSql->numOfSubs);
+  tscDebug("%p submit data to %d vnode(s)", pSql, pSql->subState.numOfSub);
 
-  while(numOfSub < pSql->numOfSubs) {
+  while(numOfSub < pSql->subState.numOfSub) {
     SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
     if (pSupporter == NULL) {
       goto _error;
     }
 
     pSupporter->pSql   = pSql;
-    pSupporter->pState = pState;
     pSupporter->index  = numOfSub;
 
     SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT);
@@ -2003,12 +1993,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
       numOfSub++;
     } else {
       tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
-               pSql->numOfSubs, tstrerror(pRes->code));
+               pSql->subState.numOfSub, tstrerror(pRes->code));
       goto _error;
     }
   }
   
-  if (numOfSub < pSql->numOfSubs) {
+  if (numOfSub < pSql->subState.numOfSub) {
     tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     goto _error;
@@ -2026,7 +2016,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 
   _error:
-  taosTFree(pState);
   return TSDB_CODE_TSC_OUT_OF_MEMORY;
 }
 
@@ -2048,7 +2037,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
 
   int32_t numOfRes = INT32_MAX;
-  for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     if (pSql->pSubs[i] == NULL) {
       continue;
     }
@@ -2238,7 +2227,7 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
   if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
     bool allSubqueryExhausted = true;
     
-    for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+    for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
       if (pSql->pSubs[i] == NULL) {
         continue;
       }
@@ -2264,7 +2253,7 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
     
     hasData = !allSubqueryExhausted;
   } else {  // otherwise, in case inner join, if any subquery exhausted, query completed.
-    for (int32_t i = 0; i < pSql->numOfSubs; ++i) {
+    for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
       if (pSql->pSubs[i] == 0) {
         continue;
       }

src/client/src/tscSystem.c

@@ -141,7 +141,7 @@ void taos_init_imp(void) {
   int64_t refreshTime = 10; // 10 seconds by default
   if (tscMetaCache == NULL) {
     tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeSqlObjInCache, "sqlObj");
+    tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
   }
 
   tscDebug("client is initialized successfully");

src/client/src/tscUtil.c

@@ -360,26 +360,26 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   
   taosTFree(pSql->pSubs);
-  pSql->numOfSubs = 0;
+  pSql->subState.numOfSub = 0;
   pSql->self = 0;
   
   tscResetSqlCmdObj(pCmd, false);
 }
 
-static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
+static void tscFreeSubobj(SSqlObj* pSql) {
-  if (pSql->numOfSubs == 0) {
+  if (pSql->subState.numOfSub == 0) {
     return;
   }
 
-  tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs);
+  tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->subState.numOfSub);
 
-  for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
+  for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
     tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i);
     taos_free_result(pSql->pSubs[i]);
     pSql->pSubs[i] = NULL;
   }
 
-  pSql->numOfSubs = 0;
+  pSql->subState.numOfSub = 0;
 }
 
 /**
@@ -389,7 +389,7 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
  *
  * @param pSql
  */
-void tscFreeSqlObjInCache(void *pSql) {
+void tscFreeRegisteredSqlObj(void *pSql) {
   assert(pSql != NULL);
 
   SSqlObj** p = (SSqlObj**)pSql;
@@ -415,7 +415,9 @@ void tscFreeSqlObj(SSqlObj* pSql) {
 
   tscDebug("%p start to free sqlObj", pSql);
 
+  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
   tscFreeSubobj(pSql);
+
   tscPartiallyFreeSqlObj(pSql);
 
   pSql->signature = NULL;
@@ -1516,13 +1518,6 @@ void tscSetFreeHeatBeat(STscObj* pObj) {
   pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
 }
 
-bool tscShouldFreeHeartBeat(SSqlObj* pHb) {
-  assert(pHb == pHb->signature);
-
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHb->cmd, 0);
-  return pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE;
-}
-
 /*
  * the following four kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
@@ -2291,7 +2286,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
      *
      * For super table join with projection query, if anyone of the subquery is exhausted, the query completed.
      */
-    pSql->numOfSubs = 0;
+    pSql->subState.numOfSub = 0;
     pCmd->command = TSDB_SQL_SELECT;
 
     tscResetForNextRetrieve(pRes);
@@ -2323,7 +2318,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
   pRes->numOfTotal = num;
   
   taosTFree(pSql->pSubs);
-  pSql->numOfSubs = 0;
+  pSql->subState.numOfSub = 0;
   pSql->fp = fp;
 
   tscDebug("%p try data in the next subclause:%d, total subclause:%d", pSql, pCmd->clauseIndex, pCmd->numOfClause);

src/common/inc/tglobal.h

@@ -34,6 +34,7 @@ extern int32_t  tsStatusInterval;
 extern int32_t  tsNumOfMnodes;
 extern int32_t  tsEnableVnodeBak;
 extern int32_t  tsEnableTelemetryReporting;
+extern char     tsEmail[];
 
 // common
 extern int      tsRpcTimer;

src/common/src/tglobal.c

@@ -42,6 +42,7 @@ int32_t  tsStatusInterval = 1;  // second
 int32_t  tsNumOfMnodes = 3;
 int32_t  tsEnableVnodeBak = 1;
 int32_t  tsEnableTelemetryReporting = 1;
+char     tsEmail[TSDB_FQDN_LEN] = {0};
 
 // common
 int32_t tsRpcTimer = 1000;
@@ -307,6 +308,8 @@ bool taosCfgDynamicOptions(char *msg) {
 
 static void doInitGlobalConfig(void) {
   osInit();
+  srand(taosSafeRand());
+
   SGlobalCfg cfg = {0};
   
   // ip address

src/connector/go

@@ -1 +1 @@
-Subproject commit 567b7b12f3fd2775c718d284beffc8c38dd6c219
+Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766

src/dnode/CMakeLists.txt

@@ -4,6 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
 
 INCLUDE_DIRECTORIES(inc)

src/dnode/src/dnodeMPeer.c

@@ -33,7 +33,8 @@ typedef struct {
 } SMPeerWorker;
 
 typedef struct {
-  int32_t       num;
+  int32_t curNum;
+  int32_t maxNum;
   SMPeerWorker *peerWorker;
 } SMPeerWorkerPool;
 
@@ -46,37 +47,44 @@ static void *dnodeProcessMnodePeerQueue(void *param);
 int32_t dnodeInitMnodePeer() {
   tsMPeerQset = taosOpenQset();
   
-  tsMPeerPool.num = 1;
+  tsMPeerPool.maxNum = 1;
-  tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.num);
+  tsMPeerPool.curNum = 0;
+  tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
 
   if (tsMPeerPool.peerWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
     SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
     pWorker->workerId = i;
+    dDebug("dnode mpeer worker:%d is created", i);
   }
 
-  dInfo("dnode mpeer is opened");
+  dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
   return 0;
 }
 
 void dnodeCleanupMnodePeer() {
-  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
     SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMPeerQset);
     }
+    dDebug("dnode mpeer worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
     SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+    dDebug("dnode mpeer worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
     }
+    dDebug("dnode mpeer worker:%d join success", i);
   }
 
+  dDebug("dnode mpeer is closed, qset:%p", tsMPeerQset);
+
   taosCloseQset(tsMPeerQset);
+  tsMPeerQset = NULL;
   taosTFree(tsMPeerPool.peerWorker);
-  dInfo("dnode mpeer is closed");
 }
 
 int32_t dnodeAllocateMnodePqueue() {
@@ -85,7 +93,7 @@ int32_t dnodeAllocateMnodePqueue() {
 
   taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
 
-  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+  for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
     SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
     pWorker->workerId = i;
 
@@ -98,7 +106,9 @@ int32_t dnodeAllocateMnodePqueue() {
     }
 
     pthread_attr_destroy(&thAttr);
-    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.num);
+
+    tsMPeerPool.curNum = i + 1;
+    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
   }
 
   dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
@@ -106,6 +116,7 @@ int32_t dnodeAllocateMnodePqueue() {
 }
 
 void dnodeFreeMnodePqueue() {
+  dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
   taosCloseQueue(tsMPeerQueue);
   tsMPeerQueue = NULL;
 }
@@ -148,7 +159,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {
   
   while (1) {
     if (taosReadQitemFromQset(tsMPeerQset, &type, (void **)&pPeerMsg, &unUsed) == 0) {
-      dDebug("dnodeProcessMnodePeerQueue: got no message from qset, exiting...");
+      dDebug("qset:%p, mnode peer got no message from qset, exiting", tsMPeerQset);
       break;
     }
 

src/dnode/src/dnodeMRead.c

@@ -33,7 +33,8 @@ typedef struct {
 } SMReadWorker;
 
 typedef struct {
-  int32_t       num;
+  int32_t curNum;
+  int32_t maxNum;
   SMReadWorker *readWorker;
 } SMReadWorkerPool;
 
@@ -46,40 +47,46 @@ static void *dnodeProcessMnodeReadQueue(void *param);
 int32_t dnodeInitMnodeRead() {
   tsMReadQset = taosOpenQset();
 
-  tsMReadPool.num = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+  tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
-  tsMReadPool.num = MAX(2, tsMReadPool.num);
+  tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
-  tsMReadPool.num = MIN(4, tsMReadPool.num);
+  tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
-  tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.num);
+  tsMReadPool.curNum = 0;
+  tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
 
   if (tsMReadPool.readWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
     SMReadWorker *pWorker = tsMReadPool.readWorker + i;
     pWorker->workerId = i;
+    dDebug("dnode mread worker:%d is created", i);
   }
 
-  dInfo("dnode mread is opened");
+  dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
   return 0;
 }
 
 void dnodeCleanupMnodeRead() {
-  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
     SMReadWorker *pWorker = tsMReadPool.readWorker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMReadQset);
     }
+    dDebug("dnode mread worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
     SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+    dDebug("dnode mread worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
     }
+    dDebug("dnode mread worker:%d start to join", i);
   }
 
-  taosCloseQset(tsMReadQset);
+  dDebug("dnode mread is closed, qset:%p", tsMReadQset);
-  free(tsMReadPool.readWorker);
 
-  dInfo("dnode mread is closed");
+  taosCloseQset(tsMReadQset);
+  tsMReadQset = NULL;
+  free(tsMReadPool.readWorker);
 }
 
 int32_t dnodeAllocateMnodeRqueue() {
@@ -88,7 +95,7 @@ int32_t dnodeAllocateMnodeRqueue() {
 
   taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
 
-  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+  for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
     SMReadWorker *pWorker = tsMReadPool.readWorker + i;
     pWorker->workerId = i;
 
@@ -101,7 +108,8 @@ int32_t dnodeAllocateMnodeRqueue() {
     }
 
     pthread_attr_destroy(&thAttr);
-    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.num);
+    tsMReadPool.curNum = i + 1;
+    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
   }
 
   dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
@@ -109,6 +117,7 @@ int32_t dnodeAllocateMnodeRqueue() {
 }
 
 void dnodeFreeMnodeRqueue() {
+  dDebug("dnode mread queue:%p is freed", tsMReadQueue);
   taosCloseQueue(tsMReadQueue);
   tsMReadQueue = NULL;
 }
@@ -156,7 +165,7 @@ static void *dnodeProcessMnodeReadQueue(void *param) {
   
   while (1) {
     if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
-      dDebug("dnodeProcessMnodeReadQueue: got no message from qset, exiting...");
+      dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
       break;
     }
 

src/dnode/src/dnodeMWrite.c

@@ -34,7 +34,8 @@ typedef struct {
 } SMWriteWorker;
 
 typedef struct {
-  int32_t        num;
+  int32_t curNum;
+  int32_t maxNum;
   SMWriteWorker *writeWorker;
 } SMWriteWorkerPool;
 
@@ -48,37 +49,44 @@ static void *dnodeProcessMnodeWriteQueue(void *param);
 int32_t dnodeInitMnodeWrite() {
   tsMWriteQset = taosOpenQset();
 
-  tsMWritePool.num = 1;
+  tsMWritePool.maxNum = 1;
-  tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.num);
+  tsMWritePool.curNum = 0;
+  tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
 
   if (tsMWritePool.writeWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
     SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
     pWorker->workerId = i;
+    dDebug("dnode mwrite worker:%d is created", i);
   }
 
-  dInfo("dnode mwrite is opened");
+  dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
   return 0;
 }
 
 void dnodeCleanupMnodeWrite() {
-  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
     SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMWriteQset);
     }
+    dDebug("dnode mwrite worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
     SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+    dDebug("dnode mwrite worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
     }
+    dDebug("dnode mwrite worker:%d join success", i);
   }
 
+  dDebug("dnode mwrite is closed, qset:%p", tsMWriteQset);
+
   taosCloseQset(tsMWriteQset);
+  tsMWriteQset = NULL;
   taosTFree(tsMWritePool.writeWorker);
-  dInfo("dnode mwrite is closed");
 }
 
 int32_t dnodeAllocateMnodeWqueue() {
@@ -87,7 +95,7 @@ int32_t dnodeAllocateMnodeWqueue() {
 
   taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
 
-  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+  for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
     SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
     pWorker->workerId = i;
 
@@ -100,7 +108,8 @@ int32_t dnodeAllocateMnodeWqueue() {
     }
 
     pthread_attr_destroy(&thAttr);
-    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.num);
+    tsMWritePool.curNum = i + 1;
+    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
   }
 
   dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
@@ -108,6 +117,7 @@ int32_t dnodeAllocateMnodeWqueue() {
 }
 
 void dnodeFreeMnodeWqueue() {
+  dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
   taosCloseQueue(tsMWriteQueue);
   tsMWriteQueue = NULL;
 }
@@ -122,11 +132,15 @@ void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
   SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
   mnodeCreateMsg(pWrite, pMsg);
 
-  dDebug("app:%p:%p, msg:%s is put into mwrite queue", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType]);
+  dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+         taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
   taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
 }
 
 static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+  dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+         taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
+
   mnodeCleanupMsg(pWrite);
   taosFreeQitem(pWrite);
 }
@@ -158,7 +172,7 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
   
   while (1) {
     if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) {
-      dDebug("dnodeProcessMnodeWriteQueue: got no message from qset, exiting...");
+      dDebug("qset:%p, mnode write got no message from qset, exiting", tsMWriteQset);
       break;
     }
 
@@ -182,8 +196,8 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
     dnodeSendRedirectMsg(pMsg, true);
     dnodeFreeMnodeWriteMsg(pWrite);
   } else {
-    dDebug("app:%p:%p, msg:%s is reput into mwrite queue, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+    dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
-           taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
+           taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
 
     taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
   }

src/dnode/src/dnodeMgmt.c

@@ -74,6 +74,7 @@ static int32_t  dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
 static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
 static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
 
 int32_t dnodeInitMgmt() {
@@ -82,6 +83,7 @@ int32_t dnodeInitMgmt() {
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeProcessDropVnodeMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
 
   dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
   dnodeReadDnodeCfg();
@@ -226,7 +228,7 @@ static void *dnodeProcessMgmtQueue(void *param) {
 
   while (1) {
     if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
-      dDebug("dnode mgmt got no message from qset, exit ...");
+      dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
       break;
     }
 
@@ -451,10 +453,34 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
 }
 
 static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
-  SMDCfgDnodeMsg *pCfg = (SMDCfgDnodeMsg *)pMsg->pCont;
+  SMDCfgDnodeMsg *pCfg = pMsg->pCont;
   return taosCfgDynamicOptions(pCfg->config);
 }
 
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
+  SMDCreateMnodeMsg *pCfg = pMsg->pCont;
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
+  if (pCfg->dnodeId != dnodeGetDnodeId()) {
+    dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
+    return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
+  }
+
+  if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
+    dError("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
+    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
+  }
+
+  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum);
+  for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) {
+    pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId);
+    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp);
+  }
+
+  dnodeStartMnode(&pCfg->mnodes);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
   if (pEpSet->numOfEps <= 0) {
     dError("mnode EP list for peer is changed, but content is invalid, discard it");
@@ -465,13 +491,6 @@ void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
   for (int i = 0; i < pEpSet->numOfEps; ++i) {
     pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
     dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
-
-    if (!mnodeIsRunning()) {
-      if (strcmp(pEpSet->fqdn[i], tsLocalFqdn) == 0 && pEpSet->port[i] == tsServerPort) {
-        dInfo("mnode index:%d %s:%u should work as master", i, pEpSet->fqdn[i], pEpSet->port[i]);
-        sdbUpdateSync();
-      }
-    }
   }
 
   tsDMnodeEpSet = *pEpSet;
@@ -516,7 +535,9 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   }
 
   vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
-  dnodeProcessModuleStatus(pCfg->moduleStatus);
+
+  // will not set mnode in status msg
+  // dnodeProcessModuleStatus(pCfg->moduleStatus);
   dnodeUpdateDnodeCfg(pCfg);
 
   dnodeUpdateMnodeInfos(pMnodes);
@@ -560,7 +581,7 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
   }
 
   dnodeSaveMnodeInfos();
-  sdbUpdateSync();
+  sdbUpdateAsync();
 }
 
 static bool dnodeReadMnodeInfos() {

src/dnode/src/dnodeModule.c

@@ -146,19 +146,19 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
   }
 }
 
-bool dnodeCheckMnodeStarting() {
+bool dnodeStartMnode(void *pMnodes) {
-  if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) return false;
+  SDMMnodeInfos *mnodes = pMnodes;
+
+  if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
+    dDebug("mnode module is already started, module status:%d", tsModuleStatus);
+    return false;
+  }
 
-  SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
-  for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
-    SDMMnodeInfo *node = &mnodes->nodeInfos[i];
-    if (node->nodeId == dnodeGetDnodeId()) {
   uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);
   dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
   dnodeProcessModuleStatus(moduleStatus);
-      return true;
-    }
-  }
 
-  return false;
+  sdbUpdateSync(mnodes);
+
+  return true;
 }

src/dnode/src/dnodePeer.c

@@ -48,6 +48,7 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToMgmtQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
 
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
@@ -170,8 +171,12 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
   rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
 }
 
-void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
+void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcEpSet epSet = {0};
   dnodeGetMnodeEpSetForPeer(&epSet);
   rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
 }
+
+void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
+  rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
+}

src/dnode/src/dnodeShell.c

@@ -156,7 +156,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
   
   dDebug("user:%s, send auth msg to mnodes", user);
   SRpcMsg rpcRsp = {0};
-  dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp);
+  dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
 
   if (rpcRsp.code != 0) {
     dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
@@ -189,7 +189,7 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) {
   rpcMsg.msgType = TSDB_MSG_TYPE_DM_CONFIG_TABLE;
 
   SRpcMsg rpcRsp = {0};
-  dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp);
+  dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
   terrno = rpcRsp.code;
   
   if (rpcRsp.code != 0) {

src/dnode/src/dnodeTelemetry.c

@@ -178,6 +178,7 @@ static void addVersionInfo(SBufferWriter* bw) {
   addStringField(bw, "version", version);
   addStringField(bw, "buildInfo", buildinfo);
   addStringField(bw, "gitInfo", gitinfo);  
+  addStringField(bw, "email", tsEmail);  
 }
 
 static void addRuntimeInfo(SBufferWriter* bw) {
@@ -261,11 +262,27 @@ static void* telemetryThread(void* param) {
   return NULL;
 }
 
+static void dnodeGetEmail(char* filepath) {
+  int fd = open(filepath, O_RDONLY);
+  if (fd < 0) {
+    return;
+  }
+  
+  if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
+    dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
+  } 
+
+  close(fd);   
+}
+
+
 int32_t dnodeInitTelemetry() {
   if (!tsEnableTelemetryReporting) {
     return 0;
   }
 
+  dnodeGetEmail("/usr/local/taos/email");  
+
   if (tsem_init(&tsExitSem, 0, 0) == -1) {
     // just log the error, it is ok for telemetry to fail
     dTrace("failed to create semaphore for telemetry, reason:%s", strerror(errno));

src/dnode/src/dnodeVRead.c

@@ -187,6 +187,7 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
 }
 
 void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) {
+  rpcFreeCont(pRead->rpcMsg.pCont);
   vnodeRelease(pVnode);
   return;
 }
@@ -198,7 +199,7 @@ static void *dnodeProcessReadQueue(void *param) {
 
   while (1) {
     if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
-      dDebug("dnodeProcessReadQueee: got no message from qset, exiting...");
+      dDebug("qset:%p dnode read got no message from qset, exiting", readQset);
       break;
     }
 

src/dnode/src/dnodeVWrite.c

@@ -24,8 +24,10 @@
 #include "twal.h"
 #include "tdataformat.h"
 #include "tglobal.h"
+#include "tsync.h"
 #include "vnode.h"
 #include "dnodeInt.h"
+#include "syncInt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMgmt.h"
 
@@ -210,19 +212,19 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
 
 static void *dnodeProcessWriteQueue(void *param) {
   SWriteWorker *pWorker = (SWriteWorker *)param;
-  SWriteMsg    *pWrite;
+  SWriteMsg *   pWrite;
-  SWalHead     *pHead;
+  SWalHead *    pHead;
   int32_t       numOfMsgs;
   int           type;
-  void         *pVnode, *item;
+  void *        pVnode, *item;
-  SRspRet      *pRspRet;
+  SRspRet *     pRspRet;
 
   dDebug("write worker:%d is running", pWorker->workerId);
 
   while (1) {
     numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
     if (numOfMsgs == 0) {
-      dDebug("dnodeProcessWriteQueee: got no message from qset, exiting...");
+      dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset);
       break;
     }
 
@@ -237,13 +239,22 @@ static void *dnodeProcessWriteQueue(void *param) {
         pHead->msgType = pWrite->rpcMsg.msgType;
         pHead->version = 0;
         pHead->len = pWrite->contLen;
-        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]);
+        dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
+               taosMsg[pWrite->rpcMsg.msgType]);
+      } else if (type == TAOS_QTYPE_CQ) {
+        pHead = (SWalHead *)((char*)item + sizeof(SSyncHead));
+        dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
+               pHead->version);
       } else {
         pHead = (SWalHead *)item;
-        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version);
+        dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
+               pHead->version);
       }
 
       int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
+      dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
+             pHead->version, tstrerror(code));
+
       if (pWrite) {
         pWrite->rpcMsg.code = code;
         if (code <= 0) pWrite->processedCount = 1;

src/inc/dnode.h

@@ -43,11 +43,12 @@ void    dnodeGetMnodeEpSetForPeer(void *epSet);
 void    dnodeGetMnodeEpSetForShell(void *epSet);
 void *  dnodeGetMnodeInfos();
 int32_t dnodeGetDnodeId();
-bool    dnodeCheckMnodeStarting();
+bool    dnodeStartMnode(void *pModes);
 
 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
-void  dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
+void  dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
+void  dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);
 void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid);
 
 void *dnodeAllocateVnodeWqueue(void *pVnode);

src/inc/mnode.h

@@ -60,7 +60,8 @@ int32_t mnodeInitSystem();
 int32_t mnodeStartSystem();
 void    mnodeCleanupSystem();
 void    mnodeStopSystem();
-void    sdbUpdateSync();
+void    sdbUpdateAsync();
+void    sdbUpdateSync(void *pMnodes);
 bool    mnodeIsRunning();
 int32_t mnodeProcessRead(SMnodeMsg *pMsg);
 int32_t mnodeProcessWrite(SMnodeMsg *pMsg);

src/inc/taosdef.h

@@ -194,9 +194,9 @@ static FORCE_INLINE bool isNull(const char *val, int32_t type) {
     case TSDB_DATA_TYPE_DOUBLE:
       return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
     case TSDB_DATA_TYPE_NCHAR:
-      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
+      return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
     case TSDB_DATA_TYPE_BINARY:
-      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
+      return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
     default:
       return false;
   };
@@ -253,8 +253,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
-#define TSDB_MAX_SQL_SHOW_LEN     256
+#define TSDB_MAX_SQL_SHOW_LEN     512
-#define TSDB_MAX_ALLOWED_SQL_LEN  (1*1024*1024U)          // sql length should be less than 8mb
+#define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 8mb
+
+#define TSDB_APPNAME_LEN          TSDB_UNI_LEN
 
 #define TSDB_MAX_BYTES_PER_ROW    16384
 #define TSDB_MAX_TAGS_LEN         16384
@@ -282,7 +284,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_SHELL_VNODE_BITS     24
 #define TSDB_SHELL_SID_MASK       0xFF
 #define TSDB_HTTP_TOKEN_LEN       20
-#define TSDB_SHOW_SQL_LEN         64
+#define TSDB_SHOW_SQL_LEN         512
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
 #define TSDB_MQTT_HOSTNAME_LEN    64

src/inc/taoserror.h

@@ -139,6 +139,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  0, 0x0339, "Vgroup alr
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE,           0, 0x033A, "Dnode not avaliable")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       0, 0x033B, "Cluster id not match")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY,                0, 0x033C, "Cluster not ready")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED,  0, 0x033D, "Dnode Id not configured")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED,  0, 0x033E, "Dnode Ep not configured")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       0, 0x0340, "Account already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT,             0, 0x0341, "Invalid account")
@@ -191,6 +193,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS,      0, 0x0506, "No write p
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR,      0, 0x0507, "Missing data file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY,            0, 0x0508, "Out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR,                0, 0x0509, "Unexpected generic error in vnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE,     0, 0x050A, "Invalid version file")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")
 
@@ -245,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores
 // sync
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sync version")
 
 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")

src/inc/taosmsg.h

@@ -59,7 +59,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_VNODE, "alter-vnode" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY5, "dummy5" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_MNODE, "create-mnode" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" )
 
@@ -305,6 +305,8 @@ typedef struct {
   char clientVersion[TSDB_VERSION_LEN];
   char msgVersion[TSDB_VERSION_LEN];
   char db[TSDB_TABLE_FNAME_LEN];
+  char appName[TSDB_APPNAME_LEN];
+  int32_t pid;
 } SCMConnectMsg;
 
 typedef struct {
@@ -719,6 +721,12 @@ typedef struct {
   char     ep[TSDB_EP_LEN];  // end point, hostname:port
 } SCMCreateDnodeMsg, SCMDropDnodeMsg;
 
+typedef struct {
+  int32_t dnodeId;
+  char    dnodeEp[TSDB_EP_LEN];  // end point, hostname:port
+  SDMMnodeInfos mnodes;
+} SMDCreateMnodeMsg;
+
 typedef struct {
   int32_t dnodeId;
   int32_t vgId;
@@ -740,6 +748,7 @@ typedef struct {
   uint32_t queryId;
   int64_t  useconds;
   int64_t  stime;
+  uint64_t qHandle;
 } SQueryDesc;
 
 typedef struct {
@@ -755,8 +764,10 @@ typedef struct {
 
 typedef struct {
   uint32_t connId;
+  int32_t  pid;
   int32_t  numOfQueries;
   int32_t  numOfStreams;
+  char     appName[TSDB_APPNAME_LEN];
   char     pData[];
 } SCMHeartBeatMsg;
 

src/inc/twal.h

@@ -51,6 +51,7 @@ int     walWrite(twalh, SWalHead *);
 void    walFsync(twalh);
 int     walRestore(twalh, void *pVnode, FWalWrite writeFp);
 int     walGetWalFile(twalh, char *name, uint32_t *index);
+int64_t walGetVersion(twalh);
 
 extern int wDebugFlag;
 

src/kit/shell/inc/shell.h

@@ -20,6 +20,7 @@
 #include "taos.h"
 #include "taosdef.h"
 #include "stdbool.h"
+#include "tsclient.h"
 
 #define MAX_USERNAME_SIZE      64
 #define MAX_DBNAME_SIZE        64

src/kit/shell/src/shellEngine.c

@@ -294,9 +294,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
 
   st = taosGetTimestampUs();
 
-  TAOS_RES* pSql = taos_query(con, command);
+  TAOS_RES* pSql = taos_query_h(con, command, &result);
-  atomic_store_ptr(&result, pSql);  // set the global TAOS_RES pointer
-
   if (taos_errno(pSql)) {
     taos_error(pSql);
     return;

src/mnode/inc/mnodeMnode.h

@@ -31,7 +31,7 @@ typedef enum {
 int32_t mnodeInitMnodes();
 void    mnodeCleanupMnodes();
 
-int32_t mnodeAddMnode(int32_t dnodeId);
+void    mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm);
 int32_t mnodeDropMnode(int32_t dnodeId);
 void    mnodeDropMnodeLocal(int32_t dnodeId);
 

src/mnode/inc/mnodeProfile.h

@@ -23,6 +23,8 @@ extern "C" {
 
 typedef struct {
   char     user[TSDB_USER_LEN];
+  char     appName[TSDB_APPNAME_LEN];  // app name that invokes taosc
+  uint32_t pid;                        // pid of app that invokes taosc
   int8_t   killed;
   uint16_t port;
   uint32_t ip;
@@ -40,7 +42,7 @@ typedef struct {
 int32_t mnodeInitProfile();
 void    mnodeCleanupProfile();
 
-SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port);
+SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app);
 SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port);
 void      mnodeReleaseConn(SConnObj *pConn);
 int32_t   mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg);

src/mnode/src/mnodeCluster.c

@@ -225,6 +225,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows,
     numOfRows++;
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }

src/mnode/src/mnodeDb.c

@@ -648,8 +648,12 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
 
   while (numOfRows < rows) {
     pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb);
+
     if (pDb == NULL) break;
-    if (pDb->pAcct != pUser->pAcct) continue;
+    if (pDb->pAcct != pUser->pAcct) {
+      mnodeDecDbRef(pDb);
+      continue;
+    }
 
     cols = 0;
 
@@ -760,6 +764,8 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
   }
 
   pShow->numOfReads += numOfRows;
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+
   mnodeDecUserRef(pUser);
   return numOfRows;
 }

src/mnode/src/mnodeDnode.c

@@ -147,7 +147,7 @@ static int32_t mnodeDnodeActionRestored() {
     mnodeCreateDnode(tsLocalEp, NULL);
     SDnodeObj *pDnode = mnodeGetDnodeByEp(tsLocalEp);
     if (pDnode != NULL) {
-      mnodeAddMnode(pDnode->dnodeId);
+      mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, false);
       mnodeDecDnodeRef(pDnode);
     }
   }
@@ -471,7 +471,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
              mnodeGetClusterId());
       return TSDB_CODE_MND_INVALID_CLUSTER_ID;
     } else {
-      mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
+      mTrace("dnode:%d, status received, access times %d openVnodes:%d:%d", pDnode->dnodeId, pDnode->lastAccess,
+             htons(pStatus->openVnodes), pDnode->openVnodes);
     }
   }
 
@@ -790,6 +791,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
     mnodeDecDnodeRef(pDnode);
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
@@ -857,6 +859,7 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC
 
   char* pWrite;
   char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" };
+  int32_t cols;
 
   while (numOfRows < rows) {
     SDnodeObj *pDnode = NULL;
@@ -864,7 +867,7 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC
     if (pDnode == NULL) break;
 
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
-      int32_t cols = 0;
+      cols = 0;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int16_t *)pWrite = pDnode->dnodeId;
@@ -891,6 +894,7 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC
     mnodeDecDnodeRef(pDnode);
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
@@ -990,6 +994,7 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v
     }
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
@@ -1082,6 +1087,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
     numOfRows = 0;
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }

src/mnode/src/mnodeMain.c

@@ -109,7 +109,7 @@ int32_t mnodeStartSystem() {
 
   mInfo("mnode is initialized successfully");
 
-  sdbUpdateSync();
+  sdbUpdateSync(NULL);
 
   return 0;
 }

src/mnode/src/mnodeMnode.c

@@ -23,6 +23,8 @@
 #include "tutil.h"
 #include "tsocket.h"
 #include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
 #include "mnodeMnode.h"
@@ -30,6 +32,7 @@
 #include "mnodeSdb.h"
 #include "mnodeShow.h"
 #include "mnodeUser.h"
+#include "mnodeVgroup.h"
 
 static void *        tsMnodeSdb = NULL;
 static int32_t       tsMnodeUpdateSize = 0;
@@ -266,7 +269,61 @@ void mnodeGetMnodeInfos(void *mnodeInfos) {
   mnodeMnodeUnLock();
 }
 
-int32_t mnodeAddMnode(int32_t dnodeId) {
+static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
+  mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp);
+
+  SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg));
+  if (pCreate == NULL) {
+    return TSDB_CODE_MND_OUT_OF_MEMORY;
+  } else {
+    pCreate->dnodeId = htonl(dnodeId);
+    tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
+    pCreate->mnodes = tsMnodeInfos;
+    bool found = false;
+    for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) {
+      if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) {
+        found = true;
+      }
+    }
+    if (!found) {
+      pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId);
+      tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
+      pCreate->mnodes.nodeNum++;
+    }
+  }
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pCreate;
+  rpcMsg.contLen = sizeof(SMDCreateMnodeMsg);
+  rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE;
+
+  SRpcMsg   rpcRsp = {0};
+  SRpcEpSet epSet = mnodeGetEpSetFromIp(pCreate->dnodeEp);
+  dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp, &epSet);
+
+  if (rpcRsp.code != TSDB_CODE_SUCCESS) {
+    mError("dnode:%d, failed to send create mnode msg, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(rpcRsp.code));
+  } else {
+    mDebug("dnode:%d, create mnode msg is disposed, mnode is created in dnode", dnodeId);
+  }
+
+  rpcFreeCont(rpcRsp.pCont);
+  return rpcRsp.code;
+}
+
+static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) {
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("failed to create mnode, reason:%s", tstrerror(code));
+  } else {
+    mDebug("mnode is created successfully");
+    mnodeUpdateMnodeEpSet();
+    sdbUpdateAsync();
+  }
+
+  return code;
+}
+
+void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
   SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
   pMnode->mnodeId = dnodeId;
   pMnode->createdTime = taosGetTimestampMs();
@@ -275,16 +332,24 @@ int32_t mnodeAddMnode(int32_t dnodeId) {
     .type  = SDB_OPER_GLOBAL,
     .table = tsMnodeSdb,
     .pObj  = pMnode,
+    .writeCb = mnodeCreateMnodeCb
   };
 
-  int32_t code = sdbInsertRow(&oper);
+  int32_t code = TSDB_CODE_SUCCESS;
-  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+  if (needConfirm) {
-    taosTFree(pMnode);
+    code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp);
   }
 
-  mnodeUpdateMnodeEpSet();
+  if (code != TSDB_CODE_SUCCESS) {
+    taosTFree(pMnode);
+    return;
+  }
 
-  return code;
+  code = sdbInsertRow(&oper);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code));
+    taosTFree(pMnode);
+  }
 }
 
 void mnodeDropMnodeLocal(int32_t dnodeId) {
@@ -296,6 +361,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) {
   }
 
   mnodeUpdateMnodeEpSet();
+  sdbUpdateAsync();
 }
 
 int32_t mnodeDropMnode(int32_t dnodeId) {
@@ -315,6 +381,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) {
   sdbDecRef(tsMnodeSdb, pMnode);
 
   mnodeUpdateMnodeEpSet();
+  sdbUpdateAsync();
 
   return code;
 }
@@ -414,6 +481,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
     mnodeDecMnodeRef(pMnode);
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
 
   return numOfRows;

src/mnode/src/mnodePeer.c

@@ -58,10 +58,15 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("%p, msg:%s in mpeer queue, will be redireced, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle,
+    mDebug("%p, msg:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle,
            taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
     for (int32_t i = 0; i < epSet->numOfEps; ++i) {
-      mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i]));
+      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
+        epSet->inUse = (i + 1) % epSet->numOfEps;
+        mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+      } else {
+        mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+      }
     }
 
     return TSDB_CODE_RPC_REDIRECT;

src/mnode/src/mnodeProfile.c

@@ -24,15 +24,9 @@
 #include "mnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
-#include "mnodeAcct.h"
-#include "mnodeDnode.h"
-#include "mnodeDb.h"
-#include "mnodeMnode.h"
 #include "mnodeProfile.h"
 #include "mnodeShow.h"
-#include "mnodeTable.h"
 #include "mnodeUser.h"
-#include "mnodeVgroup.h"
 #include "mnodeWrite.h"
 
 #define CONN_KEEP_TIME  (tsShellActivityTimer * 3)
@@ -78,7 +72,7 @@ void mnodeCleanupProfile() {
   }
 }
 
-SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
+SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app) {
 #if 0
   int32_t connSize = taosHashGetSize(tsMnodeConnCache->pHashTable);
   if (connSize > tsMaxShellConns) {
@@ -96,10 +90,13 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
     .ip     = ip,
     .port   = port,
     .connId = connId,
-    .stime  = taosGetTimestampMs()
+    .stime  = taosGetTimestampMs(),
+    .pid    = pid,
   };
 
-  tstrncpy(connObj.user, user, sizeof(connObj.user));
+  tstrncpy(connObj.user, user, tListLen(connObj.user));
+  tstrncpy(connObj.appName, app, tListLen(connObj.appName));
+
   connObj.lastAccess = connObj.stime;
 
   SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000);
@@ -183,6 +180,20 @@ static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  // app name
+  pShow->bytes[cols] = TSDB_APPNAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "app_name");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  // app pid
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "pid");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "ip:port");
@@ -191,13 +202,13 @@ static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "login time");
+  strcpy(pSchema[cols].name, "login_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "last access");
+  strcpy(pSchema[cols].name, "last_access");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -236,6 +247,16 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]);
     cols++;
 
+    // app name
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->appName, pShow->bytes[cols]);
+    cols++;
+
+    // app pid
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t*)pWrite = pConnObj->pid;
+    cols++;
+
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
@@ -254,8 +275,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 5;
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
   
   return numOfRows;
 }
@@ -299,7 +319,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "queryId");
+  strcpy(pSchema[cols].name, "query_id");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -315,9 +335,15 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = 24;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "qhandle");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "created time");
+  strcpy(pSchema[cols].name, "created_time");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -352,7 +378,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
   SConnObj *pConnObj = NULL;
   int32_t   cols = 0;
   char *    pWrite;
-  char      ipStr[TSDB_IPv4ADDR_LEN + 6];
+  char      str[TSDB_IPv4ADDR_LEN + 6] = {0};
 
   while (numOfRows < rows) {
     pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj);
@@ -362,9 +388,9 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
       SQueryDesc *pDesc = pConnObj->pQueries + i;
       cols = 0;
 
-      snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId));
+      snprintf(str, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId));
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -372,8 +398,15 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
+      snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
-      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]);
+      cols++;
+
+      char handleBuf[24] = {0};
+      snprintf(handleBuf, tListLen(handleBuf), "%p", (void*)htobe64(pDesc->qHandle));
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->bytes[cols]);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -393,8 +426,7 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 6;
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
   return numOfRows;
 }
 
@@ -522,8 +554,7 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 8;
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
   return numOfRows;
 }
 

src/mnode/src/mnodeRead.c

@@ -51,14 +51,21 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
     SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
     SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
     mnodeGetMnodeEpSetForShell(epSet);
+
+    mDebug("%p, msg:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle,
+           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
+    for (int32_t i = 0; i < epSet->numOfEps; ++i) {
+      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
+        epSet->inUse = (i + 1) % epSet->numOfEps;
+        mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+      } else {
+        mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+      }
+    }
+
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse);
-    for (int32_t i = 0; i < epSet->numOfEps; ++i) {
-      mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i]));
-    }
-
     return TSDB_CODE_RPC_REDIRECT;
   }
 

src/mnode/src/mnodeSdb.c

@@ -298,31 +298,25 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   taosFreeQitem(pOper);
 }
 
-static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(); }
+static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); }
 
-void sdbUpdateSync() {
+void sdbUpdateAsync() {
+  taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsUpdateSyncTmr);
+}
+
+void sdbUpdateSync(void *pMnodes) {
+  SDMMnodeInfos *mnodes = pMnodes;
   if (!mnodeIsRunning()) {
-    mDebug("mnode not start yet, update sync info later");
+    mDebug("mnode not start yet, update sync config later");
-    if (dnodeCheckMnodeStarting()) {
-      taosTmrReset(sdbUpdateSyncTmrFp, 1000, NULL, tsMnodeTmr, &tsUpdateSyncTmr);
-    }
     return;
   }
-  mDebug("update sync info in sdb");
+
+  mDebug("update sync config in sync module, mnodes:%p", pMnodes);
 
   SSyncCfg syncCfg = {0};
   int32_t  index = 0;
 
-  SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
+  if (mnodes == NULL) {
-  for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
-    SDMMnodeInfo *node = &mnodes->nodeInfos[i];
-    syncCfg.nodeInfo[i].nodeId = node->nodeId;
-    taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[i].nodeFqdn, &syncCfg.nodeInfo[i].nodePort);
-    syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
-    index++;
-  }
-
-  if (index == 0) {
     void *pIter = NULL;
     while (1) {
       SMnodeObj *pMnode = NULL;
@@ -342,9 +336,19 @@ void sdbUpdateSync() {
       mnodeDecMnodeRef(pMnode);
     }
     sdbFreeIter(pIter);
+    syncCfg.replica = index;
+    mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
+  } else {
+    for (index = 0; index < mnodes->nodeNum; ++index) {
+      SDMMnodeInfo *node = &mnodes->nodeInfos[index];
+      syncCfg.nodeInfo[index].nodeId = node->nodeId;
+      taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
+      syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
+    }
+    syncCfg.replica = index;
+    mDebug("mnodes info input, numOfMnodes:%d", syncCfg.replica);
   }
 
-  syncCfg.replica = index;
   syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2;
 
   bool hasThisDnode = false;
@@ -355,8 +359,15 @@ void sdbUpdateSync() {
     }
   }
 
-  if (!hasThisDnode) return;
+  if (!hasThisDnode) {
-  if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
+    sdbDebug("update sync config, this dnode not exist");
+    return;
+  }
+
+  if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) {
+    sdbDebug("update sync config, info not changed");
+    return;
+  }
 
   sdbInfo("work as mnode, replica:%d", syncCfg.replica);
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
@@ -583,7 +594,7 @@ static int sdbWrite(void *param, void *data, int type) {
       pthread_mutex_unlock(&tsSdbObj.mutex);
       sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
                pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
-      return TSDB_CODE_MND_APP_ERROR;
+      return TSDB_CODE_SYN_INVALID_VERSION;
     } else {
       tsSdbObj.version = pHead->version;
     }
@@ -1043,7 +1054,7 @@ static void *sdbWorkerFp(void *param) {
   while (1) {
     numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed);
     if (numOfMsgs == 0) {
-      sdbDebug("sdbWorkerFp: got no message from qset, exiting...");
+      sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWriteQset);
       break;
     }
 

src/mnode/src/mnodeShow.c

@@ -186,7 +186,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
     rowsToRead = pShow->numOfRows - pShow->numOfReads;
   }
 
-  /* return no more than 100 meters in one round trip */
+  /* return no more than 100 tables in one round trip */
   if (rowsToRead > 100) rowsToRead = 100;
 
   /*
@@ -244,7 +244,8 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
   int32_t connId = htonl(pHBMsg->connId);
   SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort);
   if (pConn == NULL) {
-    pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort);
+    pHBMsg->pid = htonl(pHBMsg->pid);
+    pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pHBMsg->pid, pHBMsg->appName);
   }
 
   if (pConn == NULL) {
@@ -325,7 +326,8 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
     goto connect_over;
   }
 
-  SConnObj *pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort);
+  pConnectMsg->pid = htonl(pConnectMsg->pid);
+  SConnObj *pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pConnectMsg->pid, pConnectMsg->appName);
   if (pConn == NULL) {
     code = terrno;
   } else {

src/mnode/src/mnodeTable.c

@@ -63,27 +63,27 @@ static int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t
 static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
  
-static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg);
 static void    mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg);
 
-static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg);
 static void    mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg);
 static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg);
 static void    mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg);
 
-static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg);
-static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg);
-static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg);
 
-static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg);
 static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg);
 static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg);
 static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg);
 
-static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *mnodeMsg);
+static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg);
 static void    mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg);
 
 static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName);
@@ -131,6 +131,8 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
       mnodeAddTableIntoStable(pTable->superTable, pTable);
       grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
       if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
+    } else {
+      mError("table:%s:%p, correspond stable not found suid:%" PRIu64, pTable->info.tableId, pTable, pTable->suid);
     }
   } else {
     grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
@@ -460,12 +462,14 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
     void *oldSchema = pTable->schema;
     void *oldVgHash = pTable->vgHash;
     int32_t oldRefCount = pTable->refCount;
+    int32_t oldNumOfTables = pTable->numOfTables;
 
     memcpy(pTable, pNew, sizeof(SSuperTableObj));
 
     pTable->vgHash = oldVgHash;
     pTable->refCount = oldRefCount;
     pTable->schema = pNew->schema;
+    pTable->numOfTables = oldNumOfTables;
     free(pNew);
     free(oldTableId);
     free(oldSchema);
@@ -837,10 +841,11 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
+  int64_t us = taosGetTimestampUs();
   pStable->info.tableId = strdup(pCreate->tableId);
   pStable->info.type    = TSDB_SUPER_TABLE;
   pStable->createdTime  = taosGetTimestampMs();
-  pStable->uid          = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul));
+  pStable->uid          = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
   pStable->sversion     = 0;
   pStable->tversion     = 0;
   pStable->numOfColumns = htons(pCreate->numOfColumns);
@@ -1384,9 +1389,8 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 5;
 
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   mnodeDecDbRef(pDb);
 
   return numOfRows;
@@ -1495,7 +1499,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
       continue;
     }
     if (pTable->vgHash == NULL) {
-      mError("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
+      mDebug("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
              stableName);
       mnodeDecTableRef(pTable);
 
@@ -1706,7 +1710,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   pTable->vgId         = pVgroup->vgId;
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    STagData *pTagData = (STagData *) pCreate->schema;  // it is a tag key
+    STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
     if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
     if (pMsg->pSTable == NULL) {
       mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
@@ -2251,7 +2255,8 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
   int32_t numOfTables = 0;
   SChildTableObj *pTable = NULL;
 
-  mInfo("stable:%s, all child tables:%d will dropped from sdb", pStable->info.tableId, pStable->numOfTables);
+  mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid,
+        pStable->numOfTables);
 
   while (1) {
     pIter = mnodeGetNextChildTable(pIter, &pTable);
@@ -2404,14 +2409,16 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
     }
   } else {
     if (mnodeMsg->retry++ < 10) {
-      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d result:%s thandle:%p",
+      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
+             " result:%s thandle:%p",
-             mnodeMsg->rpcMsg.handle);
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->sid,
+             pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
 
       dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
     } else {
-      mError("app:%p:%p, table:%s, failed to create in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
-             pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid,
+             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
 
       SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
       sdbDeleteRow(&oper);
@@ -2543,6 +2550,25 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = 8; // table uid
+  pSchema[cols].type = TSDB_DATA_TYPE_BIGINT;
+  strcpy(pSchema[cols].name, "uid");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "tid");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "vgId");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+
   pMeta->numOfColumns = htons(cols);
   pShow->numOfColumns = cols;
 
@@ -2568,6 +2594,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
     return 0;
   }
 
+  int32_t cols       = 0;
   int32_t numOfRows  = 0;
   SChildTableObj *pTable = NULL;
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
@@ -2608,8 +2635,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
       continue;
     }
 
-    int32_t cols = 0;
+    cols = 0;
-
     char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
 
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, pShow->bytes[cols]);
@@ -2638,14 +2664,29 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
     
     cols++;
 
+    // uid
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t*) pWrite = pTable->uid;
+    cols++;
+
+
+    // tid
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t*) pWrite = pTable->sid;
+    cols++;
+
+    //vgid
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t*) pWrite = pTable->vgId;
+    cols++;
+
     numOfRows++;
     mnodeDecTableRef(pTable);
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 4;
 
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   mnodeDecDbRef(pDb);
   free(pattern);
 
@@ -2843,9 +2884,8 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
   }
 
   pShow->numOfReads += numOfRows;
-  const int32_t NUM_OF_COLUMNS = 4;
 
-  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   mnodeDecDbRef(pDb);
 
   return numOfRows;

src/mnode/src/mnodeUser.c

@@ -386,6 +386,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi
     mnodeDecUserRef(pUser);
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }

src/mnode/src/mnodeVgroup.c

@@ -270,24 +270,26 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
   Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode 
 */
 void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
-  SVnodeLoad *pNextV = NULL;
-
   void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup;
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
-    pNextV = pVloads;
+    for (int v = 0; v < pVgroup->numOfVnodes; ++v) {
-    int32_t i;
+      if (pVgroup->vnodeGid[v].dnodeId == pDnode->dnodeId) {
-    for (i = 0; i < openVnodes; ++i) {
+        // vgroup should have a vnode on this dnode
-      if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
+        bool have = false;
+        for (int32_t i = 0; i < openVnodes; ++i) {
+          SVnodeLoad *pVload = pVloads + i;
+          if (pVgroup->vgId == pVload->vgId) {
+            have = true;
             break;
           }
-      pNextV++;
         }
 
-    if (i == openVnodes) {
+        if (have) continue;
+
         if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_DROPPING) {
           mDebug("vgId:%d, not exist in dnode:%d and status is %s, do nothing", pVgroup->vgId, pDnode->dnodeId,
                  vgroupStatus[pVgroup->status]);
@@ -297,12 +299,12 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
           mnodeSendCreateVgroupMsg(pVgroup, NULL);
         }
       }
+    }
 
     mnodeDecVgroupRef(pVgroup);
   }
 
   sdbFreeIter(pIter);
-  return;
 }
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
@@ -310,7 +312,7 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
     if (pVgid->pDnode == pDnode) {
-      mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d", pDnode->dnodeId, pVgroup->vgId, pVgid->role);
+      mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d:%s", pDnode->dnodeId, pVgroup->vgId, pVgid->role, syncRole[pVgid->role]);
       pVgid->role = pVload->role;
       if (pVload->role == TAOS_SYNC_ROLE_MASTER) {
         pVgroup->inUse = i;
@@ -723,8 +725,16 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
   while (numOfRows < rows) {
     pShow->pIter = mnodeGetNextVgroup(pShow->pIter, &pVgroup);
     if (pVgroup == NULL) break;
-    if (pVgroup->pDb != pDb) continue;
+
-    if (!mnodeFilterVgroups(pVgroup, pTable)) continue;
+    if (pVgroup->pDb != pDb) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }
+
+    if (!mnodeFilterVgroups(pVgroup, pTable)) {
+      mnodeDecVgroupRef(pVgroup);
+      continue;
+    }
 
     cols = 0;
 
@@ -772,6 +782,8 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
     numOfRows++;
   }
 
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+
   pShow->numOfReads += numOfRows;
   mnodeDecTableRef(pTable);
   mnodeDecDbRef(pDb);

src/mnode/src/mnodeWrite.c

@@ -54,12 +54,18 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+    mDebug("app:%p:%p, msg:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, pMsg,
-           epSet->inUse);
+           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
     for (int32_t i = 0; i < epSet->numOfEps; ++i) {
+      if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
+        epSet->inUse = (i + 1) % epSet->numOfEps;
+        mDebug("app:%p:%p, mnode index:%d ep:%s:%d, set inUse to %d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i],
+               htons(epSet->port[i]), epSet->inUse);
+      } else {
         mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i],
                htons(epSet->port[i]));
       }
+    }
 
     return TSDB_CODE_RPC_REDIRECT;
   }

src/os/inc/osSemphone.h

@@ -33,6 +33,8 @@ bool taosCheckPthreadValid(pthread_t thread);
 int64_t taosGetPthreadId();
 void taosResetPthread(pthread_t *thread);
 bool taosComparePthread(pthread_t first, pthread_t second);
+int32_t taosGetPId();
+int32_t taosGetCurrentAPPName(char *name, int32_t* len);
 
 #ifdef __cplusplus
 }

src/os/src/detail/osSemphone.c

@@ -34,5 +34,31 @@ bool    taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
 int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
 void    taosResetPthread(pthread_t *thread) { *thread = 0; }
 bool    taosComparePthread(pthread_t first, pthread_t second) { return first == second; }
+int32_t taosGetPId() { return getpid(); }
+
+int32_t taosGetCurrentAPPName(char *name, int32_t* len) {
+  const char* self = "/proc/self/exe";
+  char path[PATH_MAX] = {0};
+
+  if (readlink(self, path, PATH_MAX) <= 0) {
+    return -1;
+  }
+
+  path[PATH_MAX - 1] = 0;
+  char* end = strrchr(path, '/');
+  if (end == NULL) {
+    return -1;
+  }
+
+  ++end;
+
+  strcpy(name, end);
+
+  if (len != NULL) {
+    *len = strlen(name);
+  }
+
+  return 0;
+}
 
 #endif

src/os/src/detail/osSysinfo.c

@@ -203,7 +203,7 @@ static void taosGetSystemTimezone() {
   snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
 
   // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
-  uInfo("timezone not configured, set to system default:%s", tsTimezone);
+  uWarn("timezone not configured, set to system default:%s", tsTimezone);
 }
 
 /*
@@ -235,7 +235,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       strcpy(tsLocale, "en_US.UTF-8");
     } else {
       tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
-      uError("locale not configured, set to system default:%s", tsLocale);
+      uWarn("locale not configured, set to system default:%s", tsLocale);
     }
   }
 

src/os/src/linux/CMakeLists.txt

@@ -4,4 +4,4 @@ PROJECT(TDengine)
 AUX_SOURCE_DIRECTORY(. SRC)
 
 ADD_LIBRARY(os ${SRC})
-TARGET_LINK_LIBRARIES(os m rt)
+TARGET_LINK_LIBRARIES(os m rt z)

src/os/src/windows/wSemphone.c

@@ -36,3 +36,21 @@ int64_t taosGetPthreadId() {
 bool taosComparePthread(pthread_t first, pthread_t second) {
   return first.p == second.p;
 }
+
+int32_t taosGetPId() {
+  return GetCurrentProcessId();
+}
+
+int32_t taosGetCurrentAPPName(char *name, int32_t* len) {
+  char filepath[1024] = {0};
+
+  GetModuleFileName(NULL, filepath, MAX_PATH);
+  *strrchr(filepath,'.') = '\0';
+  strcpy(name, filepath);
+
+  if (len != NULL) {
+    *len = (int32_t) strlen(filepath);
+  }
+
+  return 0;
+}

src/plugins/http/src/httpQueue.c

@@ -67,7 +67,7 @@ static void *httpProcessResultQueue(void *param) {
   
   while (1) {
     if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) {
-      httpDebug("httpResultQueue: got no message from qset, exiting...");
+      httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset);
       break;
     }
 

src/query/inc/qExtbuffer.h

@@ -89,7 +89,7 @@ typedef struct SColumnModel {
 
 typedef struct SColumnOrderInfo {
   int32_t numOfCols;
-  int16_t pData[];
+  int16_t colIndex[];
 } SColumnOrderInfo;
 
 typedef struct tOrderDescriptor {

src/query/src/qExecutor.c

@@ -1537,10 +1537,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
       if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) {
         pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
       } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value
+        if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
           tVariantCreateFromBinary(&pCtx->param[1], (char*) &pQuery->fillVal[colIndex], pCtx->inputBytes, pCtx->inputType);
         }
       }
     }
+  }
 
 #if defined(_DEBUG_VIEW)
   //  int64_t *tsList = (int64_t *)primaryColumnData;
@@ -4511,7 +4513,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   int32_t code = TSDB_CODE_SUCCESS;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
-  pQuery->precision = tsdbGetCfg(tsdb)->precision;
   pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery);
   pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery);
 
@@ -6324,6 +6325,8 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
     UNUSED(ret);
   }
   
+  pQuery->precision = tsdbGetCfg(tsdb)->precision;
+
   if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) ||
       (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) {
     qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey,

src/query/src/qExtbuffer.c

@@ -344,7 +344,9 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t
     return 0;
   }
 
-  if (colIdx == 0 && tsOrder == TSDB_ORDER_DESC) {  // primary column desc order
+  assert(colIdx == 0);
+
+  if (tsOrder == TSDB_ORDER_DESC) {  // primary column desc order
     return (f1 < f2) ? 1 : -1;
   } else {  // asc
     return (f1 < f2) ? -1 : 1;
@@ -435,7 +437,7 @@ int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1,
 
   int32_t cmpCnt = pDescriptor->orderInfo.numOfCols;
   for (int32_t i = 0; i < cmpCnt; ++i) {
-    int32_t colIdx = pDescriptor->orderInfo.pData[i];
+    int32_t colIdx = pDescriptor->orderInfo.colIndex[i];
 
     char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx);
     char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx);
@@ -467,7 +469,7 @@ int32_t compare_d(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1,
 
   int32_t cmpCnt = pDescriptor->orderInfo.numOfCols;
   for (int32_t i = 0; i < cmpCnt; ++i) {
-    int32_t colIdx = pDescriptor->orderInfo.pData[i];
+    int32_t colIdx = pDescriptor->orderInfo.colIndex[i];
 
     char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx);
     char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx);
@@ -557,13 +559,13 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
   int32_t midIdx = ((end - start) >> 1) + start;
 
 #if defined(_DEBUG_VIEW)
-  int32_t f = pDescriptor->orderInfo.pData[0];
+  int32_t f = pDescriptor->orderInfo.colIndex[0];
 
   char *midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f);
   char *startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f);
   char *endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f);
 
-  int32_t colIdx = pDescriptor->orderInfo.pData[0];
+  int32_t colIdx = pDescriptor->orderInfo.colIndex[0];
   tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx);
 #endif
 
@@ -591,7 +593,7 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
 }
 
 static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t numOfRows, char *d, int32_t len) {
-  int32_t colIdx = pDescriptor->orderInfo.pData[0];
+  int32_t colIdx = pDescriptor->orderInfo.colIndex[0];
 
   for (int32_t i = 0; i < len; ++i) {
     char *startx = COLMODEL_GET_VAL(d, pDescriptor->pColumnModel, numOfRows, i, colIdx);
@@ -1075,7 +1077,7 @@ tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrder
 
   desc->orderInfo.numOfCols = numOfOrderCols;
   for (int32_t i = 0; i < numOfOrderCols; ++i) {
-    desc->orderInfo.pData[i] = orderColIdx[i];
+    desc->orderInfo.colIndex[i] = orderColIdx[i];
   }
 
   return desc;

src/rpc/src/rpcMain.c

@@ -195,7 +195,7 @@ static void  rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen);
 static void  rpcSendReqHead(SRpcConn *pConn);
 
 static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv);
-static void  rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead);
+static void  rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext);
 static void  rpcProcessConnError(void *param, void *id);
 static void  rpcProcessRetryTimer(void *, void *);
 static void  rpcProcessIdleTimer(void *param, void *tmrId);
@@ -323,7 +323,7 @@ void *rpcMallocCont(int contLen) {
     tError("failed to malloc msg, size:%d", size);
     return NULL;
   } else {
-    tDebug("malloc msg: %p", start);
+    tTrace("malloc mem: %p", start);
   }
 
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
@@ -333,7 +333,7 @@ void rpcFreeCont(void *cont) {
   if (cont) {
     char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext);
     free(temp);
-    tDebug("free mem: %p", temp);
+    tTrace("free mem: %p", temp);
   }
 }
 
@@ -542,10 +542,7 @@ void rpcCancelRequest(void *handle) {
 
   if (pContext->pConn) {
     tDebug("%s, app tries to cancel request", pContext->pConn->info);
-    pContext->pConn->pReqMsg = NULL;  
     rpcCloseConn(pContext->pConn);
-    pContext->pConn = NULL;
-    rpcFreeCont(pContext->pCont);
   }
 }
 
@@ -553,7 +550,7 @@ static void rpcFreeMsg(void *msg) {
   if ( msg ) {
     char *temp = (char *)msg - sizeof(SRpcReqContext);
     free(temp);
-    tDebug("free msg: %p", temp);
+    tTrace("free mem: %p", temp);
   }
 }
 
@@ -613,9 +610,11 @@ static void rpcReleaseConn(SRpcConn *pConn) {
     if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg);  // do not use rpcFreeMsg
   } else {
     // if there is an outgoing message, free it
-    if (pConn->outType && pConn->pReqMsg) 
+    if (pConn->outType && pConn->pReqMsg) {
+      if (pConn->pContext) pConn->pContext->pConn = NULL; 
       rpcFreeMsg(pConn->pReqMsg);
     }
+  }
 
   // memset could not be used, since lockeBy can not be reset
   pConn->inType = 0;
@@ -819,9 +818,18 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
       return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED;
     }
 
+    if (rpcContLenFromMsg(pHead->msgLen) <= 0) {
+      tDebug("%s, message body is empty, ignore", pConn->info);
+      return TSDB_CODE_RPC_APP_ERROR;
+    }
+
     pConn->inTranId = pHead->tranId;
     pConn->inType = pHead->msgType;
 
+    // start the progress timer to monitor the response from server app
+    if (pConn->connType != RPC_CONN_TCPS) 
+      pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl);
+ 
     return 0;
 }
 
@@ -881,17 +889,32 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) {
   pConn->outType = 0;
   pConn->pReqMsg = NULL;
   pConn->reqMsgLen = 0;
+  SRpcReqContext *pContext = pConn->pContext;
+
+  if (pHead->code == TSDB_CODE_RPC_REDIRECT) { 
+    if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SRpcEpSet)) {
+      // if EpSet is not included in the msg, treat it as NOT_READY
+      pHead->code = TSDB_CODE_RPC_NOT_READY; 
+    } else {
+      pContext->redirect++;
+      if (pContext->redirect > TSDB_MAX_REPLICA) {
+        pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; 
+        tWarn("%s, too many redirects, quit", pConn->info);
+      }
+    }
+  } 
 
   return TSDB_CODE_SUCCESS;
 }
 
-static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
+static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqContext **ppContext) {
   int32_t    sid;
   SRpcConn  *pConn = NULL;
 
   SRpcHead *pHead = (SRpcHead *)pRecv->msg;
 
   sid = htonl(pHead->destId);
+  *ppContext = NULL;
 
   if (pHead->msgType >= TSDB_MSG_TYPE_MAX || pHead->msgType <= 0) {
     tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType);
@@ -945,6 +968,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
         pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer*2, pConn, pRpc->tmrCtrl);
     } else {
       terrno = rpcProcessRspHead(pConn, pHead);
+      *ppContext = pConn->pContext;
     }
   }
 
@@ -1009,7 +1033,8 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   }
 
   terrno = 0;
-  pConn = rpcProcessMsgHead(pRpc, pRecv);
+  SRpcReqContext *pContext;
+  pConn = rpcProcessMsgHead(pRpc, pRecv, &pContext);
 
   if (pHead->msgType >= 1 && pHead->msgType < TSDB_MSG_TYPE_MAX) {
     tDebug("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label,
@@ -1026,10 +1051,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     if (code != 0) { // parsing error
       if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
+        if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) {
+          rpcCloseConn(pConn);
+        }
         tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 
-      rpcProcessIncomingMsg(pConn, pHead);
+      rpcProcessIncomingMsg(pConn, pHead, pContext);
     }
   }
 
@@ -1060,7 +1088,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
   rpcFreeCont(pContext->pCont); 
 }
 
-static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
+static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
 
   SRpcInfo *pRpc = pConn->pRpc;
   SRpcMsg   rpcMsg;
@@ -1070,29 +1098,18 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
   rpcMsg.pCont = pHead->content;
   rpcMsg.msgType = pHead->msgType;
   rpcMsg.code = pHead->code; 
-  rpcMsg.ahandle = pConn->ahandle;
    
   if ( rpcIsReq(pHead->msgType) ) {
-    if (rpcMsg.contLen > 0) {
+    rpcMsg.ahandle = pConn->ahandle;
     rpcMsg.handle = pConn;
     rpcAddRef(pRpc);  // add the refCount for requests
 
-      // start the progress timer to monitor the response from server app
-      if (pConn->connType != RPC_CONN_TCPS) 
-        pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl);
- 
     // notify the server app
     (*(pRpc->cfp))(&rpcMsg, NULL);
-    } else {
-      tDebug("%s, message body is empty, ignore", pConn->info);
-      rpcFreeCont(rpcMsg.pCont);
-    }
   } else {
     // it's a response
-    SRpcReqContext *pContext = pConn->pContext;
     rpcMsg.handle = pContext;
-    pConn->pContext = NULL;
+    rpcMsg.ahandle = pContext->ahandle;
-    pConn->pReqMsg = NULL;
 
     // for UDP, port may be changed by server, the port in epSet shall be used for cache
     if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
@@ -1101,27 +1118,18 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
       rpcCloseConn(pConn);
     }
 
-    if (pHead->code == TSDB_CODE_RPC_REDIRECT) { 
-      if (rpcMsg.contLen < sizeof(SRpcEpSet)) {
-        // if EpSet is not included in the msg, treat it as NOT_READY
-        pHead->code = TSDB_CODE_RPC_NOT_READY; 
-      } else {
-        pContext->redirect++;
-        if (pContext->redirect > TSDB_MAX_REPLICA) {
-          pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; 
-          tWarn("%s, too many redirects, quit", pConn->info);
-        }
-      }
-    }
-
     if (pHead->code == TSDB_CODE_RPC_REDIRECT) {
       pContext->numOfTry = 0;
       SRpcEpSet *pEpSet = (SRpcEpSet*)pHead->content;
       if (pEpSet->numOfEps > 0) {
         memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet));
-        tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps);
+        tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps,
-        for (int i=0; i<pContext->epSet.numOfEps; ++i) 
+               pContext->epSet.inUse);
+        for (int i = 0; i < pContext->epSet.numOfEps; ++i) {
           pContext->epSet.port[i] = htons(pContext->epSet.port[i]);
+          tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.fqdn[i],
+                 pContext->epSet.port[i]);
+        }
       }
       rpcSendReqToServer(pRpc, pContext);
       rpcFreeCont(rpcMsg.pCont);
@@ -1445,7 +1453,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
       pNewHead->msgLen = rpcMsgLenFromCont(origLen);
       rpcFreeMsg(pHead); // free the compressed message buffer
       pHead = pNewHead; 
-      //tTrace("decompress rpc msg, compLen:%d, after:%d", compLen, contLen);
+      tTrace("decomp malloc mem: %p", temp);
     } else {
       tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
     }

src/rpc/src/rpcTcp.c

@@ -62,7 +62,7 @@ typedef struct {
   char        label[TSDB_LABEL_LEN];
   int         numOfThreads;
   void *      shandle;
-  SThreadObj *pThreadObj;
+  SThreadObj **pThreadObj;
   pthread_t   thread;
 } SServerObj;
 
@@ -90,7 +90,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   tstrncpy(pServerObj->label, label, sizeof(pServerObj->label));
   pServerObj->numOfThreads = numOfThreads;
 
-  pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
+  pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads);
   if (pServerObj->pThreadObj == NULL) {
     tError("TCP:%s no enough memory", label);
     terrno = TAOS_SYSTEM_ERROR(errno); 
@@ -104,19 +104,28 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
   // initialize parameters in case it may encounter error later 
-  pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
+    pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1);
+    if (pThreadObj == NULL) {
+      tError("TCP:%s no enough memory", label);
+      terrno = TAOS_SYSTEM_ERROR(errno); 
+      for (int j=0; j<i; ++j) free(pServerObj->pThreadObj[j]);
+      free(pServerObj->pThreadObj);
+      free(pServerObj);
+      return NULL;
+    }
+      
+    pServerObj->pThreadObj[i] = pThreadObj;
     pThreadObj->pollFd = -1;
     taosResetPthread(&pThreadObj->thread);
     pThreadObj->processData = fp;
     tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
     pThreadObj->shandle = shandle;
-    pThreadObj++;
   }
 
   // initialize mutex, thread, fd which may fail
-  pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
+    pThreadObj = pServerObj->pThreadObj[i];
     code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
     if (code < 0) {
       tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
@@ -137,7 +146,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     }
 
     pThreadObj->threadId = i;
-    pThreadObj++;
   }
 
   pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
@@ -166,6 +174,11 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   pThreadObj->stop = true;
   eventfd_t fd = -1;
 
+  if (taosComparePthread(pThreadObj->thread, pthread_self())) {
+    pthread_detach(pthread_self());
+    return;
+  }
+
   if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) {
     // signal the thread to stop, try graceful method first,
     // and use pthread_cancel when failed
@@ -183,15 +196,11 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
     }
   }
 
-  if (taosCheckPthreadValid(pThreadObj->thread)) pthread_join(pThreadObj->thread, NULL);
+  if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) {
-  if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd);
+     pthread_join(pThreadObj->thread, NULL);
-  if (fd != -1) taosCloseSocket(fd);
-
-  while (pThreadObj->pHead) {
-    SFdObj *pFdObj = pThreadObj->pHead;
-    pThreadObj->pHead = pFdObj->next;
-    taosFreeFdObj(pFdObj);
   }
+
+  if (fd != -1) taosCloseSocket(fd);
 }
 
 void taosStopTcpServer(void *handle) {
@@ -199,7 +208,14 @@ void taosStopTcpServer(void *handle) {
 
   if (pServerObj == NULL) return;
   if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
-  if (taosCheckPthreadValid(pServerObj->thread)) pthread_join(pServerObj->thread, NULL);
+
+  if (taosCheckPthreadValid(pServerObj->thread)) {
+    if (taosComparePthread(pServerObj->thread, pthread_self())) {
+      pthread_detach(pthread_self());
+    } else {
+      pthread_join(pServerObj->thread, NULL);
+    }
+  }
 
   tDebug("%s TCP server is stopped", pServerObj->label);
 }
@@ -210,9 +226,8 @@ void taosCleanUpTcpServer(void *handle) {
   if (pServerObj == NULL) return;
 
   for (int i = 0; i < pServerObj->numOfThreads; ++i) {
-    pThreadObj = pServerObj->pThreadObj + i;
+    pThreadObj = pServerObj->pThreadObj[i];
     taosStopTcpThread(pThreadObj);
-    pthread_mutex_destroy(&(pThreadObj->mutex));
   }
 
   tDebug("%s TCP server is cleaned up", pServerObj->label);
@@ -249,7 +264,7 @@ static void *taosAcceptTcpConnection(void *arg) {
     taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
 
     // pick up the thread to handle this connection
-    pThreadObj = pServerObj->pThreadObj + threadId;
+    pThreadObj = pServerObj->pThreadObj[threadId];
 
     SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
     if (pFdObj) {
@@ -327,10 +342,8 @@ void taosCleanUpTcpClient(void *chandle) {
   SThreadObj *pThreadObj = chandle;
   if (pThreadObj == NULL) return;
 
+  tDebug ("%s TCP client will be cleaned up", pThreadObj->label);
   taosStopTcpThread(pThreadObj);
-  tDebug ("%s TCP client is cleaned up", pThreadObj->label);
-
-  taosTFree(pThreadObj);
 }
 
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
@@ -365,7 +378,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
 
 void taosCloseTcpConnection(void *chandle) {
   SFdObj *pFdObj = chandle;
-  if (pFdObj == NULL) return;
+  if (pFdObj == NULL || pFdObj->signature != pFdObj) return;
 
   SThreadObj *pThreadObj = pFdObj->pThreadObj;
   tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); 
@@ -378,7 +391,7 @@ void taosCloseTcpConnection(void *chandle) {
 int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
   SFdObj *pFdObj = chandle;
 
-  if (chandle == NULL) return -1;
+  if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
 
   return taosWriteMsg(pFdObj->fd, data, len);
 }
@@ -425,7 +438,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   } else {
-    tDebug("TCP malloc mem: %p", buffer);
+    tTrace("TCP malloc mem: %p", buffer);
   }
 
   msg = buffer + tsRpcOverhead;
@@ -503,8 +516,22 @@ static void *taosProcessTcpData(void *param) {
       pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
       if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj);
     }
+
+    if (pThreadObj->stop) break; 
   }
 
+  if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd);
+
+  while (pThreadObj->pHead) {
+    SFdObj *pFdObj = pThreadObj->pHead;
+    pThreadObj->pHead = pFdObj->next;
+    taosReportBrokenLink(pFdObj);
+  }
+
+  pthread_mutex_destroy(&(pThreadObj->mutex));
+  tDebug("%s TCP thread exits ...", pThreadObj->label);
+  taosTFree(pThreadObj);
+
   return NULL;
 }
 

src/rpc/src/rpcUdp.c

@@ -214,7 +214,7 @@ static void *taosRecvUdpData(void *param) {
       tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen);
       continue;
     } else {
-      tDebug("UDP malloc mem: %p", tmsg);
+      tTrace("UDP malloc mem: %p", tmsg);
     }
 
     tmsg += tsRpcOverhead;  // overhead for SRpcReqContext

src/sync/src/syncMain.c

@@ -215,6 +215,9 @@ void syncStop(void *param) {
 
   pthread_mutex_lock(&(pNode->mutex));
 
+  if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+  if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
+
   for (int i = 0; i < pNode->replica; ++i) {
     pPeer = pNode->peerInfo[i];
     if (pPeer) syncRemovePeer(pPeer);
@@ -223,9 +226,6 @@ void syncStop(void *param) {
   pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
   if (pPeer) syncRemovePeer(pPeer);
 
-  if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
-  if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
-
   pthread_mutex_unlock(&(pNode->mutex));
 
   syncDecNodeRef(pNode);
@@ -311,8 +311,21 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
 
   if (pNode == NULL) return 0;
 
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
+    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
+           pWalHead->version, nodeVersion);
+    for (int i = 0; i < pNode->replica; ++i) {
+      pPeer = pNode->peerInfo[i];
+      syncRestartConnection(pPeer);
+    }
+    return TSDB_CODE_SYN_INVALID_VERSION;
+  }
+
   // always update version
   nodeVersion = pWalHead->version;
+  sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
+         qtype, pWalHead->version);
+
   if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
 
   // only pkt from RPC or CQ can be forwarded
@@ -881,7 +894,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
   SSyncNode *   pNode = pPeer->pSyncNode;
   SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
 
-  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+  sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
          syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
 
   pPeer->version = pPeersStatus->version;
@@ -968,7 +981,8 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
 
   int retLen = write(pPeer->peerFd, msg, statusMsgLen);
   if (retLen == statusMsgLen) {
-    sDebug("%s, status msg is sent", pPeer->id);
+    sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
+           pPeersStatus->version, pPeersStatus->ack);
   } else {
     sDebug("%s, failed to send status msg, restart", pPeer->id);
     syncRestartConnection(pPeer);
@@ -1189,6 +1203,8 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
 static void syncMonitorFwdInfos(void *param, void *tmrId) {
   SSyncNode *pNode = param;
   SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  if (pSyncFwds == NULL) return;
+
   uint64_t   time = taosGetTimestampMs();
 
   if (pSyncFwds->fwds > 0) {

src/sync/src/taosTcpPool.c

@@ -219,7 +219,10 @@ static void *taosProcessTcpData(void *param) {
           continue;
         }
       }
+
     }
+
+    if (pThread->stop) break;
   }
 
   uDebug("%p TCP epoll thread exits", pThread);
@@ -321,5 +324,5 @@ static void taosStopPoolThread(SThreadObj *pThread) {
   }
 
   pthread_join(thread, NULL);
-  taosClose(fd);
+  if (fd >= 0) taosClose(fd);
 }

src/tsdb/src/tsdbRead.c

@@ -72,7 +72,6 @@ typedef struct STableCheckInfo {
   SCompInfo*    pCompInfo;
   int32_t       compSize;
   int32_t       numOfBlocks;    // number of qualified data blocks not the original blocks
-  SDataCols*    pDataCols;
   int32_t       chosen;         // indicate which iterator should move forward
   bool          initBuf;        // whether to initialize the in-memory skip list iterator or not
   SSkipListIterator* iter;      // mem buffer skip list iterator
@@ -117,10 +116,12 @@ typedef struct STsdbQueryHandle {
   SFileGroupIter fileIter;
   SRWHelper      rhelper;
   STableBlockInfo* pDataBlockInfo;
+
+  SDataCols     *pDataCols;        // in order to hold current file data block
   int32_t        allocSize;        // allocated data block size
-  SMemTable*     mem;              // mem-table
+  SMemTable     *mem;              // mem-table
-  SMemTable*     imem;             // imem-table, acquired from snapshot
+  SMemTable     *imem;             // imem-table, acquired from snapshot
-  SArray*        defaultLoadColumn;// default load column
+  SArray        *defaultLoadColumn;// default load column
   SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
   SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
 
@@ -181,7 +182,72 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
   return pLocalIdList;
 }
 
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
+static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
+  size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
+  assert(sizeOfGroup >= 1 && pMeta != NULL);
+
+  // allocate buffer in order to load data blocks from file
+  SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo));
+  if (pTableCheckInfo == NULL) {
+    return NULL;
+  }
+
+  // todo apply the lastkey of table check to avoid to load header file
+  for (int32_t i = 0; i < sizeOfGroup; ++i) {
+    SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i);
+
+    size_t gsize = taosArrayGetSize(group);
+    assert(gsize > 0);
+
+    for (int32_t j = 0; j < gsize; ++j) {
+      STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j);
+
+      STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable };
+      info.tableId = ((STable*)(pKeyInfo->pTable))->tableId;
+
+      assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
+                                        info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
+
+      info.tableId.tid = info.pTableObj->tableId.tid;
+      info.tableId.uid = info.pTableObj->tableId.uid;
+
+      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+        assert(info.lastKey >= pQueryHandle->window.skey);
+      } else {
+        assert(info.lastKey <= pQueryHandle->window.skey);
+      }
+
+      taosArrayPush(pTableCheckInfo, &info);
+      tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid,
+                info.tableId.tid, info.lastKey, pQueryHandle->qinfo);
+    }
+  }
+
+  taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
+  return pTableCheckInfo;
+}
+
+static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey) {
+  size_t si = taosArrayGetSize(pTableCheckInfo);
+  SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo));
+  if (pNew == NULL) {
+    return NULL;
+  }
+
+  for (int32_t j = 0; j < si; ++j) {
+    STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j);
+    STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj};
+
+    info.tableId = pCheckInfo->tableId;
+    taosArrayPush(pNew, &info);
+  }
+
+  // it is ordered already, no need to sort again.
+  taosArraySort(pNew, tsdbCheckInfoCompar);
+  return pNew;
+}
+
+static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) {
   STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
   if (pQueryHandle == NULL) {
     goto out_of_memory;
@@ -205,9 +271,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   }
 
   tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem);
-
+  assert(pCond != NULL && pCond->numOfCols > 0);
-  size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
-  assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
 
   if (ASCENDING_TRAVERSE(pCond->order)) {
     assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
@@ -216,19 +280,17 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   }
 
   // allocate buffer in order to load data blocks from file
-  int32_t numOfCols = pCond->numOfCols;
+  pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis));
-  
-  pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
   if (pQueryHandle->statis == NULL) {
     goto out_of_memory;
   }
 
-  pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));  // todo: use list instead of array?
+  pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData));  // todo: use list instead of array?
   if (pQueryHandle->pColumns == NULL) {
     goto out_of_memory;
   }
 
-  for (int32_t i = 0; i < numOfCols; ++i) {
+  for (int32_t i = 0; i < pCond->numOfCols; ++i) {
     SColumnInfoData  colInfo = {{0}, 0};
 
     colInfo.info = pCond->colList[i];
@@ -240,61 +302,47 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
     pQueryHandle->statis[i].colId = colInfo.info.colId;
   }
 
-  pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
-  if (pQueryHandle->pTableCheckInfo == NULL) {
-    goto out_of_memory;
-  }
-
-  STsdbMeta* pMeta = tsdbGetMeta(tsdb);
-  assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
-
-  // todo apply the lastkey of table check to avoid to load header file
-  for (int32_t i = 0; i < sizeOfGroup; ++i) {
-    SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
-
-    size_t gsize = taosArrayGetSize(group);
-    assert(gsize > 0);
-
-    for (int32_t j = 0; j < gsize; ++j) {
-      STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j);
-
-      STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable };
-      info.tableId = ((STable*)(pKeyInfo->pTable))->tableId;
-
-      assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
-      info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
-
-      info.tableId.tid = info.pTableObj->tableId.tid;
-      info.tableId.uid = info.pTableObj->tableId.uid;
-
-      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-        assert(info.lastKey >= pQueryHandle->window.skey);
-      } else {
-        assert(info.lastKey <= pQueryHandle->window.skey);
-      }
-
-      taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-      tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid,
-          info.tableId.tid, info.lastKey, qinfo);
-    }
-  }
-  
-  taosArraySort(pQueryHandle->pTableCheckInfo, tsdbCheckInfoCompar);
   pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true);
 
-  tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo);
+  STsdbMeta* pMeta = tsdbGetMeta(tsdb);
+  assert(pMeta != NULL);
+
+  pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock);
+  if (pQueryHandle->pDataCols == NULL) {
+    tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo);
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto out_of_memory;
+  }
 
   tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
   tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
 
   return (TsdbQueryHandleT) pQueryHandle;
 
-out_of_memory:
+  out_of_memory:
   tsdbCleanupQueryHandle(pQueryHandle);
   terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
   return NULL;
 }
 
+TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
+  STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo);
+
+  STsdbMeta* pMeta = tsdbGetMeta(tsdb);
+  assert(pMeta != NULL);
+
+  // todo apply the lastkey of table check to avoid to load header file
+  pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta);
+  if (pQueryHandle->pTableCheckInfo == NULL) {
+    tsdbCleanupQueryHandle(pQueryHandle);
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo);
+  return (TsdbQueryHandleT) pQueryHandle;
+}
+
 TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
   pCond->twindow = changeTableGroupByLastrow(groupList);
 
@@ -689,22 +737,10 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
 }
 
 static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
-  STsdbRepo *pRepo = pQueryHandle->pTsdb;
   int64_t st = taosGetTimestampUs();
 
-  if (pCheckInfo->pDataCols == NULL) {
+  STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj);
-    STsdbMeta* pMeta = tsdbGetMeta(pRepo);
+  int32_t   code = tdInitDataCols(pQueryHandle->pDataCols, pSchema);
-
-    pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
-    if (pCheckInfo->pDataCols == NULL) {
-      tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo);
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      goto _error;
-    }
-  }
-
-  STSchema* pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj);
-  int32_t code = tdInitDataCols(pCheckInfo->pDataCols, pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -1924,77 +1960,33 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
       return true;
     } else {
-      STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
+      STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
-      if (pSecQueryHandle == NULL) {
+      STsdbQueryCond cond = {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+          .order = TSDB_ORDER_ASC,
+          .numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)),
+          .twindow = win};
+
+      cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
+      if (cond.colList == NULL) {
+        terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
         return false;
       }
 
-      pSecQueryHandle->order       = TSDB_ORDER_ASC;
+      for(int32_t i = 0; i < cond.numOfCols; ++i) {
-      pSecQueryHandle->window      = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
+        SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
-      pSecQueryHandle->pTsdb       = pQueryHandle->pTsdb;
+        memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
-      pSecQueryHandle->type        = TSDB_QUERY_TYPE_ALL;
-      pSecQueryHandle->cur.fid     = -1;
-      pSecQueryHandle->cur.win     = TSWINDOW_INITIALIZER;
-      pSecQueryHandle->checkFiles  = true;
-      pSecQueryHandle->activeIndex = 0;
-      pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock;
-
-      if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        free(pSecQueryHandle);
-        return false;
       }
 
-      tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
+      STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo);
 
-      // allocate buffer in order to load data blocks from file
+      taosTFree(cond.colList);
-      int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
 
-      pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
+      pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
-      pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+      if (pSecQueryHandle->pTableCheckInfo == NULL) {
-      if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
         tsdbCleanupQueryHandle(pSecQueryHandle);
         return false;
       }
 
-      for (int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData colInfo = {{0}, 0};
-        SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
-
-        colInfo.info = pCol->info;
-        colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes);
-        if (colInfo.pData == NULL) {
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          tsdbCleanupQueryHandle(pSecQueryHandle);
-          return false;
-        }
-
-        taosArrayPush(pSecQueryHandle->pColumns, &colInfo);
-      }
-
-      size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-      pSecQueryHandle->pTableCheckInfo = taosArrayInit(si, sizeof(STableCheckInfo));
-      STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
-      assert(pMeta != NULL);
-
-      for (int32_t j = 0; j < si; ++j) {
-        STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
-        STableCheckInfo info = {
-            .lastKey = pSecQueryHandle->window.skey,
-            .pTableObj = pCheckInfo->pTableObj,
-        };
-
-        info.tableId = pCheckInfo->tableId;
-
-        taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info);
-      }
-
-      tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo);
-      tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo);
-      pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn);
-
       if (!tsdbNextDataBlock((void*) pSecQueryHandle)) {
         tsdbCleanupQueryHandle(pSecQueryHandle);
         return false;
@@ -2003,6 +1995,9 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
       tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
 
+      int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle));
+      size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo);
+
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
@@ -2016,11 +2011,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
 
       // it is ascending order
-      pQueryHandle->cur.win  = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
+      pQueryHandle->order    = TSDB_ORDER_DESC;
       pQueryHandle->window   = pQueryHandle->cur.win;
+      pQueryHandle->cur.win  = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
       pQueryHandle->cur.rows = 2;
       pQueryHandle->cur.mixBlock = true;
-      pQueryHandle->order = TSDB_ORDER_DESC;
 
       int32_t step = -1;// one step for ascending order traverse
       for (int32_t j = 0; j < si; ++j) {
@@ -2284,7 +2279,7 @@ void filterPrepare(void* expr, void* param) {
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
   } else {
-    pInfo->q = calloc(1, pSchema->bytes);
+    pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE);   // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
     tVariantDump(pCond, pInfo->q, pSchema->type, true);
   }
 }
@@ -2639,8 +2634,7 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
   SArray* group = taosArrayInit(1, sizeof(STableKeyInfo));
 
-  int32_t i = 0;
+  for(int32_t i = 0; i < size; ++i) {
-  for(; i < size; ++i) {
     STableIdInfo *id = taosArrayGet(pTableIdList, i);
 
     STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid);
@@ -2665,8 +2659,12 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa
     return terrno;
   }
 
-  pGroupInfo->numOfTables = i;
+  pGroupInfo->numOfTables = taosArrayGetSize(group);
+  if (pGroupInfo->numOfTables > 0) {
     taosArrayPush(pGroupInfo->pGroupList, &group);
+  } else {
+    taosArrayDestroy(group);
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -2683,8 +2681,6 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
       STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
       destroyTableMemIterator(pTableCheckInfo);
 
-      tdFreeDataCols(pTableCheckInfo->pDataCols);
-      pTableCheckInfo->pDataCols = NULL;
       taosTFree(pTableCheckInfo->pCompInfo);
     }
     taosArrayDestroy(pQueryHandle->pTableCheckInfo);
@@ -2708,6 +2704,9 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
 
   tsdbDestroyHelper(&pQueryHandle->rhelper);
 
+  tdFreeDataCols(pQueryHandle->pDataCols);
+  pQueryHandle->pDataCols = NULL;
+
   SIOCostSummary* pCost = &pQueryHandle->cost;
   tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p",
       pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);

src/util/src/tcache.c

@@ -107,12 +107,14 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo
   free(pNode);
 }
 
-static FORCE_INLINE void doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) {
+static FORCE_INLINE STrashElem* doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) {
   if (pElem->pData->signature != (uint64_t) pElem->pData) {
     uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData);
-    return;
+    return NULL;
   }
 
+  STrashElem* next = pElem->next;
+
   pCacheObj->numOfElemsInTrash--;
   if (pElem->prev) {
     pElem->prev->next = pElem->next;
@@ -120,9 +122,15 @@ static FORCE_INLINE void doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem
     pCacheObj->pTrash = pElem->next;
   }
 
-  if (pElem->next) {
+  if (next) {
-    pElem->next->prev = pElem->prev;
+    next->prev = pElem->prev;
   }
+
+  if (pCacheObj->numOfElemsInTrash == 0) {
+    assert(pCacheObj->pTrash == NULL);
+  }
+
+  return next;
 }
 
 static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem *pElem) {
@@ -550,8 +558,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   pCacheObj->numOfElemsInTrash++;
   __cache_unlock(pCacheObj);
 
-  uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data,
+  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
-      pCacheObj->numOfElemsInTrash);
+      pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }
 
 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
@@ -559,31 +567,30 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
 
   if (pCacheObj->numOfElemsInTrash == 0) {
     if (pCacheObj->pTrash != NULL) {
+      pCacheObj->pTrash = NULL;
       uError("cache:%s, key:inconsistency data in cache, numOfElem in trashcan:%d", pCacheObj->name, pCacheObj->numOfElemsInTrash);
     }
 
-    pCacheObj->pTrash = NULL;
     __cache_unlock(pCacheObj);
     return;
   }
 
-  STrashElem *pElem = pCacheObj->pTrash;
+  const char* stat[] = {"false", "true"};
+  uDebug("cache:%s start to cleanup trashcan, numOfElem in trashcan:%d, free:%s", pCacheObj->name,
+      pCacheObj->numOfElemsInTrash, (force? stat[1]:stat[0]));
 
+  STrashElem *pElem = pCacheObj->pTrash;
   while (pElem) {
     T_REF_VAL_CHECK(pElem->pData);
-    if (pElem->next == pElem) {
+    assert(pElem->next != pElem && pElem->prev != pElem);
-      pElem->next = NULL;
-    }
 
     if (force || (T_REF_VAL_GET(pElem->pData) == 0)) {
       uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, pElem->pData->data,
              pCacheObj->numOfElemsInTrash - 1);
 
-      STrashElem *p = pElem;
+      doRemoveElemInTrashcan(pCacheObj, pElem);
-      pElem = pElem->next;
+      doDestroyTrashcanElem(pCacheObj, pElem);
-
+      pElem = pCacheObj->pTrash;
-      doRemoveElemInTrashcan(pCacheObj, p);
-      doDestroyTrashcanElem(pCacheObj, p);
     } else {
       pElem = pElem->next;
     }

src/util/src/tqueue.c

@@ -263,6 +263,7 @@ void taosCloseQset(taos_qset param) {
 // thread to exit.
 void taosQsetThreadResume(taos_qset param) {
   STaosQset *qset = (STaosQset *)param;
+  uDebug("qset:%p, it will exit", qset);
   tsem_post(&qset->sem);
 }
 

src/vnode/CMakeLists.txt

@@ -4,6 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)

src/vnode/inc/vnodeInt.h

@@ -62,6 +62,7 @@ typedef struct {
 } SVnodeObj;
 
 int  vnodeWriteToQueue(void *param, void *pHead, int type);
+int  vnodeWriteCqMsgToQueue(void *param, void *pHead, int type);
 void vnodeInitWriteFp(void);
 void vnodeInitReadFp(void);
 

src/vnode/src/vnodeMain.c

@@ -239,8 +239,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   code = vnodeReadVersion(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
-    vnodeCleanUp(pVnode);
+    vError("vgId:%d, failed to read version, generate it from data file", pVnode->vgId);
-    return code;
+    // Allow vnode start even when read version fails, set version as walVersion or zero
+    // vnodeCleanUp(pVnode);
+    // return code;
   }
 
   pVnode->fversion = pVnode->version;
@@ -257,7 +259,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   strcpy(cqCfg.pass, tsInternalPass);
   strcpy(cqCfg.db, pVnode->db);
   cqCfg.vgId = vnode;
-  cqCfg.cqWrite = vnodeWriteToQueue;
+  cqCfg.cqWrite = vnodeWriteCqMsgToQueue;
   pVnode->cq = cqOpen(pVnode, &cqCfg);
   if (pVnode->cq == NULL) {
     vnodeCleanUp(pVnode);
@@ -292,6 +294,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   }
 
   walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
+  if (pVnode->version == 0) {
+    pVnode->version = walGetVersion(pVnode->wal);
+  }
 
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
@@ -947,6 +952,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
+  fflush(fp);
   fclose(fp);
 
   vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion);
@@ -960,7 +966,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
   cJSON  *root = NULL;
   int     maxLen = 100;
 
-  terrno = TSDB_CODE_VND_APP_ERROR;
+  terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
   sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
   FILE *fp = fopen(versionFile, "r");
   if (!fp) {
@@ -999,6 +1005,6 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
 PARSE_OVER:
   taosTFree(content);
   cJSON_Delete(root);
-  if(fp) fclose(fp);
+  if (fp) fclose(fp);
   return terrno;
 }

src/vnode/src/vnodeRead.c

@@ -54,13 +54,13 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
 
   // tsdb may be in reset state
   if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING)
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;
-    return TSDB_CODE_APP_NOT_READY;
 
   // TODO: Later, let slave to support query
   // if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
   if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role);
+    vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[msgType],
+           pVnode->syncCfg.replica, syncRole[pVnode->role]);
     return TSDB_CODE_APP_NOT_READY;
   }
 
@@ -80,7 +80,7 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) {
   taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }
 
-static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle, bool* freeHandle) {
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle) {
   bool continueExec = false;
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -106,54 +106,55 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle,
   return code;
 }
 
-static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
   pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
   pRet->len = sizeof(SRetrieveTableRsp);
 
   memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  SRetrieveTableRsp* pRsp = pRet->rsp;
+  SRetrieveTableRsp *pRsp = pRet->rsp;
 
   pRsp->completed = true;
 }
 
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
-  void    *pCont = pReadMsg->pCont;
+  void *   pCont = pReadMsg->pCont;
   int32_t  contLen = pReadMsg->contLen;
   SRspRet *pRet = &pReadMsg->rspRet;
 
-  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont;
+  SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
   memset(pRet, 0, sizeof(SRspRet));
 
   // qHandle needs to be freed correctly
   if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-    SRetrieveTableMsg* killQueryMsg = (SRetrieveTableMsg*) pReadMsg->pCont;
+    SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
     killQueryMsg->free = htons(killQueryMsg->free);
     killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
 
-    vWarn("QInfo:%p connection %p broken, kill query", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
 
-    void** qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) killQueryMsg->qhandle);
+    void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
     if (qhandle == NULL || *qhandle == NULL) {
-      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+      vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
+            pReadMsg->rpcMsg.handle);
     } else {
-      assert(*qhandle == (void*) killQueryMsg->qhandle);
+      assert(*qhandle == (void *)killQueryMsg->qhandle);
 
       qKillQuery(*qhandle);
-      qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true);
     }
 
     return TSDB_CODE_TSC_QUERY_CANCELLED;
   }
 
   int32_t code = TSDB_CODE_SUCCESS;
-  void**  handle = NULL;
+  void ** handle = NULL;
 
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
     code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
 
-    SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
+    SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code = code;
     pRsp->qhandle = 0;
 
@@ -163,7 +164,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
     // current connect is broken
     if (code == TSDB_CODE_SUCCESS) {
-      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo);
+      handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t)pQInfo);
       if (handle == NULL) {  // failed to register qhandle, todo add error test case
         vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
                tstrerror(pRsp->code));
@@ -171,13 +172,15 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
         qDestroyQueryInfo(pQInfo);  // destroy it directly
       } else {
         assert(*handle == pQInfo);
-        pRsp->qhandle = htobe64((uint64_t) pQInfo);
+        pRsp->qhandle = htobe64((uint64_t)pQInfo);
       }
 
-      if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+      if (handle != NULL &&
-        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+          vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+        vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
+               pReadMsg->rpcMsg.handle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-        qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+        qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
         return pRsp->code;
       }
     } else {
@@ -190,7 +193,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     }
   } else {
     assert(pCont != NULL);
-    void** qhandle = (void**) pCont;
+    void **qhandle = (void **)pCont;
 
     vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
 
@@ -233,15 +236,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->free = htons(pRetrieve->free);
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
+         pRetrieve->free, pReadMsg->rpcMsg.handle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
   int32_t code = TSDB_CODE_SUCCESS;
-  void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
+  void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
-  if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
+  if (handle == NULL || (*handle) != (void *)pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
-    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
+    vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void *)pRetrieve->qhandle);
 
     vnodeBuildNoResultQueryRsp(pRet);
     return code;
@@ -250,7 +254,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   if (pRetrieve->free == 1) {
     vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
     qKillQuery(*handle);
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
 
     vnodeBuildNoResultQueryRsp(pRet);
     code = TSDB_CODE_TSC_QUERY_CANCELLED;
@@ -259,9 +263,11 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   // register the qhandle to connect to quit query immediate if connection is broken
   if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
+           pReadMsg->rpcMsg.handle);
     code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qKillQuery(*handle);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
     return code;
   }
 
@@ -270,14 +276,14 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
   if (code != TSDB_CODE_SUCCESS) {
-    //TODO handle malloc failure
+    // TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     pRet->len = sizeof(SRetrieveTableRsp);
     memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
     freeHandle = true;
   } else {  // result is not ready, return immediately
     if (!buildRes) {
-      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
       return TSDB_CODE_QRY_NOT_READY;
     }
 
@@ -287,7 +293,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   // If qhandle is not added into vread queue, the query should be completed already or paused with error.
   // Here free qhandle immediately
   if (freeHandle) {
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
+    qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
   }
 
   return code;
@@ -295,13 +301,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
 // notify connection(handle) that current qhandle is created, if current connection from
 // client is broken, the query needs to be killed immediately.
-int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
+int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) {
-  SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
+  SRetrieveTableMsg *killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
-  killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
+  killQueryMsg->qhandle = htobe64((uint64_t)qhandle);
   killQueryMsg->free = htons(1);
   killQueryMsg->header.vgId = htonl(vgId);
   killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
   vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle);
-  return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
+  return rpcReportProgress(handle, (char *)killQueryMsg, sizeof(SRetrieveTableMsg));
 }

src/vnode/src/vnodeWrite.c

@@ -22,9 +22,11 @@
 #include "tutil.h"
 #include "tsdb.h"
 #include "twal.h"
+#include "tsync.h"
 #include "tdataformat.h"
 #include "vnode.h"
 #include "vnodeInt.h"
+#include "syncInt.h"
 #include "tcq.h"
 
 static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *, SRspRet *);
@@ -47,7 +49,7 @@ void vnodeInitWriteFp(void) {
 int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   int32_t    code = 0;
   SVnodeObj *pVnode = (SVnodeObj *)param1;
-  SWalHead  *pHead = param2;
+  SWalHead * pHead = param2;
 
   if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
     vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
@@ -61,40 +63,40 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
 
   // tsdb may be in reset state
   if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY;
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING) 
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY;
-    return TSDB_CODE_APP_NOT_READY;
 
   if (pHead->version == 0) {  // from client or CQ
     if (pVnode->status != TAOS_VN_STATUS_READY) {
-      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->status);
+      vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->status);
       return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
     }
 
     if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->syncCfg.replica, pVnode->role);
+      vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType],
+             pVnode->syncCfg.replica, syncRole[pVnode->role]);
       return TSDB_CODE_APP_NOT_READY;
     }
 
     // assign version
-    pVnode->version++;
+    pHead->version = pVnode->version + 1;
-    pHead->version = pVnode->version;
+    if (pVnode->delay) usleep(pVnode->delay * 1000);
-    if (pVnode->delay) usleep(pVnode->delay*1000);
 
   } else {  // from wal or forward
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
   }
 
-  pVnode->version = pHead->version;
+  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
+  int32_t syncCode = 0;
+  syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
+  if (syncCode < 0) return syncCode;
 
   // write into WAL
   code = walWrite(pVnode->wal, pHead);
   if (code < 0) return code;
 
-  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync 
+  pVnode->version = pHead->version;
-  int32_t syncCode = 0;
-  syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
-  if (syncCode < 0) return syncCode;
 
   // write data locally
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
@@ -189,9 +191,28 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR
   return TSDB_CODE_SUCCESS;
 }
 
+
+int vnodeWriteCqMsgToQueue(void *param, void *data, int type) {
+  SVnodeObj *pVnode = param;
+  SWalHead * pHead = data;
+
+  int size = sizeof(SWalHead) + pHead->len;
+  SSyncHead *pSync = (SSyncHead*) taosAllocateQitem(size + sizeof(SSyncHead));
+  SWalHead *pWal = (SWalHead *)(pSync + 1);
+  memcpy(pWal, pHead, size);
+
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  vDebug("CQ: vgId:%d, get vnode wqueue, refCount:%d", pVnode->vgId, pVnode->refCount);
+
+  taosWriteQitem(pVnode->wqueue, type, pSync);
+
+  return 0;
+}
+
+
 int vnodeWriteToQueue(void *param, void *data, int type) {
   SVnodeObj *pVnode = param;
-  SWalHead *pHead = data;
+  SWalHead * pHead = data;
 
   int size = sizeof(SWalHead) + pHead->len;
   SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
@@ -204,4 +225,3 @@ int vnodeWriteToQueue(void *param, void *data, int type) {
 
   return 0;
 }
-