Merge branch 'develop' into feature/pyconn

.gitignore

@@ -11,6 +11,7 @@ debs/
 rpms/
 mac/
 *.pyc
+.mypy_cache
 *.tmp
 *.swp
 src/connector/nodejs/node_modules/
@@ -33,5 +34,35 @@ Target/
 *.failed
 *.sql
 sim/
+psim/
+pysim/
+*.out
 *DS_Store
 
+# Doxygen Generated files
+html/
+/.vs
+/CMakeFiles/3.10.2
+/CMakeCache.txt
+/Makefile
+/*.cmake
+/src/cq/test/CMakeFiles/cqtest.dir/*.cmake
+*.cmake
+/src/cq/test/CMakeFiles/cqtest.dir/*.make
+*.make
+link.txt
+*.internal
+*.includecache
+*.marks
+Makefile
+CMakeError.log
+*.log
+/CMakeFiles/CMakeRuleHashes.txt
+/CMakeFiles/Makefile2
+/CMakeFiles/TargetDirectories.txt
+/CMakeFiles/cmake.check_cache
+/out/isenseconfig/WSL-Clang-Debug
+/out/isenseconfig/WSL-GCC-Debug
+/test/cfg
+/src/.vs
+*.o

.travis.yml

@@ -6,6 +6,7 @@
 #
 matrix:
   - os: linux
+    dist: bionic
     language: c
 
     git:
@@ -25,6 +26,7 @@ matrix:
           - python3-pip
           - python3-setuptools
           - valgrind
+          - psmisc
 
     before_script:
       - cd ${TRAVIS_BUILD_DIR}
@@ -36,6 +38,7 @@ matrix:
       - make > /dev/null
 
     after_success:
+      - travis_wait 20
       - |-
         case $TRAVIS_OS_NAME in
           linux)
@@ -46,10 +49,10 @@ matrix:
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
-          ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $?
+          ./test-all.sh smoke || travis_terminate $?
 
           cd ${TRAVIS_BUILD_DIR}/tests/pytest
-          ./smoketest.sh -g 2>&1 | tee mem-error-out.txt
+          ./valgrind-test.sh 2>&1 > mem-error-out.log
           sleep 1
 
           # Color setting
@@ -59,12 +62,12 @@ matrix:
           GREEN_UNDERLINE='\033[4;32m'
           NC='\033[0m'
 
-          grep 'ERROR SUMMARY' mem-error-out.txt | uniq | tee uniq-mem-error-out.txt
+          grep 'start to execute\|ERROR SUMMARY' mem-error-out.log|grep -v 'grep'|uniq|tee uniq-mem-error-out.log
 
-          for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` 
+          for memError in `grep 'ERROR SUMMARY' uniq-mem-error-out.log | awk '{print $4}'`
           do
             if [ -n "$memError" ]; then
-              if [ "$memError" -gt 16 ]; then
+              if [ "$memError" -gt 12 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
                          More than our threshold! ## ${NC}"
                 travis_terminate $memError
@@ -72,11 +75,11 @@ matrix:
             fi
           done
 
-          grep 'definitely lost' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt
+          grep 'start to execute\|definitely lost:' mem-error-out.log|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.log
-          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` 
+          for defiMemError in `grep 'definitely lost:' uniq-definitely-lost-out.log | awk '{print $7}'`
           do
             if [ -n "$defiMemError" ]; then
-              if [ "$defiMemError" -gt 16 ]; then
+              if [ "$defiMemError" -gt 13 ]; then
                 echo -e "${RED} ## Memory errors number valgrind reports \
                          Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
                 travis_terminate $defiMemError
@@ -88,6 +91,7 @@ matrix:
         esac
 
   - os: linux
+    dist: bionic
     language: c
     compiler: gcc
     env: COVERITY_SCAN=true
@@ -107,7 +111,7 @@ matrix:
           description: TDengine
 
         # Where email notification of build analysis results will be sent
-        notification_email: sdsang@taosdata.com
+        notification_email: sdsang@taosdata.com, slguan@taosdata.com
 
         # Commands to prepare for build_command
         # ** likely specific to your build **
@@ -115,7 +119,7 @@ matrix:
 
         # The command that will be added as an argument to "cov-build" to compile your project for analysis,
         # ** likely specific to your build **
-        build_command: make > /dev/null
+        build_command: make
 
         # Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'.
         # Take care in resource usage, and consider the build frequency allowances per
@@ -123,6 +127,7 @@ matrix:
         branch_pattern: coverity_scan
 
   - os: linux
+    dist: bionic
     language: c
     compiler: gcc
     env: ENV_COVER=true
@@ -141,6 +146,7 @@ matrix:
           - python3-pip
           - python3-setuptools
           - lcov
+          - psmisc
 
     before_script:
       - cd ${TRAVIS_BUILD_DIR}
@@ -163,17 +169,16 @@ matrix:
 
           cd ${TRAVIS_BUILD_DIR}/tests
 
-          ./test-all.sh
+          ./test-all.sh smoke COVER
 
-          if [ "$?" -ne "0" ]; then
+          TEST_RESULT=$?
-            travis_terminate $?
-          fi
 
           pkill taosd
           sleep 1
 
           cd ${TRAVIS_BUILD_DIR}
           lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
+          lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info
           lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $?
 
           gem install coveralls-lcov
@@ -199,10 +204,37 @@ matrix:
             echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
           fi
 
+          if [ "$TEST_RESULT" -ne "0" ]; then
+            travis_terminate $?
+          fi
           ;;
         esac
 
   - os: linux
+    dist: trusty
+    language: c
+    git:
+      - depth: 1
+
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+    env: 
+      - DESC="trusty/gcc-4.8 build"
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
+
+    script:
+      - cmake .. > /dev/null
+      - make > /dev/null
+
+  - os: linux
+    dist: bionic
     language: c
     compiler: clang
     env: DESC="linux/clang build"

CMakeSettings.json

@@ -0,0 +1,25 @@
+{
+  "configurations": [
+    {
+      "name": "WSL-GCC-Debug",
+      "generator": "Unix Makefiles",
+      "configurationType": "Debug",
+      "buildRoot": "${projectDir}\\build\\",
+      "installRoot": "${projectDir}\\build\\",
+      "cmakeExecutable": "/usr/bin/cmake",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "linux_x64" ],
+      "wslPath": "${defaultWSLPath}",
+      "addressSanitizerRuntimeFlags": "detect_leaks=0",
+      "variables": [
+        {
+          "name": "CMAKE_INSTALL_PREFIX",
+          "value": "/mnt/d/TDengine/TDengine/build",
+          "type": "PATH"
+        }
+      ]
+    }
+  ]
+}

README.md

@@ -44,7 +44,7 @@ sudo apt-get install maven
 Build TDengine:
 
 ```
-mkdir build && cd build
+mkdir debug && cd debug
 cmake .. && cmake --build .
 ```
 
@@ -115,251 +115,6 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo
 - [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector)
 - [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector)
 
-# How to run the test cases and how to add a new test case?
-
-### Prepare development environment
-
-1.  sudo apt install
-    build-essential cmake net-tools python-pip python-setuptools python3-pip
-    python3-setuptools valgrind
-
-2.  git clone <https://github.com/taosdata/TDengine>; cd TDengine
-
-3.  mkdir debug; cd debug; cmake ..; make ; sudo make install
-
-4.  pip install src/connector/python/linux/python2 ; pip3 install
-    src/connector/python/linux/python3
-
-### How to run TSIM test suite
-
-1.  cd \<TDengine\>/tests/script
-
-2.  sudo ./test.sh
-
-### How to run Python test suite
-
-1.  cd \<TDengine\>/tests/pytest
-
-2.  ./smoketest.sh \# for smoke test
-
-3.  ./smoketest.sh -g \# for memory leak detection test with valgrind
-
-4.  ./fulltest.sh \# for full test
-
->   Note1: TDengine daemon's configuration and data files are stored in
->   \<TDengine\>/sim directory. As a historical design, it's same place with
->   TSIM script. So after the TSIM script ran with sudo privilege, the directory
->   has been used by TSIM then the python script cannot write it by a normal
->   user. You need to remove the directory completely first before running the
->   Python test case. We should consider using two different locations to store
->   for TSIM and Python script.
-
->   Note2: if you need to debug crash problem with a core dump, you need
->   manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited"
->   before the script line. Then you can look for the core file in
->   \<TDengine\>/tests/pytest after the program crash.
-
-### How to add a new test case
-
-**1. add a new TSIM test cases:**
-
-TSIM test cases are now included in the new development branch and can be
-added to the TDengine/tests/script/test.sh script based on the manual test
-methods necessary to add test cases as described above.
-
-**2. add a new Python test cases:**
-
-**2.1 Please refer to \<TDengine\>/tests/pytest/insert/basic.py to add a new
-test case.** The new test case must implement 3 functions, where self.init()
-and self.stop() simply copy the contents of insert/basic.py and the test
-logic is implemented in self.run(). You can refer to the code in the util
-directory for more information.
-
-**2.2 Edit smoketest.sh to add the path and filename of the new test case**
-
-Note: The Python test framework may continue to be improved in the future,
-hopefully, to provide more functionality and ease of writing test cases. The
-method of writing the test case above does not exclude that it will also be
-affected.
-
-**2.3 What test.py does in detail:**
-
-test.py is the entry program for test case execution and monitoring.
-
-test.py has the following functions.
-
-\-f --file, Specifies the test case file name to be executed
--p --path, Specifies deployment path
-
-\-m --master, Specifies the master server IP for cluster deployment 
--c--cluster, test cluster function
--s--stop, terminates all running nodes
-
-\-g--valgrind, load valgrind for memory leak detection test
-
-\-h--help, display help
-
-**2.4 What util/log.py does in detail:**
-
-log.py is quite simple, the main thing is that you can print the output in
-different colors as needed. The success() should be called for successful
-test case execution and the success() will print green text. The exit() will
-print red text and exit the program, exit() should be called for test
-failure.
-
-**util/log.py**
-
-...
-
-    def info(self, info):
-
-        printf("%s %s" % (datetime.datetime.now(), info))
-
- 
-
-    def sleep(self, sec):
-
-        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
-
-        time.sleep(sec)
-
- 
-
-    def debug(self, err):
-
-        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
- 
-
-    def success(self, info):
-
-        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info))
-
- 
-
-    def notice(self, err):
-
-        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
- 
-
-    def exit(self, err):
-
-        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err))
-
-        sys.exit(1)
-
- 
-
-    def printNoPrefix(self, info):
-
-        printf("\\033[1;36m%s\\033[0m" % (info)
-
-...
-
-**2.5 What util/sql.py does in detail:**
-
-SQL.py is mainly used to execute SQL statements to manipulate the database,
-and the code is extracted and commented as follows:
-
-**util/sql.py**
-
-\# prepare() is mainly used to set up the environment for testing table and
-data, and to set up the database db for testing. do not call prepare() if you
-need to test the database operation command.
-
-def prepare(self):
-
-tdLog.info("prepare database:db")
-
-self.cursor.execute('reset query cache')
-
-self.cursor.execute('drop database if exists db')
-
-self.cursor.execute('create database db')
-
-self.cursor.execute('use db')
-
-...
-
-\# query() is mainly used to execute select statements for normal syntax input
-
-def query(self, sql):
-
-...
-
-\# error() is mainly used to execute the select statement with the wrong syntax
-input, the error will be caught as a reasonable behavior, if not caught it will
-prove that the test failed
-
-def error()
-
-...
-
-\# checkRows() is used to check the number of returned lines after calling
-query(select ...) after calling the query(select ...) to check the number of
-rows of returned results.
-
-def checkRows(self, expectRows):
-
-...
-
-\# checkData() is used to check the returned result data after calling
-query(select ...) after the query(select ...) is called, failure to meet
-expectation is
-
-def checkData(self, row, col, data):
-
-...
-
-\# getData() returns the result data after calling query(select ...) to return
-the resulting data after calling query(select ...)
-
-def getData(self, row, col):
-
-...
-
-\# execute() used to execute sql and return the number of affected rows
-
-def execute(self, sql):
-
-...
-
-\# executeTimes() Multiple executions of the same sql statement
-
-def executeTimes(self, sql, times):
-
-...
-
-\# CheckAffectedRows() Check if the number of affected rows is as expected
-
-def checkAffectedRows(self, expectAffectedRows):
-
-...
-
->   Note: Both Python2 and Python3 are currently supported by the Python test
->   case. Since Python2 is no longer officially supported by January 1, 2020, it
->   is recommended that subsequent test case development be guaranteed to run
->   correctly on Python3. For Python2, please consider being compatible if
->   appropriate without additional
->   burden. <https://nakedsecurity.sophos.com/2020/01/03/python-is-dead-long-live-python/> 
-
-### CI Covenant submission adoption principle.
-
--   Every commit / PR compilation must pass. Currently, the warning is treated
-    as an error, so the warning must also be resolved.
-
--   Test cases that already exist must pass.
-
--   Because CI is very important to support build and automatically test
-    procedure, it is necessary to manually test the test case before adding it
-    and do as many iterations as possible to ensure that the test case provides
-    stable and reliable test results when added.
-
->   Note: In the future, according to the requirements and test development
->   progress will add stress testing, performance testing, code style, 
->   and other features based on functional testing.
-
 ### Third Party Connectors
 
 The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them.
@@ -367,6 +122,10 @@ The TDengine community has also kindly built some of their own connectors! Follo
 - [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust)
 - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos)
 
+# How to run the test cases and how to add a new test case?
+  TDengine's test framework and all test cases are fully open source.
+  Please refer to [this document](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md) for how to run test and develop new test case.
+
 # TDengine Roadmap
 - Support event-driven stream computing
 - Support user defined functions

deps/CMakeLists.txt

@@ -7,3 +7,4 @@ ADD_SUBDIRECTORY(regex)
 ADD_SUBDIRECTORY(iconv)
 ADD_SUBDIRECTORY(lz4)
 ADD_SUBDIRECTORY(cJson)
+ADD_SUBDIRECTORY(MQTT-C)

deps/MQTT-C/CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 3.5)
+project(MQTT-C VERSION 1.1.2 LANGUAGES C)
+
+# MQTT-C build options
+option(MQTT_C_OpenSSL_SUPPORT "Build MQTT-C with OpenSSL support?" OFF)
+option(MQTT_C_MbedTLS_SUPPORT "Build MQTT-C with mbed TLS support?" OFF)
+option(MQTT_C_EXAMPLES "Build MQTT-C examples?" ON)
+option(MQTT_C_TESTS "Build MQTT-C tests?" OFF)
+
+list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+# MQTT-C library
+add_library(mqttc STATIC 
+    src/mqtt_pal.c
+    src/mqtt.c
+)
+target_include_directories(mqttc PUBLIC include)
+target_link_libraries(mqttc PUBLIC 
+    $<$<C_COMPILER_ID:MSVS>:ws2_32>
+)
+
+
+# Configure with OpenSSL support
+if(MQTT_C_OpenSSL_SUPPORT)
+    find_package(OpenSSL REQUIRED)
+    target_link_libraries(mqttc INTERFACE OpenSSL::SSL)
+    target_compile_definitions(mqttc PUBLIC MQTT_USE_BIO)
+endif()
+
+# Configure with mbed TLS support
+if(MQTT_C_MbedTLS_SUPPORT)
+    find_package(MbedTLS REQUIRED)
+    target_include_directories(mqttc PUBLIC ${MBEDTLS_INCLUDE_DIRS})
+    target_link_libraries(mqttc INTERFACE ${MBEDTLS_LIBRARY})
+    target_compile_definitions(mqttc PUBLIC MQTT_USE_MBEDTLS)
+endif()
+
+# Build examples
+if(MQTT_C_EXAMPLES)
+    find_package(Threads REQUIRED)
+
+    if(MQTT_C_OpenSSL_SUPPORT)
+        add_executable(bio_publisher examples/bio_publisher.c)
+        target_link_libraries(bio_publisher Threads::Threads mqttc)
+
+        add_executable(openssl_publisher examples/openssl_publisher.c)
+        target_link_libraries(openssl_publisher Threads::Threads mqttc)
+    elseif(MQTT_C_MbedTLS_SUPPORT)
+        add_executable(mbedtls_publisher examples/mbedtls_publisher.c)
+        target_link_libraries(mbedtls_publisher Threads::Threads mqttc ${MBEDX509_LIBRARY} ${MBEDCRYPTO_LIBRARY})
+    else()
+        add_executable(simple_publisher examples/simple_publisher.c)
+        target_link_libraries(simple_publisher Threads::Threads mqttc)
+
+        add_executable(simple_subscriber examples/simple_subscriber.c)
+        target_link_libraries(simple_subscriber Threads::Threads mqttc)
+
+        add_executable(reconnect_subscriber examples/reconnect_subscriber.c)
+        target_link_libraries(reconnect_subscriber Threads::Threads mqttc)
+    endif()
+endif()
+
+# Build tests
+if(MQTT_C_TESTS)    
+    find_path(CMOCKA_INCLUDE_DIR cmocka.h)
+    find_library(CMOCKA_LIBRARY cmocka)
+    if((NOT CMOCKA_INCLUDE_DIR) OR (NOT CMOCKA_LIBRARY))
+        message(FATAL_ERROR "Failed to find cmocka! Add cmocka's install prefix to CMAKE_PREFIX_PATH to resolve this error.")
+    endif()
+
+    add_executable(tests tests.c)
+    target_link_libraries(tests ${CMOCKA_LIBRARY} mqttc)
+    target_include_directories(tests PRIVATE ${CMOCKA_INCLUDE_DIR})
+endif()
+
+# Install includes and library
+# install(TARGETS mqttc 
+#     DESTINATION lib
+# )
+# install(DIRECTORY include/
+#    DESTINATION include)

deps/MQTT-C/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Liam Bindle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deps/MQTT-C/README.md

@@ -0,0 +1,108 @@
+<p align="right">
+    <a href="https://github.com/LiamBindle/MQTT-C/stargazers"><img src="https://img.shields.io/github/stars/LiamBindle/MQTT-C.svg?style=social&label=Star" style="margin-left:5em"></a>
+    <a href="https://github.com/LiamBindle/MQTT-C/network/members"><img src="https://img.shields.io/github/forks/LiamBindle/MQTT-C.svg?style=social&label=Fork"></a>
+</p>
+
+<p align="center">
+    <img width="70%" src="docs/mqtt-c-logo.png"><br>
+    <a href="https://liambindle.ca/MQTT-C"><img src="https://img.shields.io/badge/docs-passing-brightgreen.svg"></a>
+    <a href="https://github.com/LiamBindle/MQTT-C/issues"><img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg"></a>
+    <a href="https://GitHub.com/LiamBindle/MQTT-C/issues/"><img src="https://img.shields.io/github/issues/LiamBindle/MQTT-C.svg"></a>
+    <a href="https://github.com/LiamBindle/MQTT-C/issues"><img src="https://img.shields.io/github/issues-closed/LiamBindle/MQTT-C.svg"></a>
+    <a href="https://github.com/LiamBindle/MQTT-C/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg"></a>
+</p>
+
+# 
+
+MQTT-C is an [MQTT v3.1.1](http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) 
+client written in C. MQTT is a lightweight publisher-subscriber-based messaging protocol that is
+commonly used in IoT and networking applications where high-latency and low data-rate links 
+are expected. The purpose of MQTT-C is to provide a **portable** MQTT client, **written in C**, 
+for embedded systems and PC's alike. MQTT-C does this by providing a transparent Platform 
+Abstraction Layer (PAL) which makes porting to new platforms easy. MQTT-C is completely 
+thread-safe but can also run perfectly fine on single-threaded systems making MQTT-C 
+well-suited for embedded systems and microcontrollers. Finally, MQTT-C is small; there are only 
+two source files totalling less than 2000 lines.
+
+#### A note from the author
+It's been great to hear about all the places MQTT-C is being used! Please don't hesitate
+to get in touch with me or submit issues on GitHub!
+
+## Getting Started
+To use MQTT-C you first instantiate a `struct mqtt_client` and initialize it by calling
+@ref mqtt_init.
+```c
+    struct mqtt_client client; /* instantiate the client */
+    mqtt_init(&client, ...);   /* initialize the client */
+```
+Once your client is initialized you need to connect to an MQTT broker.
+```c
+    mqtt_connect(&client, ...); /* send a connection request to the broker. */
+```
+At this point the client is ready to use! For example, we can subscribe to a topic like so:
+```c
+    /* subscribe to "toaster/temperature" with a max QoS level of 0 */
+    mqtt_subscribe(&client, "toaster/temperature", 0);
+```
+And we can publish to a topic like so:
+```c
+    /* publish coffee temperature with a QoS level of 1 */
+    int temperature = 67;
+    mqtt_publish(&client, "coffee/temperature", &temperature, sizeof(int), MQTT_PUBLISH_QOS_1);
+```
+Those are the basics! From here the [examples](https://github.com/LiamBindle/MQTT-C/tree/master/examples) and [API documentation](https://liambindle.ca/MQTT-C/group__api.html) are good places to get started.
+
+## Building
+There are **only two source files** that need to be built, `mqtt.c` and `mqtt_pal.c`.
+These files are ANSI C (C89) compatible, and should compile with any C compiler.
+
+Then, simply <code>\#include <mqtt.h></code>.
+
+Alternatively, you can build MQTT-C with CMake or the provided Makefile. These are provided for convenience.
+
+## Documentation
+Pre-built documentation can be found here: [https://liambindle.ca/MQTT-C](https://liambindle.ca/MQTT-C). Be sure to check out the [examples](https://github.com/LiamBindle/MQTT-C/tree/master/examples) too.
+
+The @ref api documentation contains all the documentation application programmers should need. 
+The @ref pal documentation contains everything you should need to port MQTT-C to a new platform,
+and the other modules contain documentation for MQTT-C developers.
+
+## Testing and Building the Tests
+The MQTT-C unit tests use the [cmocka unit testing framework](https://cmocka.org/). 
+Therefore, [cmocka](https://cmocka.org/) *must* be installed on your machine to build and run 
+the unit tests. For convenience, a simple `"makefile"` is included to build the unit tests and 
+examples on UNIX-like machines. The unit tests and examples can be built as follows:
+```bash
+    $ make all
+``` 
+The unit tests and examples will be built in the `"bin/"` directory. The unit tests can be run 
+like so:
+```bash
+    $ ./bin/tests [address [port]]
+```
+Note that the \c address and \c port arguments are both optional to specify the location of the
+MQTT broker that is to be used for the tests. If no \c address is given then the 
+[Mosquitto MQTT Test Server](https://test.mosquitto.org/) will be used. If no \c port is given, 
+port 1883 will be used.
+
+## Portability
+MQTT-C provides a transparent platform abstraction layer (PAL) in `mqtt_pal.h` and `mqtt_pal.c`.
+These files declare and implement the types and calls that MQTT-C requires. Refer to 
+@ref pal for the complete documentation of the PAL.
+
+## Contributing
+Please feel free to submit issues and pull-requests [here](https://github.com/LiamBindle/MQTT-C).
+When submitting a pull-request please ensure you have *fully documented* your changes and 
+added the appropriate unit tests.
+
+
+## License
+This project is licensed under the [MIT License](https://opensource.org/licenses/MIT). See the 
+`"LICENSE"` file for more details.
+
+## Authors
+MQTT-C was initially developed as a CMPT 434 (Winter Term, 2018) final project at the University of 
+Saskatchewan by:
+- **Liam Bindle**
+- **Demilade Adeoye**
+

deps/MQTT-C/examples/bio_publisher.c

@@ -0,0 +1,156 @@
+
+/**
+ * @file
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/bio_sockets.h"
+
+
+/**
+ * @brief The function that would be called whenever a PUBLISH is received.
+ * 
+ * @note This function is not used in this example. 
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, BIO* sockfd, pthread_t *client_daemon);
+
+/**
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+
+    /* Load OpenSSL */
+    SSL_load_error_strings();
+    ERR_load_BIO_strings();
+    OpenSSL_add_all_algorithms();
+
+    /* get address (argv[1] if present) */
+    if (argc > 1) {
+        addr = argv[1];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[2] if present) */
+    if (argc > 2) {
+        port = argv[2];
+    } else {
+        port = "1883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 3) {
+        topic = argv[3];
+    } else {
+        topic = "datetime";
+    }
+
+    /* open the non-blocking TCP socket (connecting to the broker) */
+    BIO* sockfd = open_nb_socket(addr, port);
+
+    if (sockfd == NULL) {
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* setup a client */
+    struct mqtt_client client;
+    uint8_t sendbuf[2048]; /* sendbuf should be large enough to hold multiple whole mqtt messages */
+    uint8_t recvbuf[1024]; /* recvbuf should be large enough any whole mqtt message expected to be received */
+    mqtt_init(&client, sockfd, sendbuf, sizeof(sendbuf), recvbuf, sizeof(recvbuf), publish_callback);
+    mqtt_connect(&client, "publishing_client", NULL, NULL, 0, NULL, NULL, 0, 400);
+
+    /* check that we don't have any errors */
+    if (client.error != MQTT_OK) {
+        fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+
+    }
+
+    /* start publishing the time */
+    printf("%s is ready to begin publishing the time.\n", argv[0]);
+    printf("Press ENTER to publish the current time.\n");
+    printf("Press CTRL-D (or any other key) to exit.\n\n");
+    while(fgetc(stdin) == '\n') {
+        /* get the current time */
+        time_t timer;
+        time(&timer);
+        struct tm* tm_info = localtime(&timer);
+        char timebuf[26];
+        strftime(timebuf, 26, "%Y-%m-%d %H:%M:%S", tm_info);
+
+        /* print a message */
+        char application_message[256];
+        snprintf(application_message, sizeof(application_message), "The time is %s", timebuf);
+        printf("%s published : \"%s\"", argv[0], application_message);
+
+        /* publish the time */
+        mqtt_publish(&client, topic, application_message, strlen(application_message) + 1, MQTT_PUBLISH_QOS_2);
+
+        /* check for errors */
+        if (client.error != MQTT_OK) {
+            fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+            exit_example(EXIT_FAILURE, sockfd, &client_daemon);
+        }
+    }   
+
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, sockfd, &client_daemon);
+}
+
+void exit_example(int status, BIO* sockfd, pthread_t *client_daemon)
+{
+    if (sockfd != NULL) BIO_free_all(sockfd);
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    exit(status);
+}
+
+
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* not used in this example */
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/mbedtls_publisher.c

@@ -0,0 +1,163 @@
+
+/**
+ * @file
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/mbedtls_sockets.h"
+
+
+/**
+ * @brief The function that would be called whenever a PUBLISH is received.
+ * 
+ * @note This function is not used in this example. 
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, mqtt_pal_socket_handle sockfd, pthread_t *client_daemon);
+
+/**
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+    const char* ca_file;
+
+    struct mbedtls_context ctx;
+    mqtt_pal_socket_handle sockfd;
+
+    if (argc > 1) {
+        ca_file = argv[1];
+    } else {
+        printf("error: path to the CA certificate to use\n");
+        exit(1);
+    }
+
+    /* get address (argv[2] if present) */
+    if (argc > 2) {
+        addr = argv[2];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[3] if present) */
+    if (argc > 3) {
+        port = argv[3];
+    } else {
+        port = "8883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 4) {
+        topic = argv[4];
+    } else {
+        topic = "datetime";
+    }
+
+    /* open the non-blocking TCP socket (connecting to the broker) */
+    open_nb_socket(&ctx, addr, port, ca_file);
+    sockfd = &ctx.ssl_ctx;
+
+    if (sockfd == NULL) {
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* setup a client */
+    struct mqtt_client client;
+    uint8_t sendbuf[2048]; /* sendbuf should be large enough to hold multiple whole mqtt messages */
+    uint8_t recvbuf[1024]; /* recvbuf should be large enough any whole mqtt message expected to be received */
+    mqtt_init(&client, sockfd, sendbuf, sizeof(sendbuf), recvbuf, sizeof(recvbuf), publish_callback);
+    mqtt_connect(&client, "publishing_client", NULL, NULL, 0, NULL, NULL, 0, 400);
+
+    /* check that we don't have any errors */
+    if (client.error != MQTT_OK) {
+        fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+
+    }
+
+    /* start publishing the time */
+    printf("%s is ready to begin publishing the time.\n", argv[0]);
+    printf("Press ENTER to publish the current time.\n");
+    printf("Press CTRL-D (or any other key) to exit.\n\n");
+    while(fgetc(stdin) == '\n') {
+        /* get the current time */
+        time_t timer;
+        time(&timer);
+        struct tm* tm_info = localtime(&timer);
+        char timebuf[26];
+        strftime(timebuf, 26, "%Y-%m-%d %H:%M:%S", tm_info);
+
+        /* print a message */
+        char application_message[256];
+        snprintf(application_message, sizeof(application_message), "The time is %s", timebuf);
+        printf("%s published : \"%s\"", argv[0], application_message);
+
+        /* publish the time */
+        mqtt_publish(&client, topic, application_message, strlen(application_message) + 1, MQTT_PUBLISH_QOS_2);
+
+        /* check for errors */
+        if (client.error != MQTT_OK) {
+            fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+            exit_example(EXIT_FAILURE, sockfd, &client_daemon);
+        }
+    }   
+
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, sockfd, &client_daemon);
+}
+
+void exit_example(int status, mqtt_pal_socket_handle sockfd, pthread_t *client_daemon)
+{
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    mbedtls_ssl_free(sockfd);
+    /* XXX free the rest of contexts */
+    exit(status);
+}
+
+
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* not used in this example */
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/openssl_publisher.c

@@ -0,0 +1,167 @@
+
+/**
+ * @file
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/openssl_sockets.h"
+
+
+/**
+ * @brief The function that would be called whenever a PUBLISH is received.
+ * 
+ * @note This function is not used in this example. 
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, BIO* sockfd, pthread_t *client_daemon);
+
+/**
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+    const char* ca_file;
+
+    /* Load OpenSSL */
+    SSL_load_error_strings();
+    ERR_load_BIO_strings();
+    OpenSSL_add_all_algorithms();
+    SSL_library_init();
+
+    SSL_CTX* ssl_ctx;
+    BIO* sockfd;
+
+    if (argc > 1) {
+        ca_file = argv[1];
+    } else {
+        printf("error: path to the CA certificate to use\n");
+        exit(1);
+    }
+
+    /* get address (argv[2] if present) */
+    if (argc > 2) {
+        addr = argv[2];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[3] if present) */
+    if (argc > 3) {
+        port = argv[3];
+    } else {
+        port = "8883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 4) {
+        topic = argv[4];
+    } else {
+        topic = "datetime";
+    }
+
+    /* open the non-blocking TCP socket (connecting to the broker) */
+    open_nb_socket(&sockfd, &ssl_ctx, addr, port, ca_file, NULL);
+
+    if (sockfd == NULL) {
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* setup a client */
+    struct mqtt_client client;
+    uint8_t sendbuf[2048]; /* sendbuf should be large enough to hold multiple whole mqtt messages */
+    uint8_t recvbuf[1024]; /* recvbuf should be large enough any whole mqtt message expected to be received */
+    mqtt_init(&client, sockfd, sendbuf, sizeof(sendbuf), recvbuf, sizeof(recvbuf), publish_callback);
+    mqtt_connect(&client, "publishing_client", NULL, NULL, 0, NULL, NULL, 0, 400);
+
+    /* check that we don't have any errors */
+    if (client.error != MQTT_OK) {
+        fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+
+    }
+
+    /* start publishing the time */
+    printf("%s is ready to begin publishing the time.\n", argv[0]);
+    printf("Press ENTER to publish the current time.\n");
+    printf("Press CTRL-D (or any other key) to exit.\n\n");
+    while(fgetc(stdin) == '\n') {
+        /* get the current time */
+        time_t timer;
+        time(&timer);
+        struct tm* tm_info = localtime(&timer);
+        char timebuf[26];
+        strftime(timebuf, 26, "%Y-%m-%d %H:%M:%S", tm_info);
+
+        /* print a message */
+        char application_message[256];
+        snprintf(application_message, sizeof(application_message), "The time is %s", timebuf);
+        printf("%s published : \"%s\"", argv[0], application_message);
+
+        /* publish the time */
+        mqtt_publish(&client, topic, application_message, strlen(application_message) + 1, MQTT_PUBLISH_QOS_2);
+
+        /* check for errors */
+        if (client.error != MQTT_OK) {
+            fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+            exit_example(EXIT_FAILURE, sockfd, &client_daemon);
+        }
+    }   
+
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, sockfd, &client_daemon);
+}
+
+void exit_example(int status, BIO* sockfd, pthread_t *client_daemon)
+{
+    if (sockfd != NULL) BIO_free_all(sockfd);
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    exit(status);
+}
+
+
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* not used in this example */
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/reconnect_subscriber.c

@@ -0,0 +1,199 @@
+
+/**
+ * @file
+ * A simple subscriber program that performs automatic reconnections.
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/posix_sockets.h"
+
+/**
+ * @brief A structure that I will use to keep track of some data needed 
+ *        to setup the connection to the broker.
+ * 
+ * An instance of this struct will be created in my \c main(). Then, whenever
+ * \ref reconnect_client is called, this instance will be passed. 
+ */
+struct reconnect_state_t {
+    const char* hostname;
+    const char* port;
+    const char* topic;
+    uint8_t* sendbuf;
+    size_t sendbufsz;
+    uint8_t* recvbuf;
+    size_t recvbufsz;
+};
+
+
+/**
+ * @brief My reconnect callback. It will reestablish the connection whenever 
+ *        an error occurs. 
+ */
+void reconnect_client(struct mqtt_client* client, void **reconnect_state_vptr);
+
+/**
+ * @brief The function will be called whenever a PUBLISH message is received.
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, int sockfd, pthread_t *client_daemon);
+
+
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+
+    /* get address (argv[1] if present) */
+    if (argc > 1) {
+        addr = argv[1];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[2] if present) */
+    if (argc > 2) {
+        port = argv[2];
+    } else {
+        port = "1883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 3) {
+        topic = argv[3];
+    } else {
+        topic = "datetime";
+    }
+
+    /* build the reconnect_state structure which will be passed to reconnect */
+    struct reconnect_state_t reconnect_state;
+    reconnect_state.hostname = addr;
+    reconnect_state.port = port;
+    reconnect_state.topic = topic;
+    uint8_t sendbuf[2048];
+    uint8_t recvbuf[1024];
+    reconnect_state.sendbuf = sendbuf;
+    reconnect_state.sendbufsz = sizeof(sendbuf);
+    reconnect_state.recvbuf = recvbuf;
+    reconnect_state.recvbufsz = sizeof(recvbuf);
+
+    /* setup a client */
+    struct mqtt_client client;
+
+    mqtt_init_reconnect(&client, 
+                        reconnect_client, &reconnect_state, 
+                        publish_callback
+    );
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, -1, NULL);
+
+    }
+
+    /* start publishing the time */
+    printf("%s listening for '%s' messages.\n", argv[0], topic);
+    printf("Press ENTER to inject an error.\n");
+    printf("Press CTRL-D to exit.\n\n");
+    
+    /* block */
+    while(fgetc(stdin) != EOF) {
+        printf("Injecting error: \"MQTT_ERROR_SOCKET_ERROR\"\n");
+        client.error = MQTT_ERROR_SOCKET_ERROR;
+    } 
+    
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, client.socketfd, &client_daemon);
+}
+
+void reconnect_client(struct mqtt_client* client, void **reconnect_state_vptr)
+{
+    struct reconnect_state_t *reconnect_state = *((struct reconnect_state_t**) reconnect_state_vptr);
+
+    /* Close the clients socket if this isn't the initial reconnect call */
+    if (client->error != MQTT_ERROR_INITIAL_RECONNECT) {
+        close(client->socketfd);
+    }
+
+    /* Perform error handling here. */
+    if (client->error != MQTT_ERROR_INITIAL_RECONNECT) {
+        printf("reconnect_client: called while client was in error state \"%s\"\n", 
+               mqtt_error_str(client->error)
+        );
+    }
+
+    /* Open a new socket. */
+    int sockfd = open_nb_socket(reconnect_state->hostname, reconnect_state->port);
+    if (sockfd == -1) {
+        perror("Failed to open socket: ");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* Reinitialize the client. */
+    mqtt_reinit(client, sockfd, 
+                reconnect_state->sendbuf, reconnect_state->sendbufsz,
+                reconnect_state->recvbuf, reconnect_state->recvbufsz
+    );
+
+    /* Create an anonymous session */
+    const char* client_id = NULL;
+    /* Ensure we have a clean session */
+    uint8_t connect_flags = MQTT_CONNECT_CLEAN_SESSION;
+    /* Send connection request to the broker. */
+    mqtt_connect(client, client_id, NULL, NULL, 0, NULL, NULL, connect_flags, 400);
+
+    /* Subscribe to the topic. */
+    mqtt_subscribe(client, reconnect_state->topic, 0);
+}
+
+void exit_example(int status, int sockfd, pthread_t *client_daemon)
+{
+    if (sockfd != -1) close(sockfd);
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    exit(status);
+}
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* note that published->topic_name is NOT null-terminated (here we'll change it to a c-string) */
+    char* topic_name = (char*) malloc(published->topic_name_size + 1);
+    memcpy(topic_name, published->topic_name, published->topic_name_size);
+    topic_name[published->topic_name_size] = '\0';
+
+    printf("Received publish('%s'): %s\n", topic_name, (const char*) published->application_message);
+
+    free(topic_name);
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/simple_publisher.c

@@ -0,0 +1,157 @@
+
+/**
+ * @file
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/posix_sockets.h"
+
+
+/**
+ * @brief The function that would be called whenever a PUBLISH is received.
+ * 
+ * @note This function is not used in this example. 
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, int sockfd, pthread_t *client_daemon);
+
+/**
+ * A simple program to that publishes the current time whenever ENTER is pressed. 
+ */
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+
+    /* get address (argv[1] if present) */
+    if (argc > 1) {
+        addr = argv[1];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[2] if present) */
+    if (argc > 2) {
+        port = argv[2];
+    } else {
+        port = "1883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 3) {
+        topic = argv[3];
+    } else {
+        topic = "datetime";
+    }
+
+    /* open the non-blocking TCP socket (connecting to the broker) */
+    int sockfd = open_nb_socket(addr, port);
+
+    if (sockfd == -1) {
+        perror("Failed to open socket: ");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* setup a client */
+    struct mqtt_client client;
+    uint8_t sendbuf[2048]; /* sendbuf should be large enough to hold multiple whole mqtt messages */
+    uint8_t recvbuf[1024]; /* recvbuf should be large enough any whole mqtt message expected to be received */
+    mqtt_init(&client, sockfd, sendbuf, sizeof(sendbuf), recvbuf, sizeof(recvbuf), publish_callback);
+    /* Create an anonymous session */
+    const char* client_id = NULL;
+    /* Ensure we have a clean session */
+    uint8_t connect_flags = MQTT_CONNECT_CLEAN_SESSION;
+    /* Send connection request to the broker. */
+    mqtt_connect(&client, client_id, NULL, NULL, 0, NULL, NULL, connect_flags, 400);
+
+    /* check that we don't have any errors */
+    if (client.error != MQTT_OK) {
+        fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+
+    }
+
+    /* start publishing the time */
+    printf("%s is ready to begin publishing the time.\n", argv[0]);
+    printf("Press ENTER to publish the current time.\n");
+    printf("Press CTRL-D (or any other key) to exit.\n\n");
+    while(fgetc(stdin) == '\n') {
+        /* get the current time */
+        time_t timer;
+        time(&timer);
+        struct tm* tm_info = localtime(&timer);
+        char timebuf[26];
+        strftime(timebuf, 26, "%Y-%m-%d %H:%M:%S", tm_info);
+
+        /* print a message */
+        char application_message[256];
+        snprintf(application_message, sizeof(application_message), "The time is %s", timebuf);
+        printf("%s published : \"%s\"", argv[0], application_message);
+
+        /* publish the time */
+        mqtt_publish(&client, topic, application_message, strlen(application_message) + 1, MQTT_PUBLISH_QOS_0);
+
+        /* check for errors */
+        if (client.error != MQTT_OK) {
+            fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+            exit_example(EXIT_FAILURE, sockfd, &client_daemon);
+        }
+    }   
+
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, sockfd, &client_daemon);
+}
+
+void exit_example(int status, int sockfd, pthread_t *client_daemon)
+{
+    if (sockfd != -1) close(sockfd);
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    exit(status);
+}
+
+
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* not used in this example */
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/simple_subscriber.c

@@ -0,0 +1,142 @@
+
+/**
+ * @file
+ * A simple program that subscribes to a topic.
+ */
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <mqtt.h>
+#include "templates/posix_sockets.h"
+
+
+/**
+ * @brief The function will be called whenever a PUBLISH message is received.
+ */
+void publish_callback(void** unused, struct mqtt_response_publish *published);
+
+/**
+ * @brief The client's refresher. This function triggers back-end routines to 
+ *        handle ingress/egress traffic to the broker.
+ * 
+ * @note All this function needs to do is call \ref __mqtt_recv and 
+ *       \ref __mqtt_send every so often. I've picked 100 ms meaning that 
+ *       client ingress/egress traffic will be handled every 100 ms.
+ */
+void* client_refresher(void* client);
+
+/**
+ * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. 
+ */
+void exit_example(int status, int sockfd, pthread_t *client_daemon);
+
+int main(int argc, const char *argv[]) 
+{
+    const char* addr;
+    const char* port;
+    const char* topic;
+
+    /* get address (argv[1] if present) */
+    if (argc > 1) {
+        addr = argv[1];
+    } else {
+        addr = "test.mosquitto.org";
+    }
+
+    /* get port number (argv[2] if present) */
+    if (argc > 2) {
+        port = argv[2];
+    } else {
+        port = "1883";
+    }
+
+    /* get the topic name to publish */
+    if (argc > 3) {
+        topic = argv[3];
+    } else {
+        topic = "datetime";
+    }
+
+    /* open the non-blocking TCP socket (connecting to the broker) */
+    int sockfd = open_nb_socket(addr, port);
+
+    if (sockfd == -1) {
+        perror("Failed to open socket: ");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* setup a client */
+    struct mqtt_client client;
+    uint8_t sendbuf[2048]; /* sendbuf should be large enough to hold multiple whole mqtt messages */
+    uint8_t recvbuf[1024]; /* recvbuf should be large enough any whole mqtt message expected to be received */
+    mqtt_init(&client, sockfd, sendbuf, sizeof(sendbuf), recvbuf, sizeof(recvbuf), publish_callback);
+    /* Create an anonymous session */
+    const char* client_id = NULL;
+    /* Ensure we have a clean session */
+    uint8_t connect_flags = MQTT_CONNECT_CLEAN_SESSION;
+    /* Send connection request to the broker. */
+    mqtt_connect(&client, client_id, NULL, NULL, 0, NULL, NULL, connect_flags, 400);
+
+    /* check that we don't have any errors */
+    if (client.error != MQTT_OK) {
+        fprintf(stderr, "error: %s\n", mqtt_error_str(client.error));
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+    }
+
+    /* start a thread to refresh the client (handle egress and ingree client traffic) */
+    pthread_t client_daemon;
+    if(pthread_create(&client_daemon, NULL, client_refresher, &client)) {
+        fprintf(stderr, "Failed to start client daemon.\n");
+        exit_example(EXIT_FAILURE, sockfd, NULL);
+
+    }
+
+    /* subscribe */
+    mqtt_subscribe(&client, topic, 0);
+
+    /* start publishing the time */
+    printf("%s listening for '%s' messages.\n", argv[0], topic);
+    printf("Press CTRL-D to exit.\n\n");
+    
+    /* block */
+    while(fgetc(stdin) != EOF); 
+    
+    /* disconnect */
+    printf("\n%s disconnecting from %s\n", argv[0], addr);
+    sleep(1);
+
+    /* exit */ 
+    exit_example(EXIT_SUCCESS, sockfd, &client_daemon);
+}
+
+void exit_example(int status, int sockfd, pthread_t *client_daemon)
+{
+    if (sockfd != -1) close(sockfd);
+    if (client_daemon != NULL) pthread_cancel(*client_daemon);
+    exit(status);
+}
+
+
+
+void publish_callback(void** unused, struct mqtt_response_publish *published) 
+{
+    /* note that published->topic_name is NOT null-terminated (here we'll change it to a c-string) */
+    char* topic_name = (char*) malloc(published->topic_name_size + 1);
+    memcpy(topic_name, published->topic_name, published->topic_name_size);
+    topic_name[published->topic_name_size] = '\0';
+
+    printf("Received publish('%s'): %s\n", topic_name, (const char*) published->application_message);
+
+    free(topic_name);
+}
+
+void* client_refresher(void* client)
+{
+    while(1) 
+    {
+        mqtt_sync((struct mqtt_client*) client);
+        usleep(100000U);
+    }
+    return NULL;
+}

deps/MQTT-C/examples/templates/bio_sockets.h

@@ -0,0 +1,28 @@
+#ifndef __BIO_SOCKET_TEMPLATE_H__
+#define __BIO_SOCKET_TEMPLATE_H__
+
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+/*
+    A template for opening a non-blocking BIO socket.
+*/
+BIO* open_nb_socket(const char* addr, const char* port) {
+    BIO* bio = BIO_new_connect(addr);
+    BIO_set_nbio(bio, 1);
+    BIO_set_conn_port(bio, port);
+
+    /* timeout after 10 seconds */
+    int start_time = time(NULL);
+    while(BIO_do_connect(bio) == 0 && (int)time(NULL) - start_time < 10);
+
+    if (BIO_do_connect(bio) <= 0) {
+        fprintf(stderr, "Failed to open socket: BIO_do_connect returned <= 0\n");
+        return NULL;
+    }
+
+    return bio; 
+}
+
+#endif

deps/MQTT-C/examples/templates/mbedtls_sockets.h

@@ -0,0 +1,145 @@
+#ifndef __MBEDTLS_SOCKET_TEMPLATE_H__
+#define __MBEDTLS_SOCKET_TEMPLATE_H__
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <mbedtls/error.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/net_sockets.h>
+#include <mbedtls/ssl.h>
+
+#if !defined(MBEDTLS_NET_POLL_READ)
+/* compat for older mbedtls */
+#define	MBEDTLS_NET_POLL_READ	1
+#define	MBEDTLS_NET_POLL_WRITE	1
+
+int
+mbedtls_net_poll(mbedtls_net_context * ctx, uint32_t rw, uint32_t timeout)
+{
+	/* XXX this is not ideal but good enough for an example */
+	usleep(300);
+	return 1;
+}
+#endif
+
+struct mbedtls_context {
+    mbedtls_net_context net_ctx;
+    mbedtls_ssl_context ssl_ctx;
+    mbedtls_ssl_config ssl_conf;
+    mbedtls_x509_crt ca_crt;
+    mbedtls_entropy_context entropy;
+    mbedtls_ctr_drbg_context ctr_drbg;
+};
+
+void failed(const char *fn, int rv) {
+    char buf[100];
+    mbedtls_strerror(rv, buf, sizeof(buf));
+    printf("%s failed with %x (%s)\n", fn, -rv, buf);
+    exit(1);
+}
+
+void cert_verify_failed(uint32_t rv) {
+    char buf[512];
+    mbedtls_x509_crt_verify_info(buf, sizeof(buf), "\t", rv);
+    printf("Certificate verification failed (%0" PRIx32 ")\n%s\n", rv, buf);
+    exit(1);
+}
+
+/*
+    A template for opening a non-blocking mbed TLS connection.
+*/
+void open_nb_socket(struct mbedtls_context *ctx,
+                    const char *hostname,
+                    const char *port,
+                    const char *ca_file) {
+    const unsigned char *additional = (const unsigned char *)"MQTT-C";
+    size_t additional_len = 6;
+    int rv;
+
+    mbedtls_net_context *net_ctx = &ctx->net_ctx;
+    mbedtls_ssl_context *ssl_ctx = &ctx->ssl_ctx;
+    mbedtls_ssl_config *ssl_conf = &ctx->ssl_conf;
+    mbedtls_x509_crt *ca_crt = &ctx->ca_crt;
+    mbedtls_entropy_context *entropy = &ctx->entropy;
+    mbedtls_ctr_drbg_context *ctr_drbg = &ctx->ctr_drbg;
+
+    mbedtls_entropy_init(entropy);
+    mbedtls_ctr_drbg_init(ctr_drbg);
+    rv = mbedtls_ctr_drbg_seed(ctr_drbg, mbedtls_entropy_func, entropy,
+                               additional, additional_len);
+    if (rv != 0) {
+        failed("mbedtls_ctr_drbg_seed", rv);
+    }
+
+    mbedtls_x509_crt_init(ca_crt);
+    rv = mbedtls_x509_crt_parse_file(ca_crt, ca_file);
+    if (rv != 0) {
+        failed("mbedtls_x509_crt_parse_file", rv);
+    }
+
+    mbedtls_ssl_config_init(ssl_conf);
+    rv = mbedtls_ssl_config_defaults(ssl_conf,  MBEDTLS_SSL_IS_CLIENT,
+                                     MBEDTLS_SSL_TRANSPORT_STREAM,
+                                     MBEDTLS_SSL_PRESET_DEFAULT);
+    if (rv != 0) {
+        failed("mbedtls_ssl_config_defaults", rv);
+    }
+    mbedtls_ssl_conf_ca_chain(ssl_conf, ca_crt, NULL);
+    mbedtls_ssl_conf_authmode(ssl_conf, MBEDTLS_SSL_VERIFY_OPTIONAL);
+    mbedtls_ssl_conf_rng(ssl_conf, mbedtls_ctr_drbg_random, ctr_drbg);
+
+    mbedtls_net_init(net_ctx);
+    rv = mbedtls_net_connect(net_ctx, hostname, port, MBEDTLS_NET_PROTO_TCP);
+    if (rv != 0) {
+        failed("mbedtls_net_connect", rv);
+    }
+    rv = mbedtls_net_set_nonblock(net_ctx);
+    if (rv != 0) {
+        failed("mbedtls_net_set_nonblock", rv);
+    }
+
+    mbedtls_ssl_init(ssl_ctx);
+    rv = mbedtls_ssl_setup(ssl_ctx, ssl_conf);
+    if (rv != 0) {
+        failed("mbedtls_ssl_setup", rv);
+    }
+    rv = mbedtls_ssl_set_hostname(ssl_ctx, hostname);
+    if (rv != 0) {
+        failed("mbedtls_ssl_set_hostname", rv);
+    }
+    mbedtls_ssl_set_bio(ssl_ctx, net_ctx,
+                        mbedtls_net_send, mbedtls_net_recv, NULL);
+
+    for (;;) {
+        rv = mbedtls_ssl_handshake(ssl_ctx);
+        uint32_t want = 0;
+        if (rv == MBEDTLS_ERR_SSL_WANT_READ) {
+            want |= MBEDTLS_NET_POLL_READ;
+        } else if (rv == MBEDTLS_ERR_SSL_WANT_WRITE) {
+            want |= MBEDTLS_NET_POLL_WRITE;
+        } else {
+            break;
+        }
+        rv = mbedtls_net_poll(net_ctx, want, -1);
+        if (rv < 0) {
+            failed("mbedtls_net_poll", rv);
+        }
+    }
+    if (rv != 0) {
+        failed("mbedtls_ssl_handshake", rv);
+    }
+    uint32_t result = mbedtls_ssl_get_verify_result(ssl_ctx);
+    if (result != 0) {
+        if (result == (uint32_t)-1) {
+            failed("mbedtls_ssl_get_verify_result", result);
+        } else {
+            cert_verify_failed(result);
+        }
+    }
+}
+
+#endif

deps/MQTT-C/examples/templates/openssl_sockets.h

@@ -0,0 +1,52 @@
+#ifndef __OPENSSL_SOCKET_TEMPLATE_H__
+#define __OPENSSL_SOCKET_TEMPLATE_H__
+
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+/*
+    A template for opening a non-blocking OpenSSL connection.
+*/
+void open_nb_socket(BIO** bio, SSL_CTX** ssl_ctx, const char* addr, const char* port, const char* ca_file, const char* ca_path) {
+    *ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+    SSL* ssl;
+
+    /* load certificate */
+    if (!SSL_CTX_load_verify_locations(*ssl_ctx, ca_file, ca_path)) {
+        printf("error: failed to load certificate\n");
+        exit(1);
+    }
+
+    /* open BIO socket */
+    *bio = BIO_new_ssl_connect(*ssl_ctx);
+    BIO_get_ssl(*bio, &ssl);
+    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
+    BIO_set_conn_hostname(*bio, addr);
+    BIO_set_nbio(*bio, 1);
+    BIO_set_conn_port(*bio, port);
+
+    /* wait for connect with 10 second timeout */
+    int start_time = time(NULL);
+    int do_connect_rv = BIO_do_connect(*bio);
+    while(do_connect_rv <= 0 && BIO_should_retry(*bio) && (int)time(NULL) - start_time < 10) {
+        do_connect_rv = BIO_do_connect(*bio);
+    }
+    if (do_connect_rv <= 0) {
+        printf("error: %s\n", ERR_reason_error_string(ERR_get_error()));
+        BIO_free_all(*bio);
+        SSL_CTX_free(*ssl_ctx);
+        *bio = NULL;
+        *ssl_ctx=NULL;
+        return;
+    }
+
+    /* verify certificate */
+    if (SSL_get_verify_result(ssl) != X509_V_OK) {
+        /* Handle the failed verification */
+        printf("error: x509 certificate verification failed\n");
+        exit(1);
+    }
+}
+
+#endif

deps/MQTT-C/examples/templates/posix_sockets.h

@@ -0,0 +1,59 @@
+#ifndef __POSIX_SOCKET_TEMPLATE_H__
+#define __POSIX_SOCKET_TEMPLATE_H__
+
+#include <stdio.h>
+#include <sys/types.h>
+#if !defined(WIN32)
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+#include <fcntl.h>
+
+/*
+    A template for opening a non-blocking POSIX socket.
+*/
+int open_nb_socket(const char* addr, const char* port) {
+    struct addrinfo hints = {0};
+
+    hints.ai_family = AF_UNSPEC; /* IPv4 or IPv6 */
+    hints.ai_socktype = SOCK_STREAM; /* Must be TCP */
+    int sockfd = -1;
+    int rv;
+    struct addrinfo *p, *servinfo;
+
+    /* get address information */
+    rv = getaddrinfo(addr, port, &hints, &servinfo);
+    if(rv != 0) {
+        fprintf(stderr, "Failed to open socket (getaddrinfo): %s\n", gai_strerror(rv));
+        return -1;
+    }
+
+    /* open the first possible socket */
+    for(p = servinfo; p != NULL; p = p->ai_next) {
+        sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol);
+        if (sockfd == -1) continue;
+
+        /* connect to server */
+        rv = connect(sockfd, servinfo->ai_addr, servinfo->ai_addrlen);
+        if(rv == -1) continue;
+        break;
+    }  
+
+    /* free servinfo */
+    freeaddrinfo(servinfo);
+
+    /* make non-blocking */
+#if !defined(WIN32)
+    if (sockfd != -1) fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFL) | O_NONBLOCK);
+#else
+    if (sockfd != INVALID_SOCKET) {
+        int iMode = 1;
+        ioctlsocket(sockfd, FIONBIO, &iMode);
+    }
+#endif
+
+    /* return the new socket fd */
+    return sockfd;
+}
+
+#endif

deps/MQTT-C/include/mqtt_pal.h

@@ -0,0 +1,157 @@
+#ifndef __MQTT_PAL_H__
+#define __MQTT_PAL_H__
+
+/*
+MIT License
+
+Copyright(c) 2018 Liam Bindle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files(the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions :
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+/**
+ * @file
+ * @brief Includes/supports the types/calls required by the MQTT-C client.
+ * 
+ * @note This is the \em only file included in mqtt.h, and mqtt.c. It is therefore 
+ *       responsible for including/supporting all the required types and calls. 
+ * 
+ * @defgroup pal Platform abstraction layer
+ * @brief Documentation of the types and calls required to port MQTT-C to a new platform.
+ * 
+ * mqtt_pal.h is the \em only header file included in mqtt.c. Therefore, to port MQTT-C to a 
+ * new platform the following types, functions, constants, and macros must be defined in 
+ * mqtt_pal.h:
+ *  - Types:
+ *      - \c size_t, \c ssize_t
+ *      - \c uint8_t, \c uint16_t, \c uint32_t
+ *      - \c va_list
+ *      - \c mqtt_pal_time_t : return type of \c MQTT_PAL_TIME() 
+ *      - \c mqtt_pal_mutex_t : type of the argument that is passed to \c MQTT_PAL_MUTEX_LOCK and 
+ *        \c MQTT_PAL_MUTEX_RELEASE
+ *  - Functions:
+ *      - \c memcpy, \c strlen
+ *      - \c va_start, \c va_arg, \c va_end
+ *  - Constants:
+ *      - \c INT_MIN
+ * 
+ * Additionally, three macro's are required:
+ *  - \c MQTT_PAL_HTONS(s) : host-to-network endian conversion for uint16_t.
+ *  - \c MQTT_PAL_NTOHS(s) : network-to-host endian conversion for uint16_t.
+ *  - \c MQTT_PAL_TIME()   : returns [type: \c mqtt_pal_time_t] current time in seconds. 
+ *  - \c MQTT_PAL_MUTEX_LOCK(mtx_pointer) : macro that locks the mutex pointed to by \c mtx_pointer.
+ *  - \c MQTT_PAL_MUTEX_RELEASE(mtx_pointer) : macro that unlocks the mutex pointed to by 
+ *    \c mtx_pointer.
+ * 
+ * Lastly, \ref mqtt_pal_sendall and \ref mqtt_pal_recvall, must be implemented in mqtt_pal.c 
+ * for sending and receiving data using the platforms socket calls.
+ */
+
+
+/* UNIX-like platform support */
+#if defined(__unix__) || defined(__APPLE__)
+    #include <limits.h>
+    #include <string.h>
+    #include <stdarg.h>
+    #include <time.h>
+    #include <arpa/inet.h>
+    #include <pthread.h>
+
+    #define MQTT_PAL_HTONS(s) htons(s)
+    #define MQTT_PAL_NTOHS(s) ntohs(s)
+
+    #define MQTT_PAL_TIME() time(NULL)
+
+    typedef time_t mqtt_pal_time_t;
+    typedef pthread_mutex_t mqtt_pal_mutex_t;
+
+    #define MQTT_PAL_MUTEX_INIT(mtx_ptr) pthread_mutex_init(mtx_ptr, NULL)
+    #define MQTT_PAL_MUTEX_LOCK(mtx_ptr) pthread_mutex_lock(mtx_ptr)
+    #define MQTT_PAL_MUTEX_UNLOCK(mtx_ptr) pthread_mutex_unlock(mtx_ptr)
+
+    #ifndef MQTT_USE_CUSTOM_SOCKET_HANDLE
+        #ifdef MQTT_USE_MBEDTLS
+            struct mbedtls_ssl_context;
+            typedef struct mbedtls_ssl_context *mqtt_pal_socket_handle;
+        #elif defined(MQTT_USE_BIO)
+            #include <openssl/bio.h>
+            typedef BIO* mqtt_pal_socket_handle;
+        #else
+            typedef int mqtt_pal_socket_handle;
+        #endif
+    #endif
+#elif defined(_MSC_VER)
+    #include <limits.h>
+    #include <windows.h>
+    #include <time.h>
+    #include <stdint.h>
+    #include <winsock2.h>
+
+    typedef SSIZE_T ssize_t;
+    #define MQTT_PAL_HTONS(s) htons(s)
+    #define MQTT_PAL_NTOHS(s) ntohs(s)
+
+    #define MQTT_PAL_TIME() time(NULL)
+
+    typedef time_t mqtt_pal_time_t;
+    typedef CRITICAL_SECTION mqtt_pal_mutex_t;
+
+    #define MQTT_PAL_MUTEX_INIT(mtx_ptr) InitializeCriticalSection(mtx_ptr)
+    #define MQTT_PAL_MUTEX_LOCK(mtx_ptr) EnterCriticalSection(mtx_ptr)
+    #define MQTT_PAL_MUTEX_UNLOCK(mtx_ptr) LeaveCriticalSection(mtx_ptr)
+
+
+    #ifndef MQTT_USE_CUSTOM_SOCKET_HANDLE
+        #ifdef MQTT_USE_BIO
+            #include <openssl/bio.h>
+            typedef BIO* mqtt_pal_socket_handle;
+        #else
+            typedef SOCKET mqtt_pal_socket_handle;
+        #endif
+    #endif
+
+#endif
+
+/**
+ * @brief Sends all the bytes in a buffer.
+ * @ingroup pal
+ * 
+ * @param[in] fd The file-descriptor (or handle) of the socket.
+ * @param[in] buf A pointer to the first byte in the buffer to send.
+ * @param[in] len The number of bytes to send (starting at \p buf).
+ * @param[in] flags Flags which are passed to the underlying socket.
+ * 
+ * @returns The number of bytes sent if successful, an \ref MQTTErrors otherwise.
+ */
+ssize_t mqtt_pal_sendall(mqtt_pal_socket_handle fd, const void* buf, size_t len, int flags);
+
+/**
+ * @brief Non-blocking receive all the byte available.
+ * @ingroup pal
+ * 
+ * @param[in] fd The file-descriptor (or handle) of the socket.
+ * @param[in] buf A pointer to the receive buffer.
+ * @param[in] bufsz The max number of bytes that can be put into \p buf.
+ * @param[in] flags Flags which are passed to the underlying socket.
+ * 
+ * @returns The number of bytes received if successful, an \ref MQTTErrors otherwise.
+ */
+ssize_t mqtt_pal_recvall(mqtt_pal_socket_handle fd, void* buf, size_t bufsz, int flags);
+
+#endif

deps/MQTT-C/src/mqtt_pal.c

@@ -0,0 +1,203 @@
+/*
+MIT License
+
+Copyright(c) 2018 Liam Bindle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files(the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions :
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#include <mqtt.h>
+
+/** 
+ * @file 
+ * @brief Implements @ref mqtt_pal_sendall and @ref mqtt_pal_recvall and 
+ *        any platform-specific helpers you'd like.
+ * @cond Doxygen_Suppress
+ */
+
+
+#ifdef MQTT_USE_MBEDTLS
+#include <mbedtls/ssl.h>
+
+ssize_t mqtt_pal_sendall(mqtt_pal_socket_handle fd, const void* buf, size_t len, int flags) {
+    size_t sent = 0;
+    while(sent < len) {
+        int rv = mbedtls_ssl_write(fd, buf + sent, len - sent);
+        if (rv < 0) {
+            if (rv == MBEDTLS_ERR_SSL_WANT_READ ||
+                rv == MBEDTLS_ERR_SSL_WANT_WRITE
+#if defined(MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS)
+                || rv == MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS
+#endif
+#if defined(MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS)
+                || rv == MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS
+#endif
+                ) {
+                /* should call mbedtls_ssl_writer later again */
+                break;
+            }
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+        sent += (size_t) rv;
+    }
+    return sent;
+}
+
+ssize_t mqtt_pal_recvall(mqtt_pal_socket_handle fd, void* buf, size_t bufsz, int flags) {
+    const void *const start = buf;
+    int rv;
+    do {
+        rv = mbedtls_ssl_read(fd, buf, bufsz);
+        if (rv < 0) {
+            if (rv == MBEDTLS_ERR_SSL_WANT_READ ||
+                rv == MBEDTLS_ERR_SSL_WANT_WRITE
+#if defined(MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS)
+                || rv == MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS
+#endif
+#if defined(MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS)
+                || rv == MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS
+#endif
+                ) {
+                /* should call mbedtls_ssl_read later again */
+                break;
+            }
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+        buf = (char*)buf + rv;
+        bufsz -= rv;
+    } while (rv > 0);
+
+    return buf - start;
+}
+
+#elif defined(MQTT_USE_BIO)
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+ssize_t mqtt_pal_sendall(mqtt_pal_socket_handle fd, const void* buf, size_t len, int flags) {
+    size_t sent = 0;
+    while(sent < len) {
+        int tmp = BIO_write(fd, buf + sent, len - sent);
+        if (tmp > 0) {
+            sent += (size_t) tmp;
+        } else if (tmp <= 0 && !BIO_should_retry(fd)) {
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+    }
+    
+    return sent;
+}
+
+ssize_t mqtt_pal_recvall(mqtt_pal_socket_handle fd, void* buf, size_t bufsz, int flags) {
+    const void *const start = buf;
+    int rv;
+    do {
+        rv = BIO_read(fd, buf, bufsz);
+        if (rv > 0) {
+            /* successfully read bytes from the socket */
+            buf += rv;
+            bufsz -= rv;
+        } else if (!BIO_should_retry(fd)) {
+            /* an error occurred that wasn't "nothing to read". */
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+    } while (!BIO_should_read(fd));
+
+    return (ssize_t)(buf - start);
+}
+
+#elif defined(__unix__) || defined(__APPLE__)
+
+#include <errno.h>
+
+ssize_t mqtt_pal_sendall(mqtt_pal_socket_handle fd, const void* buf, size_t len, int flags) {
+    size_t sent = 0;
+    while(sent < len) {
+        ssize_t tmp = send(fd, buf + sent, len - sent, flags);
+        if (tmp < 1) {
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+        sent += (size_t) tmp;
+    }
+    return sent;
+}
+
+ssize_t mqtt_pal_recvall(mqtt_pal_socket_handle fd, void* buf, size_t bufsz, int flags) {
+    const void *const start = buf;
+    ssize_t rv;
+    do {
+        rv = recv(fd, buf, bufsz, flags);
+        if (rv > 0) {
+            /* successfully read bytes from the socket */
+            buf += rv;
+            bufsz -= rv;
+        } else if (rv < 0 && errno != EAGAIN && errno != EWOULDBLOCK) {
+            /* an error occurred that wasn't "nothing to read". */
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+    } while (rv > 0);
+
+    return buf - start;
+}
+
+#elif defined(_MSC_VER)
+
+#include <errno.h>
+
+ssize_t mqtt_pal_sendall(mqtt_pal_socket_handle fd, const void* buf, size_t len, int flags) {
+    size_t sent = 0;
+    while(sent < len) {
+        ssize_t tmp = send(fd, (char*)buf + sent, len - sent, flags);
+        if (tmp < 1) {
+            return MQTT_ERROR_SOCKET_ERROR;
+        }
+        sent += (size_t) tmp;
+    }
+    return sent;
+}
+
+ssize_t mqtt_pal_recvall(mqtt_pal_socket_handle fd, void* buf, size_t bufsz, int flags) {
+    const char *const start = buf;
+    ssize_t rv;
+    do {
+        rv = recv(fd, buf, bufsz, flags);
+        if (rv > 0) {
+            /* successfully read bytes from the socket */
+            buf = (char*)buf + rv;
+            bufsz -= rv;
+        } else if (rv < 0) {
+            int err = WSAGetLastError();
+            if (err != WSAEWOULDBLOCK) {
+                /* an error occurred that wasn't "nothing to read". */
+                return MQTT_ERROR_SOCKET_ERROR;
+            }
+        }
+    } while (rv > 0);
+
+    return (ssize_t)((char*)buf - start);
+}
+
+#else
+
+#error No PAL!
+
+#endif
+
+/** @endcond */

documentation/webdocs/markdowndocs/Connector.md

@@ -2,6 +2,8 @@
 
 TDengine provides many connectors for development, including C/C++, JAVA, Python, RESTful, Go, Node.JS, etc.
 
+NOTE: All APIs which require a SQL string as parameter, including but not limit to `taos_query`, `taos_query_a`, `taos_subscribe` in the C/C++ Connector and their counterparts in other connectors, can ONLY process one SQL statement at a time. If more than one SQL statements are provided, their behaviors are undefined.
+
 ## C/C++ API
 
 C/C++ APIs are similar to the MySQL APIs. Applications should include TDengine head file _taos.h_ to use C/C++ APIs by adding the following line in code:

documentation/webdocs/markdowndocs/Super Table-ch.md

@@ -53,10 +53,11 @@ STable从属于库，一个STable只属于一个库，但一个库可以有一
 
     说明：
 
-    1. TAGS列总长度不能超过512 bytes；
+    1. TAGS列总长度不能超过64k bytes；
     2. TAGS列的数据类型不能是timestamp；
     3. TAGS列名不能与其他列名相同;
     4. TAGS列名不能为预留关键字. 
+    5. TAGS总数的上限是128.
 
 - 显示已创建的超级表
 
@@ -114,7 +115,7 @@ INSERT INTO <tb1_name> USING <stb1_name> TAGS (<tag1_value1>, ...) VALUES (<fiel
     ALTER TABLE <stable_name> ADD TAG <new_tag_name> <TYPE>
     ```
 
-    为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过6个。
+    为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个。
 
 - 删除标签
 
@@ -202,7 +203,7 @@ INSERT INTO therm4 VALUES ('2018-01-01 00:00:00.000', 23);
 
 ###3:按标签聚合查询
 
-查询位于北京(beijing)和天津(tianjing)两个地区的温度传感器采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)、最低温度min(degree)，并将结果按所处地域(location)和传感器类型(type)进行聚合。
+查询位于北京(beijing)和天津(tianjin)两个地区的温度传感器采样值的数量count(*)、平均温度avg(degree)、最高温度max(degree)、最低温度min(degree)，并将结果按所处地域(location)和传感器类型(type)进行聚合。
 
 ```mysql
 SELECT COUNT(*), AVG(degree), MAX(degree), MIN(degree)

documentation/webdocs/markdowndocs/Super Table.md

@@ -22,11 +22,11 @@ New keyword "tags" is introduced, where tag_name is the tag name, and tag_type i
 
 Note：
 
-1. The bytes of all tags together shall be less than 512 
+1. The bytes of all tags together shall be less than 64k 
 2. Tag's data type can not be time stamp
 3. Tag name shall be different from the field name
 4. Tag name shall not be the same as system keywords
-5. Maximum number of tags is 6 
+5. Maximum number of tags is 128 
 
 For example:
 
@@ -168,7 +168,7 @@ You can add, delete and change the tags for a STable, and you can change the tag
 ALTER TABLE <stable_name> ADD TAG <new_tag_name> <TYPE>
 ```
 
-It adds a new tag to the STable with a data type. The maximum number of tags is 6. 
+It adds a new tag to the STable with a data type. The maximum number of tags is 128. 
 
 ### Drop a Tag
 

documentation/webdocs/markdowndocs/TAOS SQL-ch.md

@@ -63,11 +63,11 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
 | 3    |  BIGINT   | 8      | 长整型，范围 [-2^63+1,   2^63-1], -2^63用于NULL                                 |
 | 4    |   FLOAT   | 4      | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38]                  |
 | 5    |  DOUBLE   | 8      | 双精度浮点型，有效位数15-16，范围 [-1.7E308,   1.7E308]      |
-| 6    |  BINARY   | 自定义 | 用于记录字符串，最长不能超过504 bytes。binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节，将被自动截断。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
+| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有65526字节，但由于每行数据最多64K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
 | 7    | SMALLINT  | 2      | 短整型， 范围 [-32767, 32767], -32768用于NULL                                |
 | 8    |  TINYINT  | 1      | 单字节整型，范围 [-127, 127], -128用于NULL                                |
 | 9    |   BOOL    | 1      | 布尔型，{true,   false}                                      |
-| 10   |   NCHAR   | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将被自动截断。 |
+| 10   |   NCHAR   | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将会报错。 |
 
 **Tips**: TDengine对SQL语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。
 
@@ -106,7 +106,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
     ```mysql
     CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...])
     ```
-    说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过4096字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
+    说明：1）表的第一个字段必须是TIMESTAMP，并且系统自动将其设为主键；2）表的每行长度不能超过64K字节；3）使用数据类型binary或nchar，需指定其最长的字节数，如binary(20)，表示20字节。
 
 
 - **删除数据表**
@@ -402,7 +402,7 @@ count(tbname) |
     SELECT * FROM tb1 WHERE ts >= NOW - 1h
     ```
 
-- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且clo3的字符串是'nny'结尾的记录，结果按照时间戳降序
+- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序
 
     ```mysql
     SELECT * FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND ts <= '2018-06-02 08:00:00.000' AND col3 LIKE '%nny' ORDER BY ts DESC

documentation/webdocs/markdowndocs/TAOS SQL.md

@@ -39,8 +39,8 @@ The full list of data types is listed below.  For string types of data, we will
 | 6    |   DOUBLE    |    8    | A standard nullable double float type with 15-16 significant digits and a range of [-1.7E308, 1.7E308]​ |
 | 7    |    BOOL     |    1    | A nullable boolean type, [**`true`**, **`false`**]           |
 | 8    |  TIMESTAMP  |    8    | A nullable timestamp type with the same usage as the primary column timestamp |
-| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, any exceeded chars will be automatically truncated. This type of string only supports ASCii encoded chars. |
+| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 65526, but as maximum row size is 64K bytes, the actual upper limit will generally less than 65526. This type of string only supports ASCii encoded chars. |
-| 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, any exceeded chars will be truncated. The **`NCHAR`** type supports Unicode encoded chars. |
+| 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars. The **`NCHAR`** type supports Unicode encoded chars. |
 
 All the keywords in a SQL statement are case-insensitive, but strings values are case-sensitive and must be quoted by a pair of `'` or `"`. To quote a `'` or a `"` , you can use the escape character `\`.
 
@@ -86,7 +86,7 @@ All the keywords in a SQL statement are case-insensitive, but strings values are
     
     1) The first column must be a `timestamp`, and the system will set it as the primary key.
     
-    2) The record size is limited to 4096 bytes
+    2) The record size is limited to 64k bytes
     
     3) For `binary` or `nchar` data types, the length must be specified. For example, binary(20) means a binary data type with 20 bytes.
 

documentation/webdocs/markdowndocs/connector-ch.md

@@ -2,6 +2,8 @@
 
 TDengine提供了丰富的应用程序开发接口，其中包括C/C++、JAVA、Python、RESTful、Go等，便于用户快速开发应用。
 
+注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。
+
 ## C/C++ Connector
 
 C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）：
@@ -1207,13 +1209,51 @@ TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序
 其中，最常用的文件列出如下：
 
 + Client可执行文件: /usr/local/taos/bin/taos 软连接到 /usr/local/bin/taos
+
 + 配置文件: /usr/local/taos/cfg/taos.cfg 软连接到 /etc/taos/taos.cfg
+
 + 驱动程序目录: /usr/local/taos/driver/libtaos.1.6.5.1.dylib 软连接到 /usr/local/lib/libtaos.dylib
+
 + 驱动程序头文件: /usr/local/taos/include/taos.h 软连接到 /usr/local/include/taos.h
+
 + 日志目录（第一次运行程序时生成）：~/TDengineLog
 
   
 
+
+## MQTT客户端
+
+MQTT客户端实现了订阅MQTT Broker的特定Topic将Json数据进行转换入库的功能，任何终端只要将数据发给特定的Topic 即可,不用再编写转换器或者数据解析程序。如果终端量大，需要 Mqtt Broker 群集，这里不再详述。
+
+#### 如何配置？
+
+首先需要在 taos.cfg 中打开配置项 mqtt 用来启用， 再通过修改 mqttBrokerAddress  的值来配置连接，格式为: 
+
+> mqtt://username:password@hostname:port/path/
+
+ 例如:
+
+> mqtt://127.0.0.1:1883/taos/  mqtt://root@kissme@127.0.0.1:1883/taos/
+
+
+#### Topic 格式说明
+
+ Mqtt 的topic格式为
+
+> /<path>/<token>/<db name>/<table name>/
+
+因此TDengine的Mqtt客户端会订阅:
+
+> /taos/+/+/+/+/  
+
+例如: 
+
+>  /taos/token/db/t/  
+
+注意: 测试时如果需要使用到Mqtt Broker 推荐使用 [mosquitto](http://mosquitto.org/) ，客户端可以使用  [MQTT.fx ](http://www.jensd.de/)
+
+
+
 [1]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver
 [2]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver
 [3]: https://github.com/taosdata/TDengine

importSampleData/app/main.go

@@ -18,7 +18,7 @@ import (
 	"sync"
 	"time"
 
-	_ "github.com/taosdata/TDengine/src/connector/go/src/taosSql"
+	_ "github.com/taosdata/TDengine/src/connector/go/taosSql"
 )
 
 const (
@@ -634,6 +634,7 @@ func insertData(threadIndex, start, end int, wg  *sync.WaitGroup, successRows []
 					if  appendRows == batch {
 						// executebatch
 						insertSql := buffers.String()
+						connection.Exec("use " + db)
 						affectedRows := executeBatchInsert(insertSql, connection)
 
 						successRows[threadIndex] += affectedRows
@@ -658,6 +659,7 @@ func insertData(threadIndex, start, end int, wg  *sync.WaitGroup, successRows []
 		if  appendRows > 0 {
 			// executebatch
 			insertSql := buffers.String()
+			connection.Exec("use " + db)
 			affectedRows := executeBatchInsert(insertSql, connection)
 			
 			successRows[threadIndex] += affectedRows

packaging/cfg/taos.cfg

@@ -155,7 +155,7 @@
 # maxVnodeConnections   10000
 
 # mnode take into account while balance, for cluster version only 
-# mgmtEqualVnodeNum     4
+# mnodeEqualVnodeNum     4
 
 # number of seconds allowed for a dnode to be offline, for cluster version only 
 # offlineThreshold      864000
@@ -166,6 +166,15 @@
 # start system monitor module
 # monitor               1
 
+# start http service
+# mqtt                  0
+
+# mqtt uri  
+# mqttBrokerAddress  mqtt://username:password@hostname:1883/taos/
+
+# mqtt client name 
+# mqttBrokerClientId  taos_mqtt
+
 # maximum number of rows returned by the restful interface
 # restfulRowLimit       10240
 
@@ -244,5 +253,8 @@
 # debug flag for system monitor 
 # monitorDebugFlag      131
 
+#debug flag for mqtt client 
+# mqttDebugFlag         131
+
 # debug flag for TAOS TIMER
 # tmrDebugFlag          131

packaging/release.sh

@@ -162,18 +162,18 @@ done
 
 # output the version info to the buildinfo file.
 build_time=$(date +"%F %R")
-echo "char version[64] = \"${version}\";"                             > ${versioninfo}
+echo "char version[12] = \"${version}\";"                             > ${versioninfo}
-echo "char compatible_version[64] = \"${compatible_version}\";"      >> ${versioninfo}
+echo "char compatible_version[12] = \"${compatible_version}\";"      >> ${versioninfo}
-echo "char gitinfo[128] = \"$(git rev-parse --verify HEAD)\";"       >> ${versioninfo}
+echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";"       >> ${versioninfo}
 if [ "$verMode" != "cluster" ]; then
-  echo "char gitinfoOfInternal[128] = \"\";"                         >> ${versioninfo}
+  echo "char gitinfoOfInternal[48] = \"\";"                         >> ${versioninfo}
 else
   enterprise_dir="${top_dir}/../enterprise"
   cd ${enterprise_dir}
-  echo "char gitinfoOfInternal[128] = \"$(git rev-parse --verify HEAD)\";"  >> ${versioninfo}
+  echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";"  >> ${versioninfo}
   cd ${curr_dir}
 fi
-echo "char buildinfo[512] = \"Built by ${USER} at ${build_time}\";"  >> ${versioninfo}
+echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";"  >> ${versioninfo}
 echo ""                                                              >> ${versioninfo}
 tmp_version=$(echo $version | tr -s "." "_")
 if [ "$verMode" == "cluster" ]; then

src/client/inc/tscLog.h

@@ -22,26 +22,13 @@ extern "C" {
 
 #include "tlog.h"
 
-extern int32_t cdebugFlag;
+extern int32_t cDebugFlag;
 
-#define tscError(...)                        \
+#define tscError(...)  { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR TSC ", cDebugFlag, __VA_ARGS__); }}
-  if (cdebugFlag & DEBUG_ERROR) {            \
+#define tscWarn(...)   { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN TSC ", cDebugFlag, __VA_ARGS__); }}
-    taosPrintLog("ERROR TSC ", 255, __VA_ARGS__); \
+#define tscTrace(...)  { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }}
-  }
+#define tscDump(...)   { if (cDebugFlag & DEBUG_TRACE) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }}
-#define tscWarn(...)                                \
+#define tscPrint(...)  { taosPrintLog("TSC ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
-  if (cdebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  TSC ", cdebugFlag, __VA_ARGS__); \
-  }
-#define tscTrace(...)                         \
-  if (cdebugFlag & DEBUG_TRACE) {             \
-    taosPrintLog("TSC ", cdebugFlag, __VA_ARGS__); \
-  }
-#define tscPrint(...) \
-  { taosPrintLog("TSC ", 255, __VA_ARGS__); }
-#define tscDump(...)                                      \
-  if (cdebugFlag & DEBUG_TRACE) {                         \
-    taosPrintLongString("TSC ", cdebugFlag, __VA_ARGS__); \
-  }
 
 #ifdef __cplusplus
 }

src/client/inc/tscProfile.h

@@ -26,7 +26,7 @@ void tscAddIntoSqlList(SSqlObj *pSql);
 void tscRemoveFromSqlList(SSqlObj *pSql);
 void tscAddIntoStreamList(SSqlStream *pStream);
 void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj);
-char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj);
+int  tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj);
 void tscKillQuery(STscObj *pObj, uint32_t killId);
 void tscKillStream(STscObj *pObj, uint32_t killId);
 void tscKillConnection(STscObj *pObj);

src/client/inc/tscSecondaryMerge.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 #include "qextbuffer.h"
-#include "qinterpolation.h"
+#include "qfill.h"
 #include "taosmsg.h"
 #include "tlosertree.h"
 #include "tsclient.h"
@@ -60,22 +60,24 @@ typedef struct SLocalReducer {
   char *                 prevRowOfInput;
   tFilePage *            pResultBuf;
   int32_t                nResultBufSize;
-  char *                 pBufForInterpo;  // intermediate buffer for interpolation
+//  char *                 pBufForInterpo;  // intermediate buffer for interpolation
   tFilePage *            pTempBuffer;
   struct SQLFunctionCtx *pCtx;
   int32_t                rowSize;      // size of each intermediate result.
+  int32_t                finalRowSize; // final result row size
   int32_t                status;       // denote it is in reduce process, in reduce process, it
   bool                   hasPrevRow;   // cannot be released
   bool                   hasUnprocessedRow;
   tOrderDescriptor *     pDesc;
   SColumnModel *         resColModel;
   tExtMemBuffer **       pExtMemBuffer;      // disk-based buffer
-  SInterpolationInfo     interpolationInfo;  // interpolation support structure
+  SFillInfo*             pFillInfo;          // interpolation support structure
   char *                 pFinalRes;          // result data after interpo
   tFilePage *            discardData;
   SResultInfo *          pResInfo;
   bool                   discard;
   int32_t                offset;             // limit offset value
+  bool                   orderPrjOnSTable;   // projection query on stable
 } SLocalReducer;
 
 typedef struct SSubqueryState {
@@ -116,7 +118,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
  * create local reducer to launch the second-stage reduce process at client site
  */
 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                           SColumnModel *finalModel, SSqlCmd *pSqlCmd, SSqlRes *pRes);
+                           SColumnModel *finalModel, SSqlObj* pSql);
 
 void tscDestroyLocalReducer(SSqlObj *pSql);
 

src/client/inc/tscUtil.h

@@ -31,19 +31,19 @@ extern "C" {
 #include "tscSecondaryMerge.h"
 #include "tsclient.h"
 
-#define UTIL_TABLE_IS_SUPERTABLE(metaInfo)  \
+#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
 #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
   
-#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo)\
+#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\
-  (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
+  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
 
 #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0)
 
 typedef struct SParsedColElem {
   int16_t colIndex;
-  int16_t offset;
+  uint16_t offset;
 } SParsedColElem;
 
 typedef struct SParsedDataColInfo {
@@ -132,10 +132,7 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd);
 void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex,
                                   SSchema* pColSchema, int16_t isTag);
 
-//void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex);
+int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql);
-void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index);
-
-int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql);
 void    tscClearInterpInfo(SQueryInfo* pQueryInfo);
 
 bool tscIsInsertData(char* sqlstr);
@@ -175,7 +172,7 @@ SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn
 
 SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
                            int16_t size);
-int32_t   tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo);
+size_t   tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo);
 
 SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index);
 void      tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
@@ -183,7 +180,7 @@ void      tscSqlExprInfoDestroy(SArray* pExprInfo);
 
 SColumn* tscColumnClone(const SColumn* src);
 SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex);
-void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex);
+SArray* tscColumnListClone(const SArray* src, int16_t tableIndex);
 void tscColumnListDestroy(SArray* pColList);
 
 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
@@ -264,6 +261,11 @@ bool hasMoreVnodesToTry(SSqlObj *pSql);
 void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)());
+int  tscSetMgmtIpListFromCfg(const char *first, const char *second);
+
+void* malloc_throw(size_t size);
+void* calloc_throw(size_t nmemb, size_t size);
+char* strdup_throw(const char* str);
 
 #ifdef __cplusplus
 }

src/client/inc/tsclient.h

@@ -30,10 +30,10 @@ extern "C" {
 #include "tsqlfunction.h"
 #include "tutil.h"
 
+#include "qExecutor.h"
 #include "qsqlparser.h"
 #include "qsqltype.h"
 #include "qtsbuf.h"
-#include "queryExecutor.h"
 
 // forward declaration
 struct SSqlInfo;
@@ -49,17 +49,14 @@ typedef struct STableComInfo {
   uint8_t numOfTags;
   uint8_t precision;
   int16_t numOfColumns;
-  int16_t rowSize;
+  int32_t rowSize;
 } STableComInfo;
 
 typedef struct STableMeta {
-  // super table if it is created according to super table, otherwise, tableInfo is used
-  union {
-    struct STableMeta *pSTable;
   STableComInfo tableInfo;
-  };
   uint8_t       tableType;
   int16_t       sversion;
+  int16_t       tversion;
   SCMVgroupInfo vgroupInfo;
   int32_t       sid;       // the index of one table in a virtual node
   uint64_t      uid;       // unique id of a table
@@ -88,7 +85,7 @@ typedef struct SSqlExpr {
   int16_t   functionId;     // function id in aAgg array
   int16_t   resType;        // return value type
   int16_t   resBytes;       // length of return value
-  int16_t   interBytes;  // inter result buffer size
+  int32_t   interBytes;  // inter result buffer size
   int16_t   numOfParams;    // argument value of each function
   tVariant  param[3];       // parameters are not more than 3
   int32_t   offset;         // sub result column value of arithmetic expression.
@@ -210,11 +207,11 @@ typedef struct SQueryInfo {
   SLimitVal        slimit;
   STagCond         tagCond;
   SOrderVal        order;
-  int16_t          interpoType;  // interpolate type
+  int16_t          fillType;    // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf *  tsBuf;
-  int64_t *        defaultVal;   // default value for interpolation
+  int64_t *        fillVal;      // default value for fill
   char *           msg;          // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;  // limit for current sub clause
 
@@ -225,18 +222,15 @@ typedef struct SQueryInfo {
 typedef struct {
   int     command;
   uint8_t msgType;
-
-  union {
-    bool   existsCheck;     // check if the table exists or not
   bool    autoCreated;        // if the table is missing, on-the-fly create it. during getmeterMeta
   int8_t  dataSourceType;     // load data from file or not
-  };
 
   union {
     int32_t count;
     int32_t numOfTablesInSubmit;
   };
 
+  int32_t      insertType;
   int32_t      clauseIndex;  // index of multiple subclause query
   int8_t       parseFinished;
   short        numOfCols;
@@ -245,14 +239,12 @@ typedef struct {
   int32_t      payloadLen;
   SQueryInfo **pQueryInfo;
   int32_t      numOfClause;
-
-  SDataBlockList *pDataBlocks;  // submit data blocks after parsing sql
   char *       curSql;       // current sql, resume position of sql after parsing paused
   void *       pTableList;   // referred table involved in sql
-
+  int32_t      batchSize;    // for parameter ('?') binding and batch processing
-  // for parameter ('?') binding and batch processing
-  int32_t batchSize;
   int32_t      numOfParams;
+
+  SDataBlockList *pDataBlocks;  // submit data blocks after parsing sql
 } SSqlCmd;
 
 typedef struct SResRec {
@@ -263,7 +255,7 @@ typedef struct SResRec {
 typedef struct {
   int64_t               numOfRows;                  // num of results in current retrieved
   int64_t               numOfTotal;                 // num of total results
-  int64_t               numOfTotalInCurrentClause;  // num of total result in current subclause
+  int64_t               numOfClauseTotal;           // num of total result in current subclause
   char *                pRsp;
   int32_t               rspType;
   int32_t               rspLen;
@@ -291,16 +283,14 @@ typedef struct {
 typedef struct STscObj {
   void *             signature;
   void *             pTimer;
-  char               mgmtIp[TSDB_USER_LEN];
-  uint16_t           mgmtPort;
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
-  char               acctId[TSDB_DB_NAME_LEN];
+  char               acctId[TSDB_ACCT_LEN];
-  char               db[TSDB_TABLE_ID_LEN];
+  char               db[TSDB_DB_NAME_LEN];
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;
-  struct SSqlObj *   pSql;
+  uint32_t           connId;
   struct SSqlObj *   pHb;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
@@ -327,7 +317,7 @@ typedef struct SSqlObj {
   tsem_t           rspSem;
   SSqlCmd          cmd;
   SSqlRes          res;
-  uint8_t          numOfSubs;
+  uint16_t         numOfSubs;
   struct SSqlObj **pSubs;
   struct SSqlObj * prev, *next;
 } SSqlObj;
@@ -365,10 +355,10 @@ void    tscInitMsgsFp();
 
 int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
 
-void tscProcessMsgFromServer(SRpcMsg *rpcMsg);
+void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
 int  tscProcessSql(SSqlObj *pSql);
 
-int  tscRenewMeterMeta(SSqlObj *pSql, char *tableId);
+int  tscRenewTableMeta(SSqlObj *pSql, char *tableId);
 void tscQueueAsyncRes(SSqlObj *pSql);
 
 void tscQueueAsyncError(void(*fp), void *param, int32_t code);
@@ -409,13 +399,15 @@ void tscCloseTscObj(STscObj *pObj);
 
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos);
+void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ;
 
+int doAsyncParseSql(SSqlObj* pSql);
 void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen);
 
 void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql);
 void tscKillSTableQuery(SSqlObj *pSql);
 void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
-bool tscIsUpdateQuery(STscObj *pObj);
+bool tscIsUpdateQuery(SSqlObj* pSql);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
 
 char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
@@ -438,6 +430,9 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
 
+int32_t tscCompareTidTags(const void* p1, const void* p2);
+void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables);
+
 #ifdef __cplusplus
 }
 #endif

src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h

@@ -62,7 +62,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp
  * Method:    executeQueryImp
  * Signature: ([BJ)I
  */
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp
+JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp
   (JNIEnv *, jobject, jbyteArray, jlong);
 
 /*
@@ -71,7 +71,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp
  * Signature: (J)I
  */
 JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp
-  (JNIEnv *, jobject, jlong);
+  (JNIEnv *, jobject, jlong, jlong);
 
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector
@@ -87,7 +87,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp
  * Signature: (J)J
  */
 JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp
-  (JNIEnv *, jobject, jlong);
+  (JNIEnv *env, jobject jobj, jlong con, jlong tres);
 
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector
@@ -103,7 +103,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp
  * Signature: (J)I
  */
 JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp
-  (JNIEnv *, jobject, jlong);
+  (JNIEnv *env, jobject jobj, jlong con, jlong res);
 
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector
@@ -142,8 +142,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp
  * Method:    consumeImp
  * Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData;
  */
-JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp
+JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp
-  (JNIEnv *, jobject, jlong, jint);
+  (JNIEnv *, jobject, jlong);
 
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector

src/client/src/TSDBJNIConnector.c

@@ -13,29 +13,35 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "com_taosdata_jdbc_TSDBJNIConnector.h"
 #include "os.h"
 #include "taos.h"
-#include "tscSubquery.h"
+#include "tlog.h"
 #include "tscUtil.h"
 #include "tsclient.h"
-#include "tlog.h"
 #include "ttime.h"
 
+#include "com_taosdata_jdbc_TSDBJNIConnector.h"
+
 #define jniError(...)                                        \
-  if (jnidebugFlag & DEBUG_ERROR) {                   \
+  {                                                          \
-    taosPrintLog("ERROR JNI ", jnidebugFlag, __VA_ARGS__); \
+    if (jniDebugFlag & DEBUG_ERROR) {                        \
+      taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); \
+    }                                                        \
   }
 #define jniWarn(...)                                        \
-  if (jnidebugFlag & DEBUG_WARN) {                    \
+  {                                                         \
-    taosPrintLog("WARN  JNI ", jnidebugFlag, __VA_ARGS__); \
+    if (jniDebugFlag & DEBUG_WARN) {                        \
+      taosPrintLog("WARN JNI ", jniDebugFlag, __VA_ARGS__); \
+    }                                                       \
   }
 #define jniTrace(...)                                  \
-  if (jnidebugFlag & DEBUG_TRACE) {             \
+  {                                                    \
-    taosPrintLog("JNI ", jnidebugFlag, __VA_ARGS__); \
+    if (jniDebugFlag & DEBUG_TRACE) {                  \
+      taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \
+    }                                                  \
   }
 #define jniPrint(...) \
-  { taosPrintLog("JNI ", 255, __VA_ARGS__); }
+  { taosPrintLog("JNI ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
 
 int __init = 0;
 
@@ -127,7 +133,8 @@ void jniGetGlobalMethod(JNIEnv *env) {
   jniTrace("native method register finished");
 }
 
-JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, jstring jPath, jboolean jAutoDump) {
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode,
+                                                                               jstring jPath, jboolean jAutoDump) {
   if (jPath != NULL) {
     const char *path = (*env)->GetStringUTFChars(env, jPath, NULL);
     taosSetAllocMode(jMode, path, !!jAutoDump);
@@ -182,14 +189,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv
     }
     (*env)->ReleaseStringUTFChars(env, optionValue, charset);
   } else if (optionIndex == TSDB_OPTION_TIMEZONE) {
-    const char *timezone = (*env)->GetStringUTFChars(env, optionValue, NULL);
+    const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL);
-    if (timezone && strlen(timezone) != 0) {
+    if (tz1 && strlen(tz1) != 0) {
-      res = taos_options(TSDB_OPTION_TIMEZONE, timezone);
+      res = taos_options(TSDB_OPTION_TIMEZONE, tz1);
       jniTrace("set timezone to %s, result:%d", timezone, res);
     } else {
       jniTrace("input timezone is empty");
     }
-    (*env)->ReleaseStringUTFChars(env, optionValue, timezone);
+    (*env)->ReleaseStringUTFChars(env, optionValue, tz1);
   } else {
     jniError("option index:%d is not found", optionIndex);
   }
@@ -251,7 +258,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn
   return ret;
 }
 
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj,
+JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj,
                                                                                 jbyteArray jsql, jlong con) {
   TAOS *tscon = (TAOS *)con;
   if (tscon == NULL) {
@@ -279,60 +286,71 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J
 
   jniTrace("jobj:%p, conn:%p, sql:%s", jobj, tscon, dst);
 
-  int code = taos_query(tscon, dst);
+  SSqlObj *pSql = taos_query(tscon, dst);
-  if (code != 0) {
+  int32_t  code = taos_errno(pSql);
-    jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(tscon));
+
-    free(dst);
+  if (code != TSDB_CODE_SUCCESS) {
-    return JNI_TDENGINE_ERROR;
+    jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(pSql));
   } else {
     int32_t affectRows = 0;
-    SSqlObj *pSql = ((STscObj *)tscon)->pSql;
-
     if (pSql->cmd.command == TSDB_SQL_INSERT) {
-      affectRows = taos_affected_rows(tscon);
+      affectRows = taos_affected_rows(pSql);
       jniTrace("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows);
     } else {
       jniTrace("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code));
     }
+  }
 
   free(dst);
-    return affectRows;
+  return (jlong)pSql;
-  }
 }
 
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con) {
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con, jlong tres) {
   TAOS *tscon = (TAOS *)con;
   if (tscon == NULL) {
     jniError("jobj:%p, connection is closed", jobj);
-    return (jint)-TSDB_CODE_INVALID_CONNECTION;
+    return (jint)TSDB_CODE_TSC_INVALID_CONNECTION;
   }
 
-  return (jint)-taos_errno(tscon);
+  if ((void *)tres == NULL) {
+    jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon);
+    return JNI_RESULT_SET_NULL;
   }
 
-JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong con) {
+  TAOS_RES *pSql = (TAOS_RES *)tres;
-  TAOS *tscon = (TAOS *)con;
+
-  return (*env)->NewStringUTF(env, (const char *)taos_errstr(tscon));
+  return (jint)taos_errno(pSql);
 }
 
-JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con) {
+JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong tres) {
+  TAOS_RES *pSql = (TAOS_RES *)tres;
+  return (*env)->NewStringUTF(env, (const char *)taos_errstr(pSql));
+}
+
+JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con,
+                                                                                jlong tres) {
   TAOS *tscon = (TAOS *)con;
   if (tscon == NULL) {
     jniError("jobj:%p, connection is closed", jobj);
     return JNI_CONNECTION_NULL;
   }
 
-  jlong ret = 0;
+  if ((void *)tres == NULL) {
-
+    jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon);
-  if (tscIsUpdateQuery(tscon)) {
+    return JNI_RESULT_SET_NULL;
-    ret = 0;  // for update query, no result pointer
-    jniTrace("jobj:%p, conn:%p, no resultset", jobj, tscon);
-  } else {
-    ret = (jlong) taos_use_result(tscon);
-    jniTrace("jobj:%p, conn:%p, get resultset:%p", jobj, tscon, (void *) ret);
   }
 
-  return ret;
+  SSqlObj *pSql = (TAOS_RES *)tres;
+  STscObj *pObj = pSql->pTscObj;
+
+  if (tscIsUpdateQuery(pSql)) {
+    taos_free_result(pSql);  // free result here
+    jniTrace("jobj:%p, conn:%p, no resultset, %p", jobj, pObj, (void *)tres);
+    return 0;
+  } else {
+    jniTrace("jobj:%p, conn:%p, get resultset, %p", jobj, pObj, (void *)tres);
+    return tres;
+  }
 }
 
 JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con,
@@ -353,17 +371,21 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(
   return JNI_SUCCESS;
 }
 
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj,
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, jlong con,
-                                                                                  jlong con) {
+                                                                                  jlong res) {
   TAOS *tscon = (TAOS *)con;
   if (tscon == NULL) {
     jniError("jobj:%p, connection is closed", jobj);
     return JNI_CONNECTION_NULL;
   }
 
-  jint ret = taos_affected_rows(tscon);
+  if ((void *)res == NULL) {
+    jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon);
+    return JNI_RESULT_SET_NULL;
+  }
 
-  jniTrace("jobj:%p, conn:%p, affect rows:%d", jobj, tscon, (void *)con, ret);
+  jint ret = taos_affected_rows((SSqlObj *)res);
+  jniTrace("jobj:%p, conn:%p, sql:%p, affect rows:%d", jobj, tscon, (void *)con, res, ret);
 
   return ret;
 }
@@ -449,7 +471,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
 
   TAOS_ROW row = taos_fetch_row(result);
   if (row == NULL) {
-    int tserrno = taos_errno(tscon);
+    int tserrno = taos_errno(result);
     if (tserrno == 0) {
       jniTrace("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, res, num_fields);
       return JNI_FETCH_END;
@@ -486,14 +508,12 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
         float fv = 0;
         fv = GET_FLOAT_VAL(row[i]);
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv);
-      }
+      } break;
-        break;
       case TSDB_DATA_TYPE_DOUBLE: {
         double dv = 0;
         dv = GET_DOUBLE_VAL(row[i]);
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv);
-      }
+      } break;
-        break;
       case TSDB_DATA_TYPE_BINARY: {
         strncpy(tmp, row[i], (size_t)fields[i].bytes);  // handle the case that terminated does not exist
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp));
@@ -562,110 +582,24 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNI
   return sub;
 }
 
-static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, int num_fields) {
+JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) {
-  jobject rowobj = (*env)->NewObject(env, g_rowdataClass, g_rowdataConstructor, num_fields);
-  jniTrace("created a rowdata object, rowobj:%p", rowobj);
-
-  for (int i = 0; i < num_fields; i++) {
-    if (row[i] == NULL) {
-      continue;
-    }
-
-    switch (fields[i].type) {
-      case TSDB_DATA_TYPE_BOOL:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetBooleanFp, i, (jboolean)(*((char *)row[i]) == 1));
-        break;
-      case TSDB_DATA_TYPE_TINYINT:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteFp, i, (jbyte) * ((char *)row[i]));
-        break;
-      case TSDB_DATA_TYPE_SMALLINT:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetShortFp, i, (jshort) * ((short *)row[i]));
-        break;
-      case TSDB_DATA_TYPE_INT:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetIntFp, i, (jint) * (int *)row[i]);
-        break;
-      case TSDB_DATA_TYPE_BIGINT:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetLongFp, i, (jlong) * ((int64_t *)row[i]));
-        break;
-      case TSDB_DATA_TYPE_FLOAT: {
-        float fv = 0;
-        fv = GET_FLOAT_VAL(row[i]);
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv);
-      }
-        break;
-      case TSDB_DATA_TYPE_DOUBLE:{
-        double dv = 0;
-        dv = GET_DOUBLE_VAL(row[i]);
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv);
-      }
-        break;
-      case TSDB_DATA_TYPE_BINARY: {
-        char tmp[TSDB_MAX_BYTES_PER_ROW] = {0};
-        strncpy(tmp, row[i], (size_t) fields[i].bytes);  // handle the case that terminated does not exist
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp));
-
-        memset(tmp, 0, (size_t) fields[i].bytes);
-        break;
-      }
-      case TSDB_DATA_TYPE_NCHAR:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i,
-                              jniFromNCharToByteArray(env, (char*)row[i], fields[i].bytes));
-        break;
-      case TSDB_DATA_TYPE_TIMESTAMP:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i]));
-        break;
-      default:
-        break;
-    }
-  }
-  return rowobj;
-}
-
-JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, jint timeout) {
   jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub);
   jniGetGlobalMethod(env);
 
   TAOS_SUB *tsub = (TAOS_SUB *)sub;
-  jobject   rows = (*env)->NewObject(env, g_arrayListClass, g_arrayListConstructFp);
 
-  int64_t start = taosGetTimestampMs();
-  int count = 0;
-
-  while (true) {
   TAOS_RES *res = taos_consume(tsub);
+
   if (res == NULL) {
     jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub);
-      return NULL;
+    return 0l;
   }
 
-    TAOS_FIELD *fields = taos_fetch_fields(res);
+  return (long)res;
-    int         num_fields = taos_num_fields(res);
-    while (true) {
-      TAOS_ROW row = taos_fetch_row(res);
-      if (row == NULL) {
-        break;
-      }
-      jobject rowobj = convert_one_row(env, row, fields, num_fields);
-      (*env)->CallBooleanMethod(env, rows, g_arrayListAddFp, rowobj);
-      count++;
 }
 
-    if (count > 0) {
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub,
-      break;
+                                                                              jboolean keepProgress) {
-    }
-    if (timeout == -1) {
-      continue;
-    }
-    if (((int)(taosGetTimestampMs() - start)) >= timeout) {
-      jniTrace("jobj:%p, sub:%ld, timeout", jobj, sub);
-      break;
-    }
-  }
-
-  return rows;
-}
-
-JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, jboolean keepProgress) {
   TAOS_SUB *tsub = (TAOS_SUB *)sub;
   taos_unsubscribe(tsub, keepProgress);
 }

src/client/src/tscAsync.c

@@ -40,38 +40,39 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
 static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
 
-void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
+int doAsyncParseSql(SSqlObj* pSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   SSqlRes* pRes = &pSql->res;
-  
+  int32_t code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
-  pSql->signature = pSql;
+  if (code != TSDB_CODE_SUCCESS) {
-  pSql->param     = param;
-  pSql->pTscObj   = pObj;
-  pSql->maxRetry  = TSDB_MAX_REPLICA_NUM;
-  pSql->fp = fp;
-  
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     tscError("failed to malloc payload");
-    tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY);
+    tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
-    return;
+    return code;
-  }
-  
-  pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
-  if (pSql->sqlstr == NULL) {
-    tscError("%p failed to malloc sql string buffer", pSql);
-    tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY);
-    free(pCmd->payload);
-    return;
   }
   
   pRes->qhandle = 0;
   pRes->numOfRows = 1;
   
-  strtolower(pSql->sqlstr, sqlstr);
   tscDump("%p SQL: %s", pSql, pSql->sqlstr);
+  return tsParseSql(pSql, true);
+}
 
-  int32_t code = tsParseSql(pSql, true);
+void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
-  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+  pSql->signature = pSql;
+  pSql->param     = param;
+  pSql->pTscObj   = pObj;
+  pSql->maxRetry  = TSDB_MAX_REPLICA_NUM;
+  pSql->fp        = fp;
+  pSql->sqlstr = calloc(1, sqlLen + 1);
+  if (pSql->sqlstr == NULL) {
+    tscError("%p failed to malloc sql string buffer", pSql);
+    tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY);
+    return;
+  }
+  strtolower(pSql->sqlstr, sqlstr);
+  
+  int32_t code = doAsyncParseSql(pSql);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
   
   if (code != TSDB_CODE_SUCCESS) {
     pSql->res.code = code;
@@ -87,16 +88,16 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
     tscError("bug!!! pObj:%p", pObj);
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
     return;
   }
   
   int32_t sqlLen = strlen(sqlstr);
   if (sqlLen > tsMaxSQLStringLen) {
-    tscError("sql string too long");
+    tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
-    terrno = TSDB_CODE_INVALID_SQL;
+    terrno = TSDB_CODE_TSC_INVALID_SQL;
-    tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_SQL);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_SQL);
     return;
   }
   
@@ -105,8 +106,8 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa
   SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
     tscError("failed to malloc sqlObj");
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY);
     return;
   }
   
@@ -145,9 +146,9 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
     return;
   }
   
-  // local reducer has handle this situation during super table non-projection query.
+  // local merge has handle this situation during super table non-projection query.
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
-    pRes->numOfTotalInCurrentClause += pRes->numOfRows;
+    pRes->numOfClauseTotal += pRes->numOfRows;
   }
 
   (*pSql->fetchFp)(param, tres, numOfRows);
@@ -165,7 +166,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
   SSqlRes *pRes = &pSql->res;
 
   if ((pRes->qhandle == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) {
-    if (pRes->qhandle == 0) {
+    if (pRes->qhandle == 0 && numOfRows != 0) {
       tscError("qhandle is NULL");
     } else {
       pRes->code = numOfRows;
@@ -201,7 +202,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
   SSqlObj *pSql = (SSqlObj *)taosa;
   if (pSql == NULL || pSql->signature != pSql) {
     tscError("sql object is NULL");
-    tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
     return;
   }
 
@@ -210,7 +211,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
 
   if (pRes->qhandle == 0) {
     tscError("qhandle is NULL");
-    tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_QHANDLE);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_QHANDLE);
     return;
   }
 
@@ -222,9 +223,30 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
   tscResetForNextRetrieve(pRes);
   
   // handle the sub queries of join query
-  if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     tscFetchDatablockFromSubquery(pSql);
+  } else if (pRes->completed && pCmd->command == TSDB_SQL_FETCH) {
+    if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
+      tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode);
+      return;
     } else {
+      /*
+       * all available virtual node has been checked already, now we need to check
+       * for the next subclause queries
+       */
+      if (pCmd->clauseIndex < pCmd->numOfClause - 1) {
+        tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode);
+        return;
+      }
+    
+      /*
+       * 1. has reach the limitation
+       * 2. no remain virtual nodes to be retrieved anymore
+       */
+      (*pSql->fetchFp)(param, pSql, 0);
+    }
+    return;
+  } else { // current query is not completed, continue retrieve from node
     if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
       pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
     }
@@ -237,7 +259,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
   SSqlObj *pSql = (SSqlObj *)taosa;
   if (pSql == NULL || pSql->signature != pSql) {
     tscError("sql object is NULL");
-    tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
     return;
   }
 
@@ -246,7 +268,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
 
   if (pRes->qhandle == 0) {
     tscError("qhandle is NULL");
-    tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_QHANDLE);
+    tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_QHANDLE);
     return;
   }
 
@@ -420,11 +442,18 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   }
 
   if (pSql->pStream == NULL) {
-    // check if it is a sub-query of super table query first, if true, enter another routine
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
-    if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
+    // check if it is a sub-query of super table query first, if true, enter another routine
+    if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY)) {
+      tscTrace("%p update table meta in local cache, continue to process sql and send corresponding subquery", pSql);
+
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+      if (pTableMetaInfo->pTableMeta == NULL){
+        code = tscGetTableMeta(pSql, pTableMetaInfo);
+        assert(code == TSDB_CODE_SUCCESS);      
+      }
+      
       assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL);
 
       SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
@@ -433,34 +462,42 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0);
 
-      tscTrace("%p get metricMeta during super table query successfully", pSql);
+      // NOTE: the vgroupInfo for the queried super table must be existed here.
-      
+      assert(pTableMetaInfo->vgroupList != NULL);
-      code = tscGetTableMeta(pSql, pTableMetaInfo);
+      if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-      pRes->code = code;
+        return;
-
+      }
-      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+    } else {  // continue to process normal async query
-
-      code = tscGetSTableVgroupInfo(pSql, 0);
-      pRes->code = code;
-
-      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
-    } else {  // normal async query continues
       if (pCmd->parseFinished) {
-        tscTrace("%p re-send data to vnode in table Meta callback since sql parsed completed", pSql);
+        tscTrace("%p update table meta in local cache, continue to process sql and send corresponding query", pSql);
 
         STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
         code = tscGetTableMeta(pSql, pTableMetaInfo);
         assert(code == TSDB_CODE_SUCCESS);
 
-        if (pTableMetaInfo->pTableMeta) {
+        // if failed to process sql, go to error handler
-          // todo update the submit message according to the new table meta
+        if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-          // 1. table uid, 2. ip address
+          return;
-          code = tscSendMsgToServer(pSql);
-          if (code == TSDB_CODE_SUCCESS) return;
         }
+//          // todo update the submit message according to the new table meta
+//          // 1. table uid, 2. ip address
+//          code = tscSendMsgToServer(pSql);
+//          if (code == TSDB_CODE_SUCCESS) return;
+//        }
       } else {
+        tscTrace("%p continue parse sql after get table meta", pSql);
+
         code = tsParseSql(pSql, false);
-        if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+        if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STMT_INSERT)) {
+          STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
+          code = tscGetTableMeta(pSql, pTableMetaInfo);
+          assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->pTableMeta != NULL);
+
+          (*pSql->fp)(pSql->param, pSql, code);
+          return;
+        }
+        
+        if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
       }
     }
 
@@ -469,13 +506,13 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
     code = tscGetTableMeta(pSql, pTableMetaInfo);
     pRes->code = code;
 
-    if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+    if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
 
-    if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex);
       pRes->code = code;
 
-      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+      if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
     }
   }
 
@@ -487,15 +524,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
 
   if (pSql->pStream) {
     tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command);
-    /*
+    if (!pSql->cmd.parseFinished) {
-     * NOTE:
+      tsParseSql(pSql, false);
-     * transfer the sql function for super table query before get meter/metric meta,
+      sem_post(&pSql->rspSem);
-     * since in callback functions, only tscProcessSql(pStream->pSql) is executed!
+    }
-     */
+    return;
-    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-    
-    tscTansformSQLFuncForSTableQuery(pQueryInfo);
-    tscIncStreamExecutionCount(pSql->pStream);
   } else {
     tscTrace("%p get tableMeta successfully", pSql);
   }

src/client/src/tscFunctionImpl.c

@@ -16,8 +16,8 @@
 #include "os.h"
 #include "qast.h"
 #include "qextbuffer.h"
+#include "qfill.h"
 #include "qhistogram.h"
-#include "qinterpolation.h"
 #include "qpercentile.h"
 #include "qsyntaxtreefunction.h"
 #include "qtsbuf.h"
@@ -153,10 +153,10 @@ typedef struct SRateInfo {
 
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
-                          int16_t *bytes, int16_t *interBytes, int16_t extLength, bool isSuperTable) {
+                          int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
   if (!isValidDataType(dataType, dataBytes)) {
     tscError("Illegal data type %d or data type length %d", dataType, dataBytes);
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
   
   if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY ||
@@ -168,9 +168,10 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     return TSDB_CODE_SUCCESS;
   }
   
+  // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE
   if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct
     *type = TSDB_DATA_TYPE_BINARY;
-    *bytes = dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t); // (uid, tid) + VGID + TAGSIZE
+    *bytes = dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE;
     *interBytes = *bytes;
     return TSDB_CODE_SUCCESS;
   }
@@ -324,7 +325,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     *bytes = (int16_t)dataBytes;
     *interBytes = dataBytes + sizeof(SLastrowInfo);
   } else {
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
   
   return TSDB_CODE_SUCCESS;
@@ -340,16 +341,6 @@ bool stableQueryFunctChanged(int32_t funcId) {
  */
 void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; }
 
-void initResultInfo(SResultInfo *pResInfo) {
-  pResInfo->initialized = true;  // the this struct has been initialized flag
-  
-  pResInfo->complete = false;
-  pResInfo->hasResult = false;
-  pResInfo->numOfRes = 0;
-  
-  memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen);
-}
-
 void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) {
   assert(pResInfo->interResultBuf == NULL);
   
@@ -386,19 +377,17 @@ static bool function_setup(SQLFunctionCtx *pCtx) {
  */
 static void function_finalizer(SQLFunctionCtx *pCtx) {
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  
   if (pResInfo->hasResult != DATA_SET_FLAG) {
-    tscTrace("no result generated, result is set to NULL");
+    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+    } else {
       setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
     }
+  }
   
   doFinalizer(pCtx);
 }
 
-static bool usePreVal(SQLFunctionCtx *pCtx) {
-  return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size;
-}
-
 /*
  * count function does need the finalize, if data is missing, the default value, which is 0, is used
  * count function does not use the pCtx->interResBuf to keep the intermediate buffer
@@ -411,7 +400,7 @@ static void count_function(SQLFunctionCtx *pCtx) {
    * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true;
    * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false;
    */
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull;
   } else {
     if (pCtx->hasNull) {
@@ -477,7 +466,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32
   if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
     return BLK_DATA_NO_NEEDED;
   } else {
-    return BLK_DATA_FILEDS_NEEDED;
+    return BLK_DATA_STATIS_NEEDED;
   }
 }
 
@@ -536,7 +525,7 @@ static void do_sum(SQLFunctionCtx *pCtx) {
   int32_t notNullElems = 0;
   
   // Only the pre-computing information loaded and actual data does not loaded
-  if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) {
+  if (pCtx->preAggVals.isSet) {
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull);
     
@@ -689,7 +678,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) {
 }
 
 static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
-  return BLK_DATA_FILEDS_NEEDED;
+  return BLK_DATA_STATIS_NEEDED;
 }
 
 static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
@@ -767,7 +756,7 @@ static void avg_function(SQLFunctionCtx *pCtx) {
   SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
   double *  pVal = &pAvgInfo->sum;
   
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     // Pre-aggregation
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(notNullElems >= 0);
@@ -931,7 +920,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) {
 
 static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) {
   // data in current data block are qualified to the query
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(*notNullElems >= 0);
     
@@ -946,6 +935,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
       index = pCtx->preAggVals.statis.maxIndex;
     }
     
+    TSKEY key = TSKEY_INITIAL_VAL;
+    if (pCtx->ptsList != NULL) {
       /**
        * NOTE: work around the bug caused by invalid pre-calculated function.
        * Here the selectivity + ts will not return correct value.
@@ -956,7 +947,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
         index = 0;
       }
       
-    TSKEY key = pCtx->ptsList[index];
+      key = pCtx->ptsList[index];
+    }
     
     if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
       int64_t val = GET_INT64_VAL(tval);
@@ -1301,7 +1293,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   minMax_function_f(pCtx, index, 0);
   
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pResInfo->hasResult == DATA_SET_FLAG) {
+  if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
     char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
     *flag = DATA_SET_FLAG;
   }
@@ -1317,7 +1309,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   minMax_function_f(pCtx, index, 1);
   
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pResInfo->hasResult == DATA_SET_FLAG) {
+  if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
     char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
     *flag = DATA_SET_FLAG;
   }
@@ -1690,10 +1682,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
 }
 
 static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  assert(pCtx->order != TSDB_ORDER_ASC);
-    return;
-  }
-  
   void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
@@ -1847,13 +1836,14 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
   pResInfo->hasResult = DATA_SET_FLAG;
   
   SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf;
-  pInfo->ts = pCtx->param[0].i64Key;
+  pInfo->ts = pCtx->ptsList[0];
+  
   pInfo->hasResult = DATA_SET_FLAG;
   
   // set the result to final result buffer
   if (pResInfo->superTableQ) {
     SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes);
-    pInfo1->ts = pCtx->param[0].i64Key;
+    pInfo1->ts = pCtx->ptsList[0];
     pInfo1->hasResult = DATA_SET_FLAG;
     
     DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts);
@@ -1867,12 +1857,22 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) {
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
   if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
     if (pResInfo->hasResult != DATA_SET_FLAG) {
+      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+      } else {
         setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      }
+      
       return;
     }
   } else {
     if (pResInfo->hasResult != DATA_SET_FLAG) {
+      if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+        setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+      } else {
         setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+      }
+      
       return;
     }
   }
@@ -1893,12 +1893,12 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6
     memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen);
   } else {  // the tags are dumped from the ctx tag fields
     for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) {
-      SQLFunctionCtx* __ctx = pTagInfo->pTagCtxList[i];
+      SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i];
-      if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) {
+      if (ctx->functionId == TSDB_FUNC_TS_DUMMY) {
-        __ctx->tag = (tVariant) {.nType = TSDB_DATA_TYPE_BIGINT, .i64Key = tsKey};
+        ctx->tag = (tVariant) {.nType = TSDB_DATA_TYPE_BIGINT, .i64Key = tsKey};
       }
       
-      tVariantDump(&pTagInfo->pTagCtxList[i]->tag, dst->pTags + size, pTagInfo->pTagCtxList[i]->tag.nType);
+      tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true);
       size += pTagInfo->pTagCtxList[i]->outputBytes;
     }
   }
@@ -2215,7 +2215,6 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
 static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) {
   char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo);
   pTopBotInfo->res = (tValuePair**) tmp;
-  
   tmp += POINTER_BYTES * pCtx->param[0].i64Key;
   
   size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen;
@@ -2888,7 +2887,12 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
   SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
   
   if (pInfo->num == 0) {
+    if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+    } else {
       setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+    }
+    
     return;
   }
   
@@ -2906,15 +2910,15 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
   
   param[1][2] /= param[1][1];
   
-  sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]);
+  int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE;
+  size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}",
+      param[0][2], param[1][2]);
+  
+  varDataSetLen(pCtx->aOutputBuf, n);
   doFinalizer(pCtx);
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) {
-    return;
-  }
-  
   SET_VAL(pCtx, pCtx->size, 1);
   *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp;
 }
@@ -2965,14 +2969,8 @@ static void tag_project_function(SQLFunctionCtx *pCtx) {
   assert(pCtx->inputBytes == pCtx->outputBytes);
   
   for (int32_t i = 0; i < pCtx->size; ++i) {
-    char* output = pCtx->aOutputBuf;
+    tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true);
     
-    if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-      *(int16_t*) output = pCtx->tag.nLen;
-      output += VARSTR_HEADER_SIZE;
-    }
-    
-    tVariantDump(&pCtx->tag, output, pCtx->outputType);
     pCtx->aOutputBuf += pCtx->outputBytes;
   }
 }
@@ -2980,13 +2978,7 @@ static void tag_project_function(SQLFunctionCtx *pCtx) {
 static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   INC_INIT_VAL(pCtx, 1);
   
-  char* output = pCtx->aOutputBuf;
+  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
-  if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-    *(int16_t*) output = pCtx->tag.nLen;
-    output += VARSTR_HEADER_SIZE;
-  }
-  
-  tVariantDump(&pCtx->tag, output, pCtx->tag.nType);
   pCtx->aOutputBuf += pCtx->outputBytes;
 }
 
@@ -2999,30 +2991,12 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
  */
 static void tag_function(SQLFunctionCtx *pCtx) {
   SET_VAL(pCtx, 1, 1);
-  
+  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
-  char* output = pCtx->aOutputBuf;
-  
-  // todo refactor to dump length presented string(var string)
-  if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-    *(int16_t*) output = pCtx->tag.nLen;
-    output += VARSTR_HEADER_SIZE;
-  }
-  
-  tVariantDump(&pCtx->tag, output, pCtx->tag.nType);
 }
 
 static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   SET_VAL(pCtx, 1, 1);
-  
+  tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true);
-  char* output = pCtx->aOutputBuf;
-  
-  // todo refactor to dump length presented string(var string)
-  if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-    *(int16_t*) output = pCtx->tag.nLen;
-    output += VARSTR_HEADER_SIZE;
-  }
-  
-  tVariantDump(&pCtx->tag, output, pCtx->tag.nType);
 }
 
 static void copy_function(SQLFunctionCtx *pCtx) {
@@ -3078,7 +3052,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;  // direct previous may be null
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3110,7 +3084,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3141,7 +3115,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].dKey;
           *pTimestamp = pCtx->ptsList[i];
           pOutput += 1;
           pTimestamp += 1;
@@ -3172,7 +3146,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].dKey;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3204,7 +3178,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3236,7 +3210,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3415,8 +3389,9 @@ static void spread_function(SQLFunctionCtx *pCtx) {
   
   int32_t numOfElems = pCtx->size;
   
+  // todo : opt with pre-calculated result
   // column missing cause the hasNull to be true
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     
     // all data are null in current data block, ignore current data block
@@ -3442,14 +3417,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
         pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max));
       }
     }
-  } else {
-    if (pInfo->min > pCtx->param[1].dKey) {
-      pInfo->min = pCtx->param[1].dKey;
-    }
     
-    if (pInfo->max < pCtx->param[2].dKey) {
+    goto _spread_over;
-      pInfo->max = pCtx->param[2].dKey;
-    }
   }
   
   void *pData = GET_INPUT_CHAR(pCtx);
@@ -3617,299 +3586,6 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
   doFinalizer(pCtx);
 }
 
-static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                          int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int8_t  lastVal = TSDB_DATA_TINYINT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((char *)&data[i], type)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation
-    //      if (isNull((char *)&lastVal, type)) {
-    //        lastKey = primaryKey[i];
-    //        lastVal = data[i];
-    //      } else {
-    //        *wsum = lastVal * (primaryKey[i] - lastKey);
-    //        lastKey = primaryKey[i];
-    //        lastVal = data[i];
-    //      }
-    //    }
-  }
-}
-
-static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int16_t lastVal = TSDB_DATA_SMALLINT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  //  int64_t lastKey = 0;
-  //  int32_t lastVal = TSDB_DATA_INT_NULL;
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, int64_t *min, int64_t *max,
-                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  *min = INT64_MAX;
-  *max = INT64_MIN;
-  *minIndex = 0;
-  *maxIndex = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    *sum += data[i];
-    if (*min > data[i]) {
-      *min = data[i];
-      *minIndex = i;
-    }
-    
-    if (*max < data[i]) {
-      *max = data[i];
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-}
-
-static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, double *min, double *max, double *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  float fmin      = DBL_MAX;
-  float fmax      = -DBL_MAX;
-  double dsum     = 0;
-  *minIndex       = 0;
-  *maxIndex       = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    float fv = 0;
-    fv = GET_FLOAT_VAL(&(data[i]));
-    dsum += fv;
-    if (fmin > fv) {
-      fmin = fv;
-      *minIndex = i;
-    }
-    
-    if (fmax < fv) {
-      fmax = fv;
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-  
-  double csum = 0;
-  csum = GET_DOUBLE_VAL(sum);
-  csum += dsum;
-#ifdef _TD_ARM_32_
-  SET_DOUBLE_VAL_ALIGN(sum, &csum);
-  SET_DOUBLE_VAL_ALIGN(max, &fmax);
-  SET_DOUBLE_VAL_ALIGN(min, &fmin);
-#else
-  *sum = csum;
-  *max = fmax;
-  *min = fmin;
-#endif
-}
-
-static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, double *min, double *max, double *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  double dmin      = DBL_MAX;
-  double dmax      = -DBL_MAX;
-  double dsum      = 0;
-  *minIndex        = 0;
-  *maxIndex        = 0;
-  
-  assert(numOfRow <= INT16_MAX);
-  
-  for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) {
-      (*numOfNull) += 1;
-      continue;
-    }
-    
-    double dv = 0;
-    dv = GET_DOUBLE_VAL(&(data[i]));
-    dsum += dv;
-    if (dmin > dv) {
-      dmin = dv;
-      *minIndex = i;
-    }
-    
-    if (dmax < dv) {
-      dmax = dv;
-      *maxIndex = i;
-    }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
-  }
-  
-  double csum = 0;
-  csum = GET_DOUBLE_VAL(sum);
-  csum += dsum;
-
-
-#ifdef _TD_ARM_32_
-  SET_DOUBLE_VAL_ALIGN(sum, &csum);
-    SET_DOUBLE_VAL_ALIGN(max, &dmax);
-    SET_DOUBLE_VAL_ALIGN(min, &dmin);
-#else
-  *sum = csum;
-  *max = dmax;
-  *min = dmin;
-#endif
-}
-
-void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max,
-                   int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) {
-  int64_t *primaryKey = (int64_t *)priData;
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    for (int32_t i = 0; i < numOfRow; ++i) {
-      if (isNull(data + i * size, type)) {
-        (*numOfNull) += 1;
-        continue;
-      }
-    }
-  } else {
-    if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
-      getStatics_i8(primaryKey, type, (int8_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_SMALLINT) {
-      getStatics_i16(primaryKey, (int16_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_INT) {
-      getStatics_i32(primaryKey, (int32_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) {
-      getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_DOUBLE) {
-      getStatics_d(primaryKey, (double *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull);
-    } else if (type == TSDB_DATA_TYPE_FLOAT) {
-      getStatics_f(primaryKey, (float *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull);
-    }
-  }
-}
 
 /**
  * param[1]: start time
@@ -4134,15 +3810,15 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 /**
- * param[1]: default value/previous value of specified timestamp
- * param[2]: next value of specified timestamp
- * param[3]: denotes if the result is a precious result or interpolation results
  *
  * @param pCtx
  */
 static void interp_function(SQLFunctionCtx *pCtx) {
   // at this point, the value is existed, return directly
-  if (pCtx->param[3].i64Key == 1) {
+  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+  SInterpInfoDetail* pInfo = pResInfo->interResultBuf;
+
+  if (pCtx->size == 1) {
     char *pData = GET_INPUT_CHAR(pCtx);
     assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType);
   } else {
@@ -4150,84 +3826,74 @@ static void interp_function(SQLFunctionCtx *pCtx) {
      * use interpolation to generate the result.
      * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql
      */
-    assert(pCtx->param[3].i64Key == 2);
+    assert(pCtx->size == 2);
-    
+    if (pInfo->type == TSDB_FILL_NONE) {  // set no output result
-    SInterpInfo        interpInfo = *(SInterpInfo *)pCtx->aOutputBuf;
+      return;
-    SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail;
-    
-    /* set no output result */
-    if (pInfoDetail->type == TSDB_INTERPO_NONE) {
-      pCtx->param[3].i64Key = 0;
-    } else if (pInfoDetail->primaryCol == 1) {
-      *(TSKEY *)pCtx->aOutputBuf = pInfoDetail->ts;
-    } else {
-      if (pInfoDetail->type == TSDB_INTERPO_NULL) {
-        setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
-      } else if (pInfoDetail->type == TSDB_INTERPO_SET_VALUE) {
-        tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType);
-      } else if (pInfoDetail->type == TSDB_INTERPO_PREV) {
-        char *data = pCtx->param[1].pz;
-        char *pVal = data + TSDB_KEYSIZE;
-        
-        if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) {
-          float v = GET_DOUBLE_VAL(pVal);
-          assignVal(pCtx->aOutputBuf, (const char*) &v, pCtx->outputBytes, pCtx->outputType);
-        } else {
-          assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType);
     }
     
-      } else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) {
+    if (pInfo->primaryCol == 1) {
-        char *data1 = pCtx->param[1].pz;
+      *(TSKEY *) pCtx->aOutputBuf = pInfo->ts;
-        char *data2 = pCtx->param[2].pz;
+    } else {
+      if (pInfo->type == TSDB_FILL_NULL) {
+        if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
+        } else {
+          setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+        }
   
-        char *pVal1 = data1 + TSDB_KEYSIZE;
+        SET_VAL(pCtx, pCtx->size, 1);
-        char *pVal2 = data2 + TSDB_KEYSIZE;
+      } else if (pInfo->type == TSDB_FILL_SET_VALUE) {
+        tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true);
+      } else if (pInfo->type == TSDB_FILL_PREV) {
+        char *data = GET_INPUT_CHAR_INDEX(pCtx, 0);
+        assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType);
   
-        SPoint point1 = {.key = *(TSKEY *)data1, .val = &pCtx->param[1].i64Key};
+        SET_VAL(pCtx, pCtx->size, 1);
-        SPoint point2 = {.key = *(TSKEY *)data2, .val = &pCtx->param[2].i64Key};
+      } else if (pInfo->type == TSDB_FILL_LINEAR) {
+        char *data1 = GET_INPUT_CHAR_INDEX(pCtx, 0);
+        char *data2 = GET_INPUT_CHAR_INDEX(pCtx, 1);
       
-        SPoint point = {.key = pInfoDetail->ts, .val = pCtx->aOutputBuf};
+        TSKEY key1 = pCtx->ptsList[0];
+        TSKEY key2 = pCtx->ptsList[1];
+      
+        SPoint point1 = {.key = key1, .val = data1};
+        SPoint point2 = {.key = key2, .val = data2};
+      
+        SPoint point = {.key = pInfo->ts, .val = pCtx->aOutputBuf};
       
         int32_t srcType = pCtx->inputType;
         if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) ||
             srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) {
-          point1.val = pVal1;
+          point1.val = data1;
+          point2.val = data2;
         
-          point2.val = pVal2;
+          if (isNull(data1, srcType) || isNull(data2, srcType)) {
-          
-          if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) {
             setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
           } else {
             taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
           }
         } else if (srcType == TSDB_DATA_TYPE_FLOAT) {
-          float v1 = GET_DOUBLE_VAL(pVal1);
+          point1.val = data1;
-          float v2 = GET_DOUBLE_VAL(pVal2);
+          point2.val = data2;
         
-          point1.val = &v1;
+          if (isNull(data1, srcType) || isNull(data2, srcType)) {
-          point2.val = &v2;
-          
-          if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) {
             setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
           } else {
             taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
           }
         
+        } else {
+          if (srcType == TSDB_DATA_TYPE_BINARY || srcType == TSDB_DATA_TYPE_NCHAR) {
+            setVardataNull(pCtx->aOutputBuf, pCtx->inputType);
           } else {
             setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
           }
         }
       }
-    
-    free(interpInfo.pInterpDetail);
     }
     
-  pCtx->size = pCtx->param[3].i64Key;
+  }
   
-  tVariantDestroy(&pCtx->param[1]);
-  tVariantDestroy(&pCtx->param[2]);
-  
-  // data in the check operation are all null, not output
   SET_VAL(pCtx, pCtx->size, 1);
 }
 
@@ -5191,7 +4857,7 @@ SQLAggFuncElem aAggs[] = {{
                               doFinalizer,
                               noop1,
                               copy_function,
-                              no_data_info,
+                              data_req_load_info,
                           },
                           {
                               // 28
@@ -5285,10 +4951,10 @@ SQLAggFuncElem aAggs[] = {{
                           },
                           {
                               // 34
-                              "tid_tag",   // return table id and the corresponding tags for join match
+                              "tid_tag",   // return table id and the corresponding tags for join match and subscribe
                               TSDB_FUNC_TID_TAG,
                               TSDB_FUNC_TID_TAG,
-                              TSDB_FUNCSTATE_MO,
+                              TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE,
                               function_setup,
                               noop1,
                               noop2,

src/client/src/tscLocal.c

@@ -26,7 +26,7 @@
 #include "tschemautil.h"
 #include "tname.h"
 
-static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, size_t valueLength);
+static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
 
 static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) {
   char buf[512] = {0};
@@ -56,7 +56,7 @@ static int32_t getToStringLength(const char *pData, int32_t length, int32_t type
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_BIGINT:
-      len = sprintf(buf, "%" PRId64 "", *(int64_t *)pData);
+      len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
       break;
     case TSDB_DATA_TYPE_BOOL:
       len = MAX_BOOL_TYPE_LENGTH;
@@ -122,7 +122,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
   int32_t numOfRows = tscGetNumOfColumns(pMeta);
   int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta);
 
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     numOfRows = numOfRows + tscGetNumOfTags(pMeta);
   }
 
@@ -132,7 +132,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
   for (int32_t i = 0; i < numOfRows; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
 
     char *type = tDataTypeDesc[pSchema[i].type].aName;
 
@@ -156,11 +156,12 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
     if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) {
       char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
-      STR_WITH_SIZE_TO_VARSTR(output, "TAG", 3);
+      const char *src = "TAG";
+      STR_WITH_SIZE_TO_VARSTR(output, src, strlen(src));
     }
   }
 
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return 0;
   }
 
@@ -170,7 +171,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // field name
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
     char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN);
+    STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN - 1);
 
     // type name
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
@@ -191,7 +192,8 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // tag value
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
     char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
-    STR_WITH_SIZE_TO_VARSTR(target, "TAG", 3);
+    const char *src = "TAG";
+    STR_WITH_SIZE_TO_VARSTR(target, src, strlen(src));
 
     pTagValue += pSchema[i].bytes;
   }
@@ -209,18 +211,18 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   pQueryInfo->order.order = TSDB_ORDER_ASC;
 
-  TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE};
+  TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
-  strncpy(f.name, "Field", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Field", sizeof(f.name));
   
   SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
-      TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE, TSDB_COL_NAME_LEN, false);
+      (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
   
-  rowLen += (TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE);
+  rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
 
   f.bytes = typeColLength;
   f.type = TSDB_DATA_TYPE_BINARY;
-  strncpy(f.name, "Type", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Type", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength,
@@ -230,7 +232,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
 
   f.bytes = sizeof(int32_t);
   f.type = TSDB_DATA_TYPE_INT;
-  strncpy(f.name, "Length", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Length", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
@@ -240,7 +242,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
 
   f.bytes = noteColLength;
   f.type = TSDB_DATA_TYPE_BINARY;
-  strncpy(f.name, "Note", TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, "Note", sizeof(f.name));
   
   pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength,
@@ -273,22 +275,37 @@ static void tscProcessCurrentUser(SSqlObj *pSql) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  tscSetLocalQueryResult(pSql, pSql->pTscObj->user, pExpr->aliasName, TSDB_USER_LEN);
+  pExpr->resBytes = TSDB_USER_LEN + TSDB_DATA_TYPE_BINARY;
+  pExpr->resType = TSDB_DATA_TYPE_BINARY;
+  
+  char* vx = calloc(1, pExpr->resBytes);
+  STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, TSDB_USER_LEN);
+  
+  tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
+  free(vx);
 }
 
 static void tscProcessCurrentDB(SSqlObj *pSql) {
-  char db[TSDB_DB_NAME_LEN + 1] = {0};
+  char db[TSDB_DB_NAME_LEN] = {0};
   extractDBName(pSql->pTscObj->db, db);
   
-  // no use db is invoked before.
-  if (strlen(db) == 0) {
-    setNull(db, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN);
-  }
-  
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  tscSetLocalQueryResult(pSql, db, pExpr->aliasName, TSDB_DB_NAME_LEN);
+  pExpr->resType = TSDB_DATA_TYPE_BINARY;
+  
+  size_t t = strlen(db);
+  pExpr->resBytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE;
+  
+  char* vx = calloc(1, pExpr->resBytes);
+  if (t == 0) {
+    setVardataNull(vx, TSDB_DATA_TYPE_BINARY);
+  } else {
+    STR_WITH_SIZE_TO_VARSTR(vx, db, t);
+  }
+  
+  tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
+  free(vx);
 }
 
 static void tscProcessServerVer(SSqlObj *pSql) {
@@ -296,26 +313,44 @@ static void tscProcessServerVer(SSqlObj *pSql) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  tscSetLocalQueryResult(pSql, v, pExpr->aliasName, tListLen(pSql->pTscObj->sversion));
+  pExpr->resType = TSDB_DATA_TYPE_BINARY;
+  
+  size_t t = strlen(v);
+  pExpr->resBytes = t + VARSTR_HEADER_SIZE;
+  
+  char* vx = calloc(1, pExpr->resBytes);
+  STR_WITH_SIZE_TO_VARSTR(vx, v, t);
+  tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
+  
+  tfree(vx);
 }
 
 static void tscProcessClientVer(SSqlObj *pSql) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  tscSetLocalQueryResult(pSql, version, pExpr->aliasName, strlen(version));
+  pExpr->resType = TSDB_DATA_TYPE_BINARY;
+  
+  size_t t = strlen(version);
+  pExpr->resBytes = t + VARSTR_HEADER_SIZE;
+  
+  char* v = calloc(1, pExpr->resBytes);
+  STR_WITH_SIZE_TO_VARSTR(v, version, t);
+  tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
+  
+  tfree(v);
 }
 
 static void tscProcessServStatus(SSqlObj *pSql) {
   STscObj* pObj = pSql->pTscObj;
   
   if (pObj->pHb != NULL) {
-    if (pObj->pHb->res.code == TSDB_CODE_NETWORK_UNAVAIL) {
+    if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-      pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL;
+      pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
       return;
     }
   } else {
-    if (pSql->res.code == TSDB_CODE_NETWORK_UNAVAIL) {
+    if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
       return;
     }
   }
@@ -323,10 +358,11 @@ static void tscProcessServStatus(SSqlObj *pSql) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
-  tscSetLocalQueryResult(pSql, "1", pExpr->aliasName, 2);
+  int32_t val = 1;
+  tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t));
 }
 
-void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, size_t valueLength) {
+void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
@@ -336,8 +372,10 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa
   pQueryInfo->order.order = TSDB_ORDER_ASC;
   
   tscFieldInfoClear(&pQueryInfo->fieldsInfo);
+  pQueryInfo->fieldsInfo.pFields = taosArrayInit(1, sizeof(TAOS_FIELD));
+  pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(1, sizeof(SFieldSupInfo));
   
-  TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_BINARY, columnName, valueLength);
+  TAOS_FIELD f = tscCreateField(type, columnName, valueLength);
   tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   
   tscInitResObjForLocalQuery(pSql, 1, valueLength);
@@ -346,7 +384,7 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa
   SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0);
   pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprList, 0);
   
-  strncpy(pRes->data, val, pField->bytes);
+  memcpy(pRes->data, val, pField->bytes);
 }
 
 int tscProcessLocalCmd(SSqlObj *pSql) {
@@ -376,20 +414,17 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
   } else if (pCmd->command == TSDB_SQL_SERV_STATUS) {
     tscProcessServStatus(pSql);
   } else {
-    pSql->res.code = TSDB_CODE_INVALID_SQL;
+    pSql->res.code = TSDB_CODE_TSC_INVALID_SQL;
     tscError("%p not support command:%d", pSql, pCmd->command);
   }
 
   // keep the code in local variable in order to avoid invalid read in case of async query
   int32_t code = pSql->res.code;
-
+  if (code == TSDB_CODE_SUCCESS) {
-  if (pSql->fp != NULL) {  // callback function
+    (*pSql->fp)(pSql->param, pSql, code);
-    if (code == 0) {
-      (*pSql->fp)(pSql->param, pSql, 0);
   } else {
     tscQueueAsyncRes(pSql);
   }
-  }
 
   return code;
 }

src/client/src/tscParseInsert.c

@@ -42,35 +42,42 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 
 static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
-  int32_t numType = isValidNumber(pToken);
+  if (pToken->n == 0) {
-  if (TK_ILLEGAL == numType) {
+    return TK_ILLEGAL;
-    return numType;
   }
   
   int32_t radix = 10;
-  if (numType == TK_HEX) {
+  
-    radix = 16;
+  int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin
-  } else if (numType == TK_OCT) {
+  if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
-    radix = 8;
+    radix = radixList[pToken->type - TK_HEX];
-  } else if (numType == TK_BIN) {
-    radix = 2;
   }
 
   errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
   
-  return numType;
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  }
+
+  return pToken->type;
 }
 
 static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
-  int32_t numType = isValidNumber(pToken);
+  if (pToken->n == 0) {
-  if (TK_ILLEGAL == numType) {
+    return TK_ILLEGAL;
-    return numType;
   }
   
   errno = 0;
   *value = strtod(pToken->z, endPtr);
-  return numType;
+  
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  } else {
+    return pToken->type;
+  }
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -127,7 +134,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
     }
 
     if (getTimestampInUsFromStr(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     if (timePrec == TSDB_TIME_PRECISION_MILLI) {
@@ -305,8 +312,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
-        varDataSetLen(payload, sizeof(int8_t));
+        setVardataNull(payload, TSDB_DATA_TYPE_BINARY);
-        *(uint8_t*) varDataVal(payload) = TSDB_DATA_BINARY_NULL;
       } else { // too long values will return invalid sql, not be truncated automatically
         if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor
           return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z);
@@ -319,8 +325,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        varDataSetLen(payload, sizeof(int32_t));
+        setVardataNull(payload, TSDB_DATA_TYPE_NCHAR);
-        *(uint32_t*) varDataVal(payload) = TSDB_DATA_NCHAR_NULL;
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
         size_t output = 0;
@@ -418,15 +423,15 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
       }
 
       strcpy(error, "client out of memory");
-      *code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      *code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       return -1;
     }
 
-    if (((sToken.type != TK_NOW) && (sToken.type != TK_INTEGER) && (sToken.type != TK_STRING) &&
+    int16_t type = sToken.type;
-         (sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) ||
+    if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
-        (sToken.n == 0) || (sToken.type == TK_RP)) {
+         type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) {
       tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z);
-      *code = TSDB_CODE_INVALID_SQL;
+      *code = TSDB_CODE_TSC_INVALID_SQL;
       return -1;
     }
 
@@ -458,13 +463,13 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
     bool    isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
     int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, error, str, isPrimaryKey, timePrec);
     if (ret != TSDB_CODE_SUCCESS) {
-      *code = TSDB_CODE_INVALID_SQL;
+      *code = TSDB_CODE_TSC_INVALID_SQL;
       return -1;  // NOTE: here 0 mean error!
     }
 
     if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) {
       tscInvalidSQLErrMsg(error, "client time/server time can not be mixed up", sToken.z);
-      *code = TSDB_CODE_INVALID_TIME_STAMP;
+      *code = TSDB_CODE_TSC_INVALID_TIME_STAMP;
       return -1;
     }
   }
@@ -521,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe
 
   if (spd->hasVal[0] == false) {
     strcpy(error, "primary timestamp column can not be null");
-    *code = TSDB_CODE_INVALID_SQL;
+    *code = TSDB_CODE_TSC_INVALID_SQL;
     return -1;
   }
 
@@ -555,7 +560,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe
     *str += index;
     if (sToken.n == 0 || sToken.type != TK_RP) {
       tscInvalidSQLErrMsg(error, ") expected", *str);
-      *code = TSDB_CODE_INVALID_SQL;
+      *code = TSDB_CODE_TSC_INVALID_SQL;
       return -1;
     }
 
@@ -564,7 +569,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe
 
   if (numOfRows <= 0) {
     strcpy(error, "no any data points");
-    *code = TSDB_CODE_INVALID_SQL;
+    *code = TSDB_CODE_TSC_INVALID_SQL;
     return -1;
   } else {
     return numOfRows;
@@ -606,7 +611,7 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
       // do nothing, if allocate more memory failed
       pDataBlock->nAllocSize = nAllocSizeOld;
       *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize;
-      return TSDB_CODE_CLI_OUT_OF_MEMORY;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
   }
 
@@ -682,13 +687,13 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   int32_t maxNumOfRows;
   ret = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows);
   if (TSDB_CODE_SUCCESS != ret) {
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  int32_t code = TSDB_CODE_INVALID_SQL;
+  int32_t code = TSDB_CODE_TSC_INVALID_SQL;
   char *  tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
   if (NULL == tmpTokenBuf) {
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
   int32_t numOfRows = tsParseValues(str, dataBuf, pTableMeta, maxNumOfRows, spd, pCmd->payload, &code, tmpTokenBuf);
@@ -767,7 +772,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
   }
 
   if (numOfColList == 0 && cstart != NULL) {
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
   
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX);
@@ -788,15 +793,15 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     }
 
     STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
-    tscSetTableId(pSTableMeterMetaInfo, &sToken, pSql);
+    tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql);
 
-    strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN);
+    tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
     code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
 
-    if (!UTIL_TABLE_IS_SUPERTABLE(pSTableMeterMetaInfo)) {
+    if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMeterMetaInfo)) {
       return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z);
     }
 
@@ -829,9 +834,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
         sql += index;
 
         if (TK_STRING == sToken.type) {
-          sToken.n = strdequote(sToken.z);
+          strdequote(sToken.z);
-          strtrim(sToken.z);
+          sToken.n = strtrim(sToken.z);
-          sToken.n = (uint32_t)strlen(sToken.z);
         }
 
         if (sToken.type == TK_RP) {
@@ -920,25 +924,37 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
       for (int32_t i = 0; i < spd.numOfCols; ++i) {
         if (!spd.hasVal[i]) {  // current tag column do not have any value to insert, set it to null
+          if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+            setVardataNull(ptr, pTagSchema[i].type);
+          } else {
             setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes);
           }
+        }
 
         ptr += pTagSchema[i].bytes;
       }
     }
 
+    // 3. calculate the actual data size of STagData
+    pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen);
+    for (int32_t t = 0; t < numOfTags; ++t) {
+      pTag->dataLen += pTagSchema[t].bytes;
+      pCmd->payloadLen += pTagSchema[t].bytes;
+    }
+    pTag->dataLen = htonl(pTag->dataLen);
+
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }
 
-    int32_t ret = tscSetTableId(pTableMetaInfo, &tableToken, pSql);
+    int32_t ret = tscSetTableFullName(pTableMetaInfo, &tableToken, pSql);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
 
     createTable = true;
     code = tscGetMeterMetaEx(pSql, pTableMetaInfo, true);
-    if (TSDB_CODE_ACTION_IN_PROGRESS == code) {
+    if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
       return code;
     }
     
@@ -948,10 +964,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     } else {
       sql = sToken.z;
     }
-    code = tscGetTableMeta(pSql, pTableMetaInfo);
+    code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false);
     
     if (pCmd->curSql == NULL) {
-      assert(code == TSDB_CODE_ACTION_IN_PROGRESS);
+      assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS);
     }
   }
 
@@ -965,7 +981,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
   }
 
   if (*sqlstr == NULL) {
-    code = TSDB_CODE_INVALID_SQL;
+    code = TSDB_CODE_TSC_INVALID_SQL;
   }
   
   return code;
@@ -973,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
 
 int validateTableName(char *tblName, int len) {
   char buf[TSDB_TABLE_ID_LEN] = {0};
-  strncpy(buf, tblName, len);
+  tstrncpy(buf, tblName, sizeof(buf));
 
   SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
   tSQLGetToken(buf, &token.type);
@@ -1016,7 +1032,9 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
     pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   }
 
-  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
+  // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536
+  // but TSDB_PAYLOAD_SIZE is 65380
+  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
     return code;
   }
 
@@ -1028,7 +1046,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
 
     pSql->cmd.pDataBlocks = tscCreateBlockArrayList();
     if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) {
-      code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       goto _error_clean;
     }
   } else {
@@ -1057,7 +1075,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
        * Otherwise, create the first submit block and submit to virtual node.
        */
       if (totalNum == 0) {
-        code = TSDB_CODE_INVALID_SQL;
+        code = TSDB_CODE_TSC_INVALID_SQL;
         goto _error_clean;
       } else {
         break;
@@ -1072,7 +1090,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
       goto _error_clean;
     }
 
-    if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
+    if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
       goto _error_clean;
     }
 
@@ -1085,7 +1103,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
        * And during the getMeterMetaCallback function, the sql string will be parsed from the
        * interrupted position.
        */
-      if (TSDB_CODE_ACTION_IN_PROGRESS == code) {
+      if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
         tscTrace("%p waiting for get table meta during insert, then resume from offset: %" PRId64 " , %s", pSql,
             pos, pCmd->curSql);
         return code;
@@ -1097,7 +1115,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
       goto _error_clean;       // TODO: should _clean or _error_clean to async flow ????
     }
 
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL);
       goto _error_clean;
     }
@@ -1190,9 +1208,8 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
         str += index;
 
         if (TK_STRING == sToken.type) {
-          sToken.n = strdequote(sToken.z);
+          strdequote(sToken.z);
-          strtrim(sToken.z);
+          sToken.n = strtrim(sToken.z);
-          sToken.n = (uint32_t)strlen(sToken.z);
         }
 
         if (sToken.type == TK_RP) {
@@ -1280,7 +1297,7 @@ _clean:
 
 int tsParseInsertSql(SSqlObj *pSql) {
   if (!pSql->pTscObj->writeAuth) {
-    return TSDB_CODE_NO_RIGHTS;
+    return TSDB_CODE_TSC_NO_WRITE_AUTH;
   }
 
   int32_t  index = 0;
@@ -1296,8 +1313,8 @@ int tsParseInsertSql(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = NULL;
   tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
 
-  uint16_t type = (sToken.type == TK_INSERT)? TSDB_QUERY_TYPE_INSERT:TSDB_QUERY_TYPE_IMPORT;
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
-  TSDB_QUERY_SET_TYPE(pQueryInfo->type, type);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, pCmd->insertType);
 
   sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
   if (sToken.type != TK_INTO) {
@@ -1325,7 +1342,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
      * Set the fp before parse the sql string, in case of getTableMeta failed, in which
      * the error handle callback function can rightfully restore the user-defined callback function (fp).
      */
-    if (initialParse) {
+    if (initialParse && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
       pSql->fetchFp = pSql->fp;
       pSql->fp = (void(*)())tscHandleMultivnodeInsert;
     }
@@ -1337,9 +1354,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
       return ret;
     }
     
-    SSqlInfo SQLInfo = {0};
+    SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr);
-    tSQLParse(&SQLInfo, pSql->sqlstr);
-
     ret = tscToSQLCmd(pSql, &SQLInfo);
     SQLInfoDestroy(&SQLInfo);
   }
@@ -1347,7 +1362,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
   /*
    * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function,
    * so do NOT use pRes->code to determine if the getTableMeta/getMetricMeta function
-   * invokes new threads to get data from mgmt node or simply retrieves data from cache.
+   * invokes new threads to get data from mnode or simply retrieves data from cache.
    *
    * do NOT assign return code to pRes->code for the same reason since it may be released by another thread
    * pRes->code = ret;
@@ -1497,7 +1512,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
     }
     pCmd->count = 1;
 
-    strncpy(path, pDataBlock->filename, PATH_MAX);
+    tstrncpy(path, pDataBlock->filename, sizeof(path));
 
     FILE *fp = fopen(path, "r");
     if (fp == NULL) {
@@ -1505,7 +1520,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) {
       continue;
     }
 
-    strncpy(pTableMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN);
+    tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
     memset(pDataBlock->pData, 0, pDataBlock->nAllocSize);
 
     int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo);

src/client/src/tscPrepare.c

@@ -12,6 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "os.h"
 
 #include "taos.h"
 #include "tsclient.h"
@@ -20,9 +21,9 @@
 #include "taosmsg.h"
 #include "tstrbuild.h"
 #include "tscLog.h"
+#include "tscSubquery.h"
 
 int tsParseInsertSql(SSqlObj *pSql);
-int taos_query_imp(STscObj* pObj, SSqlObj* pSql);
 
 ////////////////////////////////////////////////////////////////////////////////
 // functions for normal statement preparation
@@ -60,7 +61,7 @@ static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_
     size *= 2;
     void* tmp = realloc(stmt->parts, sizeof(SNormalStmtPart) * size);
     if (tmp == NULL) {
-      return TSDB_CODE_CLI_OUT_OF_MEMORY;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     stmt->sizeParts = size;
     stmt->parts = (SNormalStmtPart*)tmp;
@@ -131,7 +132,7 @@ static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
       case TSDB_DATA_TYPE_NCHAR:
         var->pz = (char*)malloc((*tb->length) + 1);
         if (var->pz == NULL) {
-          return TSDB_CODE_CLI_OUT_OF_MEMORY;
+          return TSDB_CODE_TSC_OUT_OF_MEMORY;
         }
         memcpy(var->pz, tb->buffer, (*tb->length));
         var->pz[*tb->length] = 0;
@@ -140,7 +141,7 @@ static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
 
       default:
         tscTrace("param %d: type mismatch or invalid", i);
-        return TSDB_CODE_INVALID_VALUE;
+        return TSDB_CODE_TSC_INVALID_VALUE;
     }
   }
   
@@ -185,7 +186,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
   if (normal->numParams > 0) {
     normal->params = calloc(normal->numParams, sizeof(tVariant));
     if (normal->params == NULL) {
-      return TSDB_CODE_CLI_OUT_OF_MEMORY;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
   }
 
@@ -194,7 +195,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
 
 static char* normalStmtBuildSql(STscStmt* stmt) {
   SNormalStmt* normal = &stmt->normal;
-  SStringBuilder sb = {0};
+  SStringBuilder sb; memset(&sb, 0, sizeof(sb));
 
   if (taosStringBuilderSetJmp(&sb) != 0) {
     taosStringBuilderDestroy(&sb);
@@ -262,12 +263,16 @@ static char* normalStmtBuildSql(STscStmt* stmt) {
 
 static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
   if (bind->is_null != NULL && *(bind->is_null)) {
-    setNull(data, param->type, param->bytes);
+    if (param->type == TSDB_DATA_TYPE_BINARY || param->type == TSDB_DATA_TYPE_NCHAR) {
+      setVardataNull(data + param->offset, param->type);
+    } else {
+      setNull(data + param->offset, param->type, param->bytes);
+    }
     return TSDB_CODE_SUCCESS;
   }
 
   if (bind->buffer_type != param->type) {
-    return TSDB_CODE_INVALID_VALUE;
+    return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
   short size = 0;
@@ -294,20 +299,23 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
 
     case TSDB_DATA_TYPE_BINARY:
       if ((*bind->length) > param->bytes) {
-        return TSDB_CODE_INVALID_VALUE;
+        return TSDB_CODE_TSC_INVALID_VALUE;
       }
       size = (short)*bind->length;
-      break;
+      STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size);
-    
-    case TSDB_DATA_TYPE_NCHAR:
-      if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes, NULL)) {
-        return TSDB_CODE_INVALID_VALUE;
-      }
       return TSDB_CODE_SUCCESS;
     
+    case TSDB_DATA_TYPE_NCHAR: {
+      size_t output = 0;
+      if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) {
+        return TSDB_CODE_TSC_INVALID_VALUE;
+      }      
+      varDataSetLen(data + param->offset, output);
+      return TSDB_CODE_SUCCESS;
+    }
     default:
       assert(false);
-      return TSDB_CODE_INVALID_VALUE;
+      return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
   memcpy(data + param->offset, bind->buffer, size);
@@ -335,7 +343,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
         const double factor = 1.5;
         void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor));
         if (tmp == NULL) {
-          return TSDB_CODE_CLI_OUT_OF_MEMORY;
+          return TSDB_CODE_TSC_OUT_OF_MEMORY;
         }
         pBlock->pData = (char*)tmp;
         pBlock->nAllocSize = (uint32_t)(totalDataSize * factor);
@@ -383,14 +391,6 @@ static int insertStmtAddBatch(STscStmt* stmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int insertStmtPrepare(STscStmt* stmt) {
-  SSqlObj *pSql = stmt->pSql;
-  pSql->cmd.numOfParams = 0;
-  pSql->cmd.batchSize = 0;
-
-  return tsParseInsertSql(pSql);
-}
-
 static int insertStmtReset(STscStmt* pStmt) {
   SSqlCmd* pCmd = &pStmt->pSql->cmd;
   if (pCmd->batchSize > 2) {
@@ -415,7 +415,7 @@ static int insertStmtReset(STscStmt* pStmt) {
 static int insertStmtExecute(STscStmt* stmt) {
   SSqlCmd* pCmd = &stmt->pSql->cmd;
   if (pCmd->batchSize == 0) {
-    return TSDB_CODE_INVALID_VALUE;
+    return TSDB_CODE_TSC_INVALID_VALUE;
   }
   if ((pCmd->batchSize % 2) == 1) {
     ++pCmd->batchSize;
@@ -447,18 +447,20 @@ static int insertStmtExecute(STscStmt* stmt) {
   SSqlRes *pRes = &pSql->res;
   pRes->numOfRows = 0;
   pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
   
   pRes->qhandle = 0;
 
+  pSql->cmd.insertType = 0;
+  pSql->fetchFp    = waitForQueryRsp;
+  pSql->fp         = (void(*)())tscHandleMultivnodeInsert;
+
   tscDoQuery(pSql);
 
-  // tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+  // wait for the callback function to post the semaphore
-  if (pRes->code != TSDB_CODE_SUCCESS) {
+  tsem_wait(&pSql->rspSem);
-    tscPartiallyFreeSqlObj(pSql);
+  return pSql->res.code;
-  }
 
-  return pRes->code;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -467,22 +469,23 @@ static int insertStmtExecute(STscStmt* stmt) {
 TAOS_STMT* taos_stmt_init(TAOS* taos) {
   STscObj* pObj = (STscObj*)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
     tscError("connection disconnected");
     return NULL;
   }
 
   STscStmt* pStmt = calloc(1, sizeof(STscStmt));
   if (pStmt == NULL) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("failed to allocate memory for statement");
     return NULL;
   }
+  pStmt->taos = pObj;
 
   SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
     free(pStmt);
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("failed to allocate memory for statement");
     return NULL;
   }
@@ -490,6 +493,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) {
   tsem_init(&pSql->rspSem, 0, 0);
   pSql->signature     = pSql;
   pSql->pTscObj       = pObj;
+  pSql->maxRetry      = TSDB_MAX_REPLICA_NUM;
 
   pStmt->pSql = pSql;
   return pStmt;
@@ -497,22 +501,55 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) {
 
 int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   STscStmt* pStmt = (STscStmt*)stmt;
-  if (length == 0) {
-    length = strlen(sql);
-  }
-  char* sqlstr = (char*)malloc(length + 1);
-  if (sqlstr == NULL) {
-    tscError("failed to malloc sql string buffer");
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
-  }
-  memcpy(sqlstr, sql, length);
-  sqlstr[length] = 0;
-  strtolower(sqlstr, sqlstr);
 
-  pStmt->pSql->sqlstr = sqlstr;
+  if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) {
-  if (tscIsInsertData(sqlstr)) {
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
+    return TSDB_CODE_TSC_DISCONNECTED;
+  }
+
+  SSqlObj* pSql = pStmt->pSql;
+  size_t   sqlLen = strlen(sql);
+  
+  //doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
+  SSqlCmd *pCmd    = &pSql->cmd;
+  SSqlRes *pRes    = &pSql->res;
+  pSql->param      = (void*) pSql;
+  pSql->fp         = waitForQueryRsp;
+  pSql->cmd.insertType = TSDB_QUERY_TYPE_STMT_INSERT;
+  
+  if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
+    tscError("%p failed to malloc payload buffer", pSql);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  
+  pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
+  
+  if (pSql->sqlstr == NULL) {
+    tscError("%p failed to malloc sql string buffer", pSql);
+    free(pCmd->payload);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  
+  pRes->qhandle = 0;
+  pRes->numOfRows = 1;
+  
+  strtolower(pSql->sqlstr, sql);
+  tscDump("%p SQL: %s", pSql, pSql->sqlstr);
+
+  if (tscIsInsertData(pSql->sqlstr)) {  
     pStmt->isInsert = true;
-    return insertStmtPrepare(pStmt);
+    
+    pSql->cmd.numOfParams = 0;
+    pSql->cmd.batchSize   = 0;
+    
+    int32_t code = tsParseSql(pSql, true);
+    if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+      // wait for the callback function to post the semaphore
+      tsem_wait(&pSql->rspSem);
+      return pSql->res.code;
+    }
+    
+    return code;
   }
 
   pStmt->isInsert = false;
@@ -551,7 +588,7 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) {
   if (pStmt->isInsert) {
     return insertStmtAddBatch(pStmt);
   }
-  return TSDB_CODE_OPS_NOT_SUPPORT;
+  return TSDB_CODE_COM_OPS_NOT_SUPPORT;
 }
 
 int taos_stmt_reset(TAOS_STMT* stmt) {
@@ -570,11 +607,13 @@ int taos_stmt_execute(TAOS_STMT* stmt) {
   } else {
     char* sql = normalStmtBuildSql(pStmt);
     if (sql == NULL) {
-      ret = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      ret = TSDB_CODE_TSC_OUT_OF_MEMORY;
     } else {
       tfree(pStmt->pSql->sqlstr);
       pStmt->pSql->sqlstr = sql;
-      ret = taos_query_imp(pStmt->taos, pStmt->pSql);
+      SSqlObj* pSql = taos_query((TAOS*)pStmt->taos, pStmt->pSql->sqlstr);
+      ret = taos_errno(pSql);
+      taos_free_result(pSql);
     }
   }
   return ret;

src/client/src/tscProfile.c

@@ -19,12 +19,11 @@
 #include "ttime.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "taosmsg.h"
 
 void  tscSaveSlowQueryFp(void *handle, void *tmrId);
 void *tscSlowQueryConn = NULL;
 bool  tscSlowQueryConnInitialized = false;
-TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
-                     void *param, void **taos);
 
 void tscInitConnCb(void *param, TAOS_RES *result, int code) {
   char *sql = param;
@@ -90,22 +89,32 @@ void tscSaveSlowQueryFp(void *handle, void *tmrId) {
 }
 
 void tscSaveSlowQuery(SSqlObj *pSql) {
-  const static int64_t SLOW_QUERY_INTERVAL = 3000000L;
+  const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable
-  if (pSql->res.useconds < SLOW_QUERY_INTERVAL) return;
+  size_t size = 200;  // other part of sql string, expect the main sql str
+  
+  if (pSql->res.useconds < SLOW_QUERY_INTERVAL) {
+    return;
+  }
 
   tscTrace("%p query time:%" PRId64 " sql:%s", pSql, pSql->res.useconds, pSql->sqlstr);
+  int32_t sqlSize = TSDB_SLOW_QUERY_SQL_LEN + size;
   
-  char *sql = malloc(200);
+  char *sql = malloc(sqlSize);
-  int   len = snprintf(sql, 200, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName,
+  if (sql == NULL) {
+    tscError("%p failed to allocate memory to sent slow query to dnode", pSql);
+    return;
+  }
+  
+  int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName,
           pSql->pTscObj->user, pSql->stime, pSql->res.useconds);
-  int sqlLen = snprintf(sql + len, TSDB_SHOW_SQL_LEN, "%s", pSql->sqlstr);
+  int sqlLen = snprintf(sql + len, TSDB_SLOW_QUERY_SQL_LEN, "%s", pSql->sqlstr);
-  if (sqlLen > TSDB_SHOW_SQL_LEN - 1) {
+  if (sqlLen > TSDB_SLOW_QUERY_SQL_LEN - 1) {
-    sqlLen = len + TSDB_SHOW_SQL_LEN - 1;
+    sqlLen = len + TSDB_SLOW_QUERY_SQL_LEN - 1;
   } else {
     sqlLen += len;
   }
-  strcpy(sql + sqlLen, "')");
   
+  strcpy(sql + sqlLen, "')");
   taosTmrStart(tscSaveSlowQueryFp, 200, sql, tscTmr);
 }
 
@@ -200,25 +209,25 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
 
   if (pStream) {
     tscTrace("%p stream:%p is killed, streamId:%d", pStream->pSql, pStream, killId);
-  }
-
     if (pStream->callback) {
       pStream->callback(pStream->param);
     }
     taos_close_stream(pStream);
+  } else {
+    tscError("failed to kill stream, streamId:%d not exist", killId);
+  }
 }
 
-char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) {
+int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
-  char *  pMax = pMsg + TSDB_PAYLOAD_SIZE - 256;
+  SCMHeartBeatMsg *pHeartbeat = pMsg;
+  int allocedQueriesNum = pHeartbeat->numOfQueries;
+  int allocedStreamsNum = pHeartbeat->numOfStreams;
   
-  SQqueryList *pQList = (SQqueryList *)pMsg;
+  pHeartbeat->numOfQueries = 0;
-  pQList->numOfQueries = 0;
+  SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData;
-
-  SQueryDesc *pQdesc = (SQueryDesc*)(pMsg + sizeof(SQqueryList));
 
   // We extract the lock to tscBuildHeartBeatMsg function.
-  /* pthread_mutex_lock (&pObj->mutex); */
+
-  pMsg += sizeof(SQqueryList);
   SSqlObj *pSql = pObj->sqlList;
   while (pSql) {
     /*
@@ -230,49 +239,46 @@ char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) {
       continue;
     }
 
-    strncpy(pQdesc->sql, pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
+    tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql));
-    pQdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
+    pQdesc->stime = htobe64(pSql->stime);
-    pQdesc->stime = pSql->stime;
+    pQdesc->queryId = htonl(pSql->queryId);
-    pQdesc->queryId = pSql->queryId;
+    pQdesc->useconds = htobe64(pSql->res.useconds);
-    pQdesc->useconds = pSql->res.useconds;
 
-    pQList->numOfQueries++;
+    pHeartbeat->numOfQueries++;
     pQdesc++;
     pSql = pSql->next;
-    pMsg += sizeof(SQueryDesc);
+    if (pHeartbeat->numOfQueries >= allocedQueriesNum) break;
-    if (pMsg > pMax) break;
   }
 
-  SStreamList *pSList = (SStreamList *)pMsg;
+  pHeartbeat->numOfStreams = 0;
-  pSList->numOfStreams = 0;
+  SStreamDesc *pSdesc = (SStreamDesc *)pQdesc;
 
-  SStreamDesc *pSdesc = (SStreamDesc*) (pMsg + sizeof(SStreamList));
-
-  pMsg += sizeof(SStreamList);
   SSqlStream *pStream = pObj->streamList;
   while (pStream) {
-    strncpy(pSdesc->sql, pStream->pSql->sqlstr, TSDB_SHOW_SQL_LEN - 1);
+    tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
-    pSdesc->sql[TSDB_SHOW_SQL_LEN - 1] = 0;
+    pSdesc->streamId = htonl(pStream->streamId);
-    pSdesc->streamId = pStream->streamId;
+    pSdesc->num = htobe64(pStream->num);
-    pSdesc->num = pStream->num;
 
-    pSdesc->useconds = pStream->useconds;
+    pSdesc->useconds = htobe64(pStream->useconds);
-    pSdesc->stime = pStream->stime - pStream->interval;
+    pSdesc->stime = htobe64(pStream->stime - pStream->interval);
-    pSdesc->ctime = pStream->ctime;
+    pSdesc->ctime = htobe64(pStream->ctime);
 
-    pSdesc->slidingTime = pStream->slidingTime;
+    pSdesc->slidingTime = htobe64(pStream->slidingTime);
-    pSdesc->interval = pStream->interval;
+    pSdesc->interval = htobe64(pStream->interval);
 
-    pSList->numOfStreams++;
+    pHeartbeat->numOfStreams++;
     pSdesc++;
     pStream = pStream->next;
-    pMsg += sizeof(SStreamDesc);
+    if (pHeartbeat->numOfStreams >= allocedStreamsNum) break;
-    if (pMsg > pMax) break;
   }
 
-  /* pthread_mutex_unlock (&pObj->mutex); */
+  int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
+                   sizeof(SCMHeartBeatMsg);
+  pHeartbeat->connId = htonl(pObj->connId);
+  pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
+  pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
 
-  return pMsg;
+  return msgLen;
 }
 
 void tscKillConnection(STscObj *pObj) {

src/client/src/tscSchemaUtil.c

@@ -115,7 +115,7 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) {
 
     // 3. valid column names
     for (int32_t j = i + 1; j < numOfCols; ++j) {
-      if (strncasecmp(pSchema[i].name, pSchema[j].name, TSDB_COL_NAME_LEN) == 0) {
+      if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
         return false;
       }
     }
@@ -131,13 +131,6 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol)
   assert(pTableMeta != NULL);
   
   SSchema* pSchema = (SSchema*) pTableMeta->schema;
-#if 0
-  if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
-    assert (pTableMeta->pSTable != NULL);
-    pSchema = pTableMeta->pSTable->schema;
-  }
-#endif
-
   return &pSchema[startCol];
 }
 
@@ -168,6 +161,8 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   pTableMeta->sid = pTableMetaMsg->sid;
   pTableMeta->uid = pTableMetaMsg->uid;
   pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
+  pTableMeta->sversion = pTableMetaMsg->sversion;
+  pTableMeta->tversion = pTableMetaMsg->tversion;
   
   memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
   

src/client/src/tscSql.c

@@ -52,70 +52,55 @@ static bool validPassword(const char* passwd) {
   return validImpl(passwd, TSDB_PASSWORD_LEN);
 }
 
-STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port,
+SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port,
                        void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) {
   taos_init();
   
   if (!validUserName(user)) {
-    terrno = TSDB_CODE_INVALID_ACCT;
+    terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH;
     return NULL;
   }
 
   if (!validPassword(pass)) {
-    terrno = TSDB_CODE_INVALID_PASS;
+    terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH;
     return NULL;
   }
 
+  if (ip) {
+    if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL;
+    if (port) tscMgmtIpSet.port[0] = port;
+  } 
+ 
   void *pDnodeConn = NULL;
   if (tscInitRpc(user, pass, &pDnodeConn) != 0) {
-    terrno = TSDB_CODE_NETWORK_UNAVAIL;
+    terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return NULL;
   }
  
-  tscMgmtIpSet.numOfIps = 0;
-
-  if (ip && ip[0]) {
-    tscMgmtIpSet.inUse = 0;
-    tscMgmtIpSet.numOfIps = 1;
-    strcpy(tscMgmtIpSet.fqdn[0], ip);
-    tscMgmtIpSet.port[0] = port? port: tsDnodeShellPort;
-  } else {
-    if (tsFirst[0] != 0) {
-      taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
-      tscMgmtIpSet.numOfIps++;
-    }
-
-    if (tsSecond[0] != 0) {
-      taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
-      tscMgmtIpSet.numOfIps++;
-    }
-  }
-  
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
     return NULL;
   }
 
   pObj->signature = pObj;
 
-  strncpy(pObj->user, user, TSDB_USER_LEN);
+  tstrncpy(pObj->user, user, sizeof(pObj->user));
   taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
-  pObj->mgmtPort = port ? port : tsDnodeShellPort;
 
   if (db) {
     int32_t len = strlen(db);
     /* db name is too long */
-    if (len > TSDB_DB_NAME_LEN) {
+    if (len >= TSDB_DB_NAME_LEN) {
-      terrno = TSDB_CODE_INVALID_DB;
+      terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       rpcClose(pDnodeConn);
       free(pObj);
       return NULL;
     }
 
-    char tmp[TSDB_DB_NAME_LEN + 1] = {0};
+    char tmp[TSDB_DB_NAME_LEN] = {0};
-    strcpy(tmp, db);
+    tstrncpy(tmp, db, sizeof(tmp));
 
     strdequote(tmp);
     strtolower(pObj->db, tmp);
@@ -125,7 +110,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (NULL == pSql) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
     free(pObj);
     return NULL;
@@ -134,10 +119,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   pSql->pTscObj = pObj;
   pSql->signature = pSql;
   pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
-  
   tsem_init(&pSql->rspSem, 0, 0);
   
-  pObj->pSql = pSql;
   pObj->pDnodeConn = pDnodeConn;
   
   pSql->fp = fp;
@@ -148,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   pSql->cmd.command = TSDB_SQL_CONNECT;
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
     free(pSql);
     free(pObj);
@@ -157,41 +140,37 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
   tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg);
-  return pObj;
+  return pSql;
 }
 
 static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
-  STscObj *pObj = (STscObj *)param;
+  SSqlObj *pSql = (SSqlObj *) tres;
-  assert(pObj != NULL && pObj->pSql != NULL);
+  assert(pSql != NULL);
   
-  if (code < 0) {
+  sem_post(&pSql->rspSem);
-    pObj->pSql->res.code = code;
-  }
-  
-  sem_post(&pObj->pSql->rspSem);
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  tscTrace("try to create a connection to %s", ip);
+  tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db);
-
-  STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL);
-  if (pObj != NULL) {
-    SSqlObj* pSql = pObj->pSql;
-    assert(pSql != NULL);
 
+  STscObj* pObj = NULL;
+  SSqlObj *pSql = taosConnectImpl(ip, user, pass, db, port, syncConnCallback, NULL, (void**) &pObj);
+  if (pSql != NULL) {
     pSql->fp = syncConnCallback;
-    pSql->param = pObj;
+    pSql->param = pSql;
     
     tscProcessSql(pSql);
     sem_wait(&pSql->rspSem);
     
     if (pSql->res.code != TSDB_CODE_SUCCESS) {
       terrno = pSql->res.code;
+      taos_free_result(pSql);
       taos_close(pObj);
       return NULL;
     }
     
-    tscTrace("%p DB connection is opening", pObj);
+    tscTrace("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn);
+    taos_free_result(pSql);
   
     // version compare only requires the first 3 segments of the version string
     int code = taosCheckVersion(version, taos_get_server_info(pObj), 3);
@@ -209,17 +188,14 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
 
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos) {
-  STscObj* pObj = taosConnectImpl(ip, user, pass, db, port, fp, param, taos);
+  SSqlObj* pSql = taosConnectImpl(ip, user, pass, db, port, fp, param, taos);
-  if (pObj == NULL) {
+  if (pSql == NULL) {
     return NULL;
   }
   
-  SSqlObj* pSql = pObj->pSql;
-  
   pSql->res.code = tscProcessSql(pSql);
-  tscTrace("%p DB async connection is opening", pObj);
+  tscTrace("%p DB async connection is opening", taos);
-  
+  return taos;
-  return pObj;
 }
 
 void taos_close(TAOS *taos) {
@@ -236,85 +212,41 @@ void taos_close(TAOS *taos) {
   }
 }
 
-int taos_query_imp(STscObj *pObj, SSqlObj *pSql) {
+void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
-  SSqlRes *pRes = &pSql->res;
+  assert(tres != NULL);
-  SSqlCmd *pCmd = &pSql->cmd;
-  
-  pRes->numOfRows  = 1;
-  pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
-
-  pCmd->curSql = NULL;
-  if (NULL != pCmd->pTableList) {
-    taosHashCleanup(pCmd->pTableList);
-    pCmd->pTableList = NULL;
-  }
-
-  tscDump("%p pObj:%p, SQL: %s", pSql, pObj, pSql->sqlstr);
-
-  pRes->code = (uint8_t)tsParseSql(pSql, false);
-
-  /*
-   * set the qhandle to 0 before return in order to erase the qhandle value assigned in the previous successful query.
-   * If qhandle is NOT set 0, the function of taos_free_result() will send message to server by calling tscProcessSql()
-   * to free connection, which may cause segment fault, when the parse phrase is not even successfully executed.
-   */
-  pRes->qhandle = 0;
-
-  if (pRes->code == TSDB_CODE_SUCCESS) {
-    tscDoQuery(pSql);
-  }
-
-  if (pRes->code == TSDB_CODE_SUCCESS) {
-    tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pObj), pObj);
-  } else {
-    tscError("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pObj), pObj);
-  }
-
-  if (pRes->code != TSDB_CODE_SUCCESS) {
-    tscPartiallyFreeSqlObj(pSql);
-  }
-
-  return pRes->code;
-}
-
-static void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
-  assert(param != NULL);
-  SSqlObj *pSql = ((STscObj *)param)->pSql;
-  
-  // valid error code is less than 0
-  if (code < 0) {
-    pSql->res.code = code;
-  }
   
+  SSqlObj *pSql = (SSqlObj *) tres;
   sem_post(&pSql->rspSem);
 }
 
-int taos_query(TAOS *taos, const char *sqlstr) {
+TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return TSDB_CODE_DISCONNECTED;
-  }
-  
-  SSqlObj* pSql = pObj->pSql;
-  
-  size_t sqlLen = strlen(sqlstr);
-  doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
-
-  // wait for the callback function to post the semaphore
-  sem_wait(&pSql->rspSem);
-  return pSql->res.code;
-}
-
-TAOS_RES *taos_use_result(TAOS *taos) {
-  STscObj *pObj = (STscObj *)taos;
-  if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
     return NULL;
   }
   
-  return pObj->pSql;
+  int32_t sqlLen = strlen(sqlstr);
+  if (sqlLen > tsMaxSQLStringLen) {
+    tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
+    terrno = TSDB_CODE_TSC_INVALID_SQL;
+    return NULL;
+  }
+  
+  taosNotePrintTsc(sqlstr);
+  
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  if (pSql == NULL) {
+    tscError("failed to malloc sqlObj");
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return NULL;
+  }
+  
+  doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
+
+  // wait for the callback function to post the semaphore
+  tsem_wait(&pSql->rspSem);
+  return pSql;
 }
 
 int taos_result_precision(TAOS_RES *res) {
@@ -347,18 +279,18 @@ int taos_num_fields(TAOS_RES *res) {
   return num;
 }
 
-int taos_field_count(TAOS *taos) {
+int taos_field_count(TAOS_RES *tres) {
-  STscObj *pObj = (STscObj *)taos;
+  SSqlObj* pSql = (SSqlObj*) tres;
-  if (pObj == NULL || pObj->signature != pObj) return 0;
+  if (pSql == NULL || pSql->signature != pSql) return 0;
 
-  return taos_num_fields(pObj->pSql);
+  return taos_num_fields(pSql);
 }
 
-int taos_affected_rows(TAOS *taos) {
+int taos_affected_rows(TAOS_RES *tres) {
-  STscObj *pObj = (STscObj *)taos;
+  SSqlObj* pSql = (SSqlObj*) tres;
-  if (pObj == NULL || pObj->signature != pObj) return 0;
+  if (pSql == NULL || pSql->signature != pSql) return 0;
 
-  return (pObj->pSql->res.numOfRows);
+  return (pSql->res.numOfRows);
 }
 
 TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
@@ -400,9 +332,8 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
   SSqlObj *pSql = (SSqlObj *)res;
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
-  STscObj *pObj = pSql->pTscObj;
 
-  if (pRes->qhandle == 0 || pObj->pSql != pSql) {
+  if (pRes->qhandle == 0 || pSql->signature != pSql) {
     *rows = NULL;
     return 0;
   }
@@ -421,7 +352,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
 
   // secondary merge has handle this situation
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
-    pRes->numOfTotalInCurrentClause += pRes->numOfRows;
+    pRes->numOfClauseTotal += pRes->numOfRows;
   }
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
@@ -440,17 +371,13 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
 
 static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
   SSqlObj* pSql = (SSqlObj*) tres;
-  
-  if (numOfRows < 0) { // set the error code
-    pSql->res.code = -numOfRows;
-  }
   sem_post(&pSql->rspSem);
 }
 
 TAOS_ROW taos_fetch_row(TAOS_RES *res) {
   SSqlObj *pSql = (SSqlObj *)res;
   if (pSql == NULL || pSql->signature != pSql) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
   
@@ -463,15 +390,20 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   }
   
-  // current data are exhausted, fetch more data
+  // current data set are exhausted, fetch more data from node
-  if (pRes->row >= pRes->numOfRows && pRes->completed != true &&
+  if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql)) &&
       (pCmd->command == TSDB_SQL_RETRIEVE ||
        pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE ||
-       pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE ||
+       pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE ||
        pCmd->command == TSDB_SQL_FETCH ||
        pCmd->command == TSDB_SQL_SHOW ||
        pCmd->command == TSDB_SQL_SELECT ||
-       pCmd->command == TSDB_SQL_DESCRIBE_TABLE)) {
+       pCmd->command == TSDB_SQL_DESCRIBE_TABLE ||
+       pCmd->command == TSDB_SQL_SERV_STATUS ||
+       pCmd->command == TSDB_SQL_CURRENT_DB ||
+       pCmd->command == TSDB_SQL_SERV_VERSION ||
+       pCmd->command == TSDB_SQL_CLI_VERSION ||
+       pCmd->command == TSDB_SQL_CURRENT_USER )) {
     taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj);
     sem_wait(&pSql->rspSem);
   }
@@ -488,7 +420,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   int nRows = 0;
 
   if (pSql == NULL || pSql->signature != pSql) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
     *rows = NULL;
     return 0;
   }
@@ -504,8 +436,8 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
     pSql->cmd.command = pQueryInfo->command;
     pCmd->clauseIndex++;
 
-    pRes->numOfTotal += pRes->numOfTotalInCurrentClause;
+    pRes->numOfTotal += pRes->numOfClauseTotal;
-    pRes->numOfTotalInCurrentClause = 0;
+    pRes->numOfClauseTotal = 0;
     pRes->rspType = 0;
 
     pSql->numOfSubs = 0;
@@ -531,114 +463,81 @@ int taos_select_db(TAOS *taos, const char *db) {
 
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return TSDB_CODE_DISCONNECTED;
+    return TSDB_CODE_TSC_DISCONNECTED;
   }
 
   snprintf(sql, tListLen(sql), "use %s", db);
-  return taos_query(taos, sql);
+  SSqlObj* pSql = taos_query(taos, sql);
+  int32_t code = pSql->res.code;
+  taos_free_result(pSql);
+  
+  return code;
 }
 
-void taos_free_result_imp(TAOS_RES *res, int keepCmd) {
+void taos_free_result(TAOS_RES *res) {
-  if (res == NULL) return;
-
   SSqlObj *pSql = (SSqlObj *)res;
-  SSqlRes *pRes = &pSql->res;
+  tscTrace("%p start to free result", res);
-  SSqlCmd *pCmd = &pSql->cmd;
-
-  tscTrace("%p start to free result", pSql);
-
-  if (pSql->signature != pSql) return;
-
-  if (pRes == NULL || pRes->qhandle == 0) {
-    /* Query rsp is not received from vnode, so the qhandle is NULL */
-    tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp);
-    STscObj* pTscObj = pSql->pTscObj;
-    
-    if (pTscObj->pSql != pSql) {
-      tscTrace("%p SqlObj is freed by app", pSql);
-      tscFreeSqlObj(pSql);
-    } else {
-      if (keepCmd) {
-        tscFreeSqlResult(pSql);
-      } else {
-        tscPartiallyFreeSqlObj(pSql);
-      }
-    }
   
+  if (pSql == NULL || pSql->signature != pSql) {
+    tscTrace("%p result has been freed", pSql);
     return;
   }
   
-  // set freeFlag to 1 in retrieve message if there are un-retrieved results
+  SSqlRes *pRes = &pSql->res;
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  // The semaphore can not be changed while freeing async sub query objects.
+  if (pRes == NULL || pRes->qhandle == 0) {
+    tscTrace("%p SqlObj is freed by app, qhandle is null", pSql);
+    tscFreeSqlObj(pSql);
+    return;
+  }
+
+  // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   if (pQueryInfo == NULL) {
-    tscPartiallyFreeSqlObj(pSql);
+    tscFreeSqlObj(pSql);
     return;
   }
 
   pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
   /*
-   * case 1. Partial data have been retrieved from vnodes, but not all data has been retrieved yet.
+   * If the query process is cancelled by user in stable query, tscProcessSql should not be called
-   *         We need to recycle the connection by noticing the vnode return 0 results.
-   * case 2. When the query response is received from vnodes and the numOfRows is set to 0, the user calls
-   *         taos_free_result before the taos_fetch_row is called in non-stream computing,
-   *         we need to recycle the connection.
-   * case 3. If the query process is cancelled by user in stable query, tscProcessSql should not be called
    * for each subquery. Because the failure of execution tsProcessSql may trigger the callback function
    * be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport
    */
-  if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_SHOW || pCmd->command == TSDB_SQL_RETRIEVE ||
+  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false &&
-      pCmd->command == TSDB_SQL_FETCH) &&
+      (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_SHOW ||
-      (pRes->code != TSDB_CODE_QUERY_CANCELLED && ((pRes->numOfRows > 0 && pCmd->command < TSDB_SQL_LOCAL && pRes->completed == false) ||
+       pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_FETCH) &&
-       (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows == 0 && pCmd->command == TSDB_SQL_SELECT &&
+       (pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
-        pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)))) {
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
 
-    tscTrace("%p code:%d, numOfRows:%d, command:%d", pSql, pRes->code, pRes->numOfRows, pCmd->command);
+    tscTrace("%p start to send msg to free qhandle in dnode, command:%s", pSql, sqlCmd[pCmd->command]);
-
     pSql->freed = 1;
     tscProcessSql(pSql);
 
-    /*
+    // in case of sync model query, waits for response and then goes on
-     *  If release connection msg is sent to vnode, the corresponding SqlObj for async query can not be freed instantly,
+    if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
-     *  since its free operation is delegated to callback function, which is tscProcessMsgFromServer.
+      sem_wait(&pSql->rspSem);
-     */
+    }
-    STscObj* pObj = pSql->pTscObj;
-    if (pObj->pSql == pSql) {
-      pObj->pSql = NULL;
   }
-  } else { // if no free resource msg is sent to vnode, we free this object immediately.
-    STscObj* pTscObj = pSql->pTscObj;
 
-    if (pTscObj->pSql != pSql) {
   tscFreeSqlObj(pSql);
   tscTrace("%p sql result is freed by app", pSql);
-    } else {
-      if (keepCmd) {
-        tscFreeSqlResult(pSql);
-        tscTrace("%p sql result is freed while sql command is kept", pSql);
-      } else {
-        tscPartiallyFreeSqlObj(pSql);
-        tscTrace("%p sql result is freed by app", pSql);
 }
-    }
-  }
-}
-
-void taos_free_result(TAOS_RES *res) { taos_free_result_imp(res, 0); }
 
 // todo should not be used in async query
-int taos_errno(TAOS *taos) {
+int taos_errno(TAOS_RES *tres) {
-  STscObj *pObj = (STscObj *)taos;
+  SSqlObj *pSql = (SSqlObj *) tres;
 
-  if (pObj == NULL || pObj->signature != pObj) {
+  if (pSql == NULL || pSql->signature != pSql) {
     return terrno;
   }
 
-  return pObj->pSql->res.code;
+  return pSql->res.code;
 }
 
 /*
@@ -646,7 +545,7 @@ int taos_errno(TAOS *taos) {
  * why the sql is invalid
  */
 static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) {
-  if (code != TSDB_CODE_INVALID_SQL) {
+  if (code != TSDB_CODE_TSC_INVALID_SQL) {
     return false;
   }
 
@@ -661,13 +560,12 @@ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) {
 }
 
 // todo should not be used in async model
-char *taos_errstr(TAOS *taos) {
+char *taos_errstr(TAOS_RES *tres) {
-  STscObj *pObj = (STscObj *)taos;
+  SSqlObj *pSql = (SSqlObj *) tres;
 
-  if (pObj == NULL || pObj->signature != pObj)
+  if (pSql == NULL || pSql->signature != pSql) {
     return (char*) tstrerror(terrno);
-
+  }
-  SSqlObj* pSql = pObj->pSql;
 
   if (hasAdditionalErrorInfo(pSql->res.code, &pSql->cmd)) {
     return pSql->cmd.payload;
@@ -709,7 +607,7 @@ void taos_stop_query(TAOS_RES *res) {
   if (pSql->signature != pSql) return;
   tscTrace("%p start to cancel query", res);
 
-  pSql->res.code = TSDB_CODE_QUERY_CANCELLED;
+  pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
@@ -794,30 +692,31 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
 int taos_validate_sql(TAOS *taos, const char *sql) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return TSDB_CODE_DISCONNECTED;
+    return TSDB_CODE_TSC_DISCONNECTED;
   }
 
-  SSqlObj *pSql = pObj->pSql;
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
   
   pRes->numOfRows  = 1;
   pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   tscTrace("%p Valid SQL: %s pObj:%p", pSql, sql, pObj);
 
   int32_t sqlLen = strlen(sql);
   if (sqlLen > tsMaxSQLStringLen) {
     tscError("%p sql too long", pSql);
-    pRes->code = TSDB_CODE_INVALID_SQL;
+    pRes->code = TSDB_CODE_TSC_INVALID_SQL;
     return pRes->code;
   }
 
   pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
   if (pSql->sqlstr == NULL) {
-    pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to malloc sql string buffer", pSql);
     tscTrace("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
     return pRes->code;
@@ -849,7 +748,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
   pCmd->command = TSDB_SQL_MULTI_META;
   pCmd->count = 0;
 
-  int   code = TSDB_CODE_INVALID_TABLE_ID;
+  int   code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   char *str = (char *)tblNameList;
 
   SQueryInfo *pQueryInfo = NULL;
@@ -876,26 +775,24 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
     tblName[len] = '\0';
 
     str = nextStr + 1;
-
+    len = strtrim(tblName);
-    strtrim(tblName);
-    len = (uint32_t)strlen(tblName);
 
     SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName};
     tSQLGetToken(tblName, &sToken.type);
 
     // Check if the table name available or not
     if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {
-      code = TSDB_CODE_INVALID_TABLE_ID;
+      code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
       sprintf(pCmd->payload, "table name is invalid");
       return code;
     }
 
-    if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
+    if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
       return code;
     }
 
     if (++pCmd->count > TSDB_MULTI_METERMETA_MAX_NUM) {
-      code = TSDB_CODE_INVALID_TABLE_ID;
+      code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
       sprintf(pCmd->payload, "tables over the max number");
       return code;
     }
@@ -903,7 +800,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
     if (payloadLen + strlen(pTableMetaInfo->name) + 128 >= pCmd->allocSize) {
       char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen);
       if (pNewMem == NULL) {
-        code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+        code = TSDB_CODE_TSC_OUT_OF_MEMORY;
         sprintf(pCmd->payload, "failed to allocate memory");
         return code;
       }
@@ -927,15 +824,15 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
 
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return TSDB_CODE_DISCONNECTED;
+    return TSDB_CODE_TSC_DISCONNECTED;
   }
 
-  SSqlObj *pSql = pObj->pSql;
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
   SSqlRes *pRes = &pSql->res;
 
   pRes->numOfTotal = 0;  // the number of getting table meta from server
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   pRes->code = 0;
 
@@ -945,13 +842,13 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   int32_t tblListLen = strlen(tableNameList);
   if (tblListLen > MAX_TABLE_NAME_LENGTH) {
     tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH);
-    pRes->code = TSDB_CODE_INVALID_SQL;
+    pRes->code = TSDB_CODE_TSC_INVALID_SQL;
     return pRes->code;
   }
 
   char *str = calloc(1, tblListLen + 1);
   if (str == NULL) {
-    pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to malloc sql string buffer", pSql);
     return pRes->code;
   }

src/client/src/tscStream.c

@@ -19,6 +19,7 @@
 #include "tscLog.h"
 #include "tscUtil.h"
 #include "tsched.h"
+#include "tcache.h"
 #include "tsclient.h"
 #include "ttime.h"
 #include "ttimer.h"
@@ -77,31 +78,24 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
   int code = tscGetTableMeta(pSql, pTableMetaInfo);
   pSql->res.code = code;
 
-  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+  if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-
-  if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
     code = tscGetSTableVgroupInfo(pSql, 0);
     pSql->res.code = code;
-
-    if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
   }
 
-  tscTansformSQLFuncForSTableQuery(pQueryInfo);
-
   // failed to get meter/metric meta, retry in 10sec.
   if (code != TSDB_CODE_SUCCESS) {
     int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision);
     tscError("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime);
-  
     tscSetRetryTimer(pStream, pSql, retryDelayTime);
-    return;
-  }
 
+  } else {
+    tscTansformSQLFuncForSTableQuery(pQueryInfo);
     tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name);
-  tscProcessSql(pStream->pSql);
+    tscDoQuery(pStream->pSql);
-
     tscIncStreamExecutionCount(pStream);
   }
+}
 
 static void tscProcessStreamTimer(void *handle, void *tmrId) {
   SSqlStream *pStream = (SSqlStream *)handle;
@@ -147,7 +141,8 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
              retryDelay);
 
     STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
-    tscClearTableMetaInfo(pTableMetaInfo, true);
+    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true);
+    tfree(pTableMetaInfo->vgroupList);
   
     tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
     return;
@@ -165,7 +160,7 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) {
   if (timestamp != actualTimestamp) {
     // reset the timestamp of each agg point by using start time of each interval
     *((int64_t *)pRes->data) = actualTimestamp;
-    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64 "", pSql, pStream, timestamp, actualTimestamp);
+    tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64, pSql, pStream, timestamp, actualTimestamp);
   }
 }
 
@@ -208,7 +203,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
     
     if (pStream->numOfRes == 0) {
-      if (pQueryInfo->interpoType == TSDB_INTERPO_SET_VALUE || pQueryInfo->interpoType == TSDB_INTERPO_NULL) {
+      if (pQueryInfo->fillType == TSDB_FILL_SET_VALUE || pQueryInfo->fillType == TSDB_FILL_NULL) {
         SSqlRes *pRes = &pSql->res;
 
         /* failed to retrieve any result in this retrieve */
@@ -223,7 +218,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
           int16_t     offset = tscFieldInfoGetOffset(pQueryInfo, i);
           TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
 
-          assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->defaultVal[i]), pField->bytes, pField->type);
+          assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type);
           row[i] = pSql->res.data + offset;
         }
 
@@ -259,7 +254,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
              pStream->numOfRes);
 
     // release the metric/meter meta information reference, so data in cache can be updated
-    tscClearTableMetaInfo(pTableMetaInfo, false);
+
+    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+    tfree(pTableMetaInfo->vgroupList);
     tscSetNextLaunchTimer(pStream, pSql);
   }
 }
@@ -287,10 +284,10 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer)
       return;
     }
   
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              now + timer, timer, delay, pStream->stime, etime);
   } else {
-    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64 "", pStream->pSql, pStream,
+    tscTrace("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
              pStream->stime, timer, delay, pStream->stime - pStream->interval, pStream->stime - 1);
   }
 
@@ -380,7 +377,7 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   
   if (pQueryInfo->intervalTime < minIntervalTime) {
-    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->intervalTime, minIntervalTime);
     pQueryInfo->intervalTime = minIntervalTime;
   }
@@ -397,14 +394,14 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
   if (pQueryInfo->slidingTime == -1) {
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
   } else if (pQueryInfo->slidingTime < minSlidingTime) {
-    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream,
         pQueryInfo->slidingTime, minSlidingTime);
 
     pQueryInfo->slidingTime = minSlidingTime;
   }
 
   if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
-    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream,
+    tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream,
             pQueryInfo->slidingTime, pQueryInfo->intervalTime);
 
     pQueryInfo->slidingTime = pQueryInfo->intervalTime;
@@ -433,11 +430,11 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
   } else {             // timewindow based aggregation stream
     if (stime == 0) {  // no data in meter till now
       stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval;
-      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64 "", pSql, pStream, stime);
+      tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
     } else {
       int64_t newStime = (stime / pStream->interval) * pStream->interval;
       if (newStime != stime) {
-        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64 "", pSql, pStream, stime, newStime);
+        tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
         stime = newStime;
       }
     }
@@ -459,14 +456,14 @@ static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) {
   return (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? timer / 1000L : timer;
 }
 
-static void setErrorInfo(STscObj* pObj, int32_t code, char* info) {
+static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) {
-  if (pObj == NULL) {
+  if (pSql == NULL) {
     return;
   }
 
-  SSqlCmd* pCmd = &pObj->pSql->cmd;
+  SSqlCmd* pCmd = &pSql->cmd;
 
-  pObj->pSql->res.code = code;
+  pSql->res.code = code;
   
   if (info != NULL) {
     strncpy(pCmd->payload, info, pCmd->payloadLen);
@@ -480,57 +477,40 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
 
   SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
-    setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL);
     return NULL;
   }
 
   pSql->signature = pSql;
+  pSql->param = pSql;
   pSql->pTscObj = pObj;
+
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
-  int ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
-  if (TSDB_CODE_SUCCESS != ret) {
-    setErrorInfo(pObj, ret, NULL);
-    free(pSql);
-    return NULL;
-  }
-
-  pSql->sqlstr = strdup(sqlstr);
-  if (pSql->sqlstr == NULL) {
-    setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL);
-
-    tfree(pSql);
-    return NULL;
-  }
-
-  tsem_init(&pSql->rspSem, 0, 0);
-
-  SSqlInfo SQLInfo = {0};
-  tSQLParse(&SQLInfo, pSql->sqlstr);
-
-  tscResetSqlCmdObj(&pSql->cmd);
-  ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
-  if (TSDB_CODE_SUCCESS != ret) {
-    setErrorInfo(pObj, ret, NULL);
-    tscError("%p open stream failed, sql:%s, code:%d", pSql, sqlstr, TSDB_CODE_CLI_OUT_OF_MEMORY);
-    tscFreeSqlObj(pSql);
-    return NULL;
-  }
-  
-  pRes->code = tscToSQLCmd(pSql, &SQLInfo);
-  SQLInfoDestroy(&SQLInfo);
-
-  if (pRes->code != TSDB_CODE_SUCCESS) {
-    setErrorInfo(pObj, pRes->code, pCmd->payload);
 
+  SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream));
+  if (pStream == NULL) {
     tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code);
     tscFreeSqlObj(pSql);
     return NULL;
   }
+  pSql->pStream = pStream;
 
-  SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream));
+  pSql->sqlstr = calloc(1, strlen(sqlstr) + 1);
-  if (pStream == NULL) {
+  if (pSql->sqlstr == NULL) {
-    setErrorInfo(pObj, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL);
+    tscError("%p failed to malloc sql string buffer", pSql);
+    tscFreeSqlObj(pSql);
+    return NULL;;
+  }
+  strtolower(pSql->sqlstr, sqlstr);
+
+  tsem_init(&pSql->rspSem, 0, 0);
+  int32_t code = doAsyncParseSql(pSql);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    sem_wait(&pSql->rspSem);
+  }
+
+  if (pRes->code != TSDB_CODE_SUCCESS) {
+    setErrorInfo(pSql, pRes->code, pCmd->payload);
 
     tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code);
     tscFreeSqlObj(pSql);
@@ -550,13 +530,13 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
   pStream->ctime = taosGetTimestamp(pStream->precision);
   pStream->etime = pQueryInfo->window.ekey;
 
-  pSql->pStream = pStream;
   tscAddIntoStreamList(pStream);
 
   tscSetSlidingWindowInfo(pSql, pStream);
   pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, stime);
 
   int64_t starttime = tscGetLaunchTimestamp(pStream);
+  pCmd->command = TSDB_SQL_SELECT;
   taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer);
 
   tscTrace("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,

src/client/src/tscSub.c

@@ -14,8 +14,6 @@
  */
 
 #include "os.h"
-
-#include "shash.h"
 #include "taos.h"
 #include "trpc.h"
 #include "tsclient.h"
@@ -44,8 +42,7 @@ typedef struct SSub {
   int                     interval;
   TAOS_SUBSCRIBE_CALLBACK fp;
   void *                  param;
-  int                     numOfTables;
+  SArray* progress;
-  SSubscriptionProgress * progress;
 } SSub;
 
 
@@ -57,92 +54,113 @@ static int tscCompareSubscriptionProgress(const void* a, const void* b) {
   return 0;
 }
 
-TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid) {
+TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
-  if (sub == NULL)
+  if (sub == NULL) {
-    return 0;
+    return dflt;
-
-  SSub* pSub = (SSub*)sub;
-  for (int s = 0, e = pSub->numOfTables; s < e;) {
-    int m = (s + e) / 2;
-    SSubscriptionProgress* p = pSub->progress + m;
-    if (p->uid > uid)
-      e = m;
-    else if (p->uid < uid)
-      s = m + 1;
-    else
-      return p->key;
   }
+  SSub* pSub = (SSub*)sub;
 
-  return 0;
+  SSubscriptionProgress target = {.uid = uid, .key = 0};
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  if (p == NULL) {
+    return dflt;
+  }
+  return p->key;
 }
 
 void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   if( sub == NULL)
     return;
-
   SSub* pSub = (SSub*)sub;
-  for (int s = 0, e = pSub->numOfTables; s < e;) {
+
-    int m = (s + e) / 2;
+  SSubscriptionProgress target = {.uid = uid, .key = ts};
-    SSubscriptionProgress* p = pSub->progress + m;
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
-    if (p->uid > uid)
+  if (p != NULL) {
-      e = m;
+    p->key = ts;
-    else if (p->uid < uid)
-      s = m + 1;
-    else {
-      if (ts >= p->key) p->key = ts;
-      break;
   }
 }
+
+
+static void asyncCallback(void *param, TAOS_RES *tres, int code) {
+  assert(param != NULL);
+  SSqlObj *pSql = ((SSqlObj *)param);
+  
+  pSql->res.code = code;
+  sem_post(&pSql->rspSem);
 }
 
 
 static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) {
-  SSub* pSub = calloc(1, sizeof(SSub));
+  SSub* pSub = NULL;
-  if (pSub == NULL) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for subscription");
-    return NULL;
-  }
 
-  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  TRY( 8 ) {
-  if (pSql == NULL) {
+    SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj));
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    CLEANUP_PUSH_FREE(true, pSql);
-    tscError("failed to allocate SSqlObj for subscription");
+    SSqlCmd *pCmd = &pSql->cmd;
-    goto _pSql_failed;
+    SSqlRes *pRes = &pSql->res;
+
+    if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+      THROW(TAOS_SYSTEM_ERROR(errno));
     }
+    CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem);
 
     pSql->signature = pSql;
+    pSql->param = pSql;
     pSql->pTscObj = pObj;
+    pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
+    pSql->fp = asyncCallback;
 
-  char* sqlstr = (char*)malloc(strlen(sql) + 1);
+    int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
-  if (sqlstr == NULL) {
+    if (code != TSDB_CODE_SUCCESS) {
-    tscError("failed to allocate sql string for subscription");
+      THROW(code);
-    goto failed;
     }
-  strcpy(sqlstr, sql);
+    CLEANUP_PUSH_FREE(true, pCmd->payload);
-  strtolower(sqlstr, sqlstr);
-  pSql->sqlstr = sqlstr;
 
-  tsem_init(&pSql->rspSem, 0, 0);
+    pRes->qhandle = 0;
-
-  SSqlRes *pRes = &pSql->res;
     pRes->numOfRows = 1;
-  pRes->numOfTotal = 0;
 
+    pSql->sqlstr = strdup_throw(sql);
+    CLEANUP_PUSH_FREE(true, pSql->sqlstr);
+    strtolower(pSql->sqlstr, pSql->sqlstr);
+
+    code = tsParseSql(pSql, false);
+    if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+      // wait for the callback function to post the semaphore
+      sem_wait(&pSql->rspSem);
+      code = pSql->res.code;
+    }
+    if (code != TSDB_CODE_SUCCESS) {
+      tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code));
+      THROW( code );
+    }
+
+    if (pSql->cmd.command != TSDB_SQL_SELECT) {
+      tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
+      THROW( -1 );  // TODO
+    }
+
+    pSub = calloc_throw(1, sizeof(SSub));
+    CLEANUP_PUSH_FREE(true, pSub);
     pSql->pSubscription = pSub;
     pSub->pSql = pSql;
     pSub->signature = pSub;
     strncpy(pSub->topic, topic, sizeof(pSub->topic));
     pSub->topic[sizeof(pSub->topic) - 1] = 0;
+    pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
+    if (pSub->progress == NULL) {
+      THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
+    }
+
+    CLEANUP_EXECUTE();
+
+  } CATCH( code ) {
+    tscError("failed to create subscription object: %s", tstrerror(code));
+    CLEANUP_EXECUTE();
+    pSub = NULL;
+
+  } END_TRY
+
   return pSub;
-
-failed:
-  tfree(sqlstr);
-
-_pSql_failed:
-  tfree(pSql);
-  tfree(pSub);
-  return NULL;
 }
 
 
@@ -159,61 +177,71 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
 }
 
 
-int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
+static SArray* getTableList( SSqlObj* pSql ) {
-  int code = (uint8_t)tsParseSql(pSub->pSql, false);
+  const char* p = strstr( pSql->sqlstr, " from " );
-  if (code != TSDB_CODE_SUCCESS) {
+  char* sql = alloca(strlen(p) + 32);
-    tscError("failed to parse sql statement: %s", pSub->topic);
+  sprintf(sql, "select tbid(tbname)%s", p);
-    return 0;
+  
+  SSqlObj* pSql1 = taos_query(pSql->pTscObj, sql);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    tscError("failed to retrieve table id: %s", tstrerror(terrno));
+    return NULL;
   }
 
-  SSqlCmd* pCmd = &pSub->pSql->cmd;
+  TAOS_ROW row;
-  if (pCmd->command != TSDB_SQL_SELECT) {
+  SArray* result = taosArrayInit( 128, sizeof(STidTags) );
-    tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
+  while ((row = taos_fetch_row(pSql1))) {
-    return 0;
+    STidTags tags;
+    memcpy(&tags, row[0], sizeof(tags));
+    taosArrayPush(result, &tags);
   }
 
-  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0);
+  taos_free_result(pSql1);
-  int numOfTables = 0;
+  
-  if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+  return result;
-//    SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
-//    for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
-//      SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
-//      numOfTables += pVnodeSidList->numOfSids;
-//    }
 }
 
-  SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress));
-  if (progress == NULL) {
-    tscError("failed to allocate memory for progress: %s", pSub->topic);
-    return 0;
-  }
 
-  if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
-    numOfTables = 1;
+  SSqlObj* pSql = pSub->pSql;
-    int64_t uid = pTableMetaInfo->pTableMeta->uid;
-    progress[0].uid = uid;
-    progress[0].key = tscGetSubscriptionProgress(pSub, uid);
-  } else {
-//    SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
-//    numOfTables = 0;
-//    for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
-//      SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
-//      for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) {
-//        STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
-//        int64_t uid = pTableMetaInfo->uid;
-//        progress[numOfTables].uid = uid;
-//        progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid);
-//      }
-//    }
-    qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress);
-  }
 
-  free(pSub->progress);
+  SSqlCmd* pCmd = &pSql->cmd;
-  pSub->numOfTables = numOfTables;
-  pSub->progress = progress;
 
   pSub->lastSyncTime = taosGetTimestampMs();
 
+  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
+  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
+    STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
+    SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+    if (p == NULL) {
+      taosArrayClear(pSub->progress);
+      taosArrayPush(pSub->progress, &target);
+    }
+    return 1;
+  }
+
+  SArray* tables = getTableList(pSql);
+  size_t numOfTables = taosArrayGetSize(tables);
+
+  SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
+  for( size_t i = 0; i < numOfTables; i++ ) {
+    STidTags* tt = taosArrayGet( tables, i );
+    SSubscriptionProgress p = { .uid = tt->uid };
+    p.key = tscGetSubscriptionProgress(pSub, tt->uid, INT64_MIN);
+    taosArrayPush(progress, &p);
+  }
+  taosArraySort(progress, tscCompareSubscriptionProgress);
+
+  taosArrayDestroy(pSub->progress);
+  pSub->progress = progress;
+
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+    taosArraySort( tables, tscCompareTidTags );
+    tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+  }
+  taosArrayDestroy(tables);
+
   return 1;
 }
 
@@ -248,32 +276,22 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
     return 0;
   }
 
-  if (fgets(buf, sizeof(buf), fp) == NULL || atoi(buf) < 0) {
+  SArray* progress = pSub->progress;
-    tscTrace("invalid subscription progress file: %s", pSub->topic);
+  taosArrayClear(progress);
-    fclose(fp);
+  while( 1 ) {
-    return 0;
-  }
-
-  int numOfTables = atoi(buf);
-  SSubscriptionProgress* progress = calloc(numOfTables, sizeof(SSubscriptionProgress));
-  for (int i = 0; i < numOfTables; i++) {
     if (fgets(buf, sizeof(buf), fp) == NULL) {
       fclose(fp);
-      free(progress);
       return 0;
     }
-    int64_t uid, key;
+    SSubscriptionProgress p;
-    sscanf(buf, "%" SCNd64 ":%" SCNd64, &uid, &key);
+    sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key);
-    progress[i].uid = uid;
+    taosArrayPush(progress, &p);
-    progress[i].key = key;
   }
 
   fclose(fp);
 
-  qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress);
+  taosArraySort(progress, tscCompareSubscriptionProgress);
-  pSub->numOfTables = numOfTables;
+  tscTrace("subscription progress loaded, %d tables: %s", taosArrayGetSize(progress), pSub->topic);
-  pSub->progress = progress;
-  tscTrace("subscription progress loaded, %d tables: %s", numOfTables, pSub->topic);
   return 1;
 }
 
@@ -283,7 +301,9 @@ void tscSaveSubscriptionProgress(void* sub) {
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
   if (access(path, 0) != 0) {
-    mkdir(path, 0777);
+    if (mkdir(path, 0777) != 0 && errno != EEXIST) {
+      tscError("failed to create subscribe dir: %s", path);
+    }
   }
 
   sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
@@ -294,11 +314,10 @@ void tscSaveSubscriptionProgress(void* sub) {
   }
 
   fputs(pSub->pSql->sqlstr, fp);
-  fprintf(fp, "\n%d\n", pSub->numOfTables);
+  fprintf(fp, "\n");
-  for (int i = 0; i < pSub->numOfTables; i++) {
+  for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) {
-    int64_t uid = pSub->progress[i].uid;
+    SSubscriptionProgress* p = taosArrayGet(pSub->progress, i);
-    TSKEY key = pSub->progress[i].key;
+    fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key);
-    fprintf(fp, "%" PRId64 ":%" PRId64 "\n", uid, key);
   }
 
   fclose(fp);
@@ -307,7 +326,7 @@ void tscSaveSubscriptionProgress(void* sub) {
 TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) {
   STscObj* pObj = (STscObj*)taos;
   if (pObj == NULL || pObj->signature != pObj) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
     tscError("connection disconnected");
     return NULL;
   }
@@ -363,35 +382,34 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
     tscRemoveFromSqlList(pSql);
 
     if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) {
-      tscTrace("begin meter synchronization");
+      tscTrace("begin table synchronization");
-      char* sqlstr = pSql->sqlstr;
-      pSql->sqlstr = NULL;
-      taos_free_result_imp(pSql, 0);
-      pSql->sqlstr = sqlstr;
-      taosCacheEmpty(tscCacheHandle);
       if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
-      tscTrace("meter synchronization completed");
+      tscTrace("table synchronization completed");
-    } else {
+    }
+
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
     
     uint32_t type = pQueryInfo->type;
-      taos_free_result_imp(pSql, 1);
+    tscFreeSqlResult(pSql);
     pRes->numOfRows = 1;
-      pRes->numOfTotal = 0;
     pRes->qhandle = 0;
     pSql->cmd.command = TSDB_SQL_SELECT;
     pQueryInfo->type = type;
 
     tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
-    }
 
+    pSql->fp = asyncCallback;
+    pSql->param = pSql;
     tscDoQuery(pSql);
-    if (pRes->code != TSDB_CODE_NOT_ACTIVE_TABLE) {
+    sem_wait(&pSql->rspSem);
-      break;
+
+    if (pRes->code != TSDB_CODE_SUCCESS) {
+      continue;
     }
     // meter was removed, make sync time zero, so that next retry will
     // do synchronization first
     pSub->lastSyncTime = 0;
+    break;
   }
 
   if (pRes->code != TSDB_CODE_SUCCESS) {
@@ -421,7 +439,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   tscFreeSqlObj(pSub->pSql);
-  free(pSub->progress);
+  taosArrayDestroy(pSub->progress);
   memset(pSub, 0, sizeof(*pSub));
   free(pSub);
 }

src/client/src/tscSubquery.c

@@ -330,7 +330,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
     pNewQueryInfo->limit = pSupporter->limit;
   
     // fetch the join tag column
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, 0);
       assert(pQueryInfo->tagCond.joinInfo.hasJoin);
     
@@ -348,7 +348,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
   
   //prepare the subqueries object failed, abort
   if (!success) {
-    pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql,
         pSql->numOfSubs, pSql->res.code);
     freeJoinSubqueryObj(pSql);
@@ -412,10 +412,6 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et)
 
 static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) {
   SSqlObj* pParentSql = pSupporter->pObj;
-//  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
-  
-//  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
   
 //  if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) {
@@ -467,77 +463,51 @@ static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj*
   }
 }
 
-int32_t tagsOrderCompar(const void* p1, const void* p2) {
+int32_t tscCompareTidTags(const void* p1, const void* p2) {
-  STidTags* t1 = (STidTags*) p1;
+  const STidTags* t1 = (const STidTags*) p1;
-  STidTags* t2 = (STidTags*) p2;
+  const STidTags* t2 = (const STidTags*) p2;
   
   if (t1->vgId != t2->vgId) {
     return (t1->vgId > t2->vgId) ? 1 : -1;
-  } else {
+  }
   if (t1->tid != t2->tid) {
     return (t1->tid > t2->tid) ? 1 : -1;
-    } else {
+  }
   return 0;
 }
-  }
-}
 
-static void doBuildVgroupTableInfo(SArray* res, STableMetaInfo* pTableMetaInfo) {
+void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) {
-  SArray* pGroup = taosArrayInit(4, sizeof(SVgroupTableInfo));
+  SArray* result = taosArrayInit( 4, sizeof(SVgroupTableInfo) );
+  SArray* vgTables = NULL;
+  STidTags* prev = NULL;
 
-  SArray* vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo));
+  size_t numOfTables = taosArrayGetSize( tables );
-  int32_t size = taosArrayGetSize(res);
+  for( size_t i = 0; i < numOfTables; i++ ) {
-  
+    STidTags* tt = taosArrayGet( tables, i );
-  STidTags* prev = taosArrayGet(res, 0);
-  int32_t prevVgId = prev->vgId;
-  
-  STableIdInfo item = {.uid = prev->uid, .tid = prev->tid, .key = INT64_MIN};
-  taosArrayPush(vgTableIdItem, &item);
-  
-  for(int32_t k = 1; k < size; ++k) {
-    STidTags* t1 = taosArrayGet(res, k);
-    if (prevVgId != t1->vgId) {
-      
-      SVgroupTableInfo info = {0};
 
+    if( prev == NULL || tt->vgId != prev->vgId ) {
       SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
+
+      SVgroupTableInfo info = {{ 0 }};
       for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) {
-        if (prevVgId == pvg->vgroups[m].vgId) {
+        if( tt->vgId == pvg->vgroups[m].vgId ) {
           info.vgInfo = pvg->vgroups[m];
           break;
         }
       }
-      
       assert( info.vgInfo.numOfIps != 0 );
-      info.itemList = vgTableIdItem;
-      taosArrayPush(pGroup, &info);
 
-      vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo));
+      vgTables = taosArrayInit( 4, sizeof(STableIdInfo) );
-      STableIdInfo item1 = {.uid = t1->uid, .tid = t1->tid, .key = INT64_MIN};
+      info.itemList = vgTables;
-      taosArrayPush(vgTableIdItem, &item1);
+      taosArrayPush( result, &info );
-      prevVgId = t1->vgId;
-    } else {
-      taosArrayPush(vgTableIdItem, &item);
-    }
     }
 
-  if (taosArrayGetSize(vgTableIdItem) > 0) {
+    STableIdInfo item = { .uid = tt->uid, .tid = tt->tid, .key = INT64_MIN };
-    SVgroupTableInfo info = {0};
+    taosArrayPush( vgTables, &item );
-    SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
+    prev = tt;
-    
-    for(int32_t m = 0; m < pvg->numOfVgroups; ++m) {
-      if (prevVgId == pvg->vgroups[m].vgId) {
-        info.vgInfo = pvg->vgroups[m];
-        break;
-      }
   }
 
-    assert(info.vgInfo.numOfIps != 0);
+  pTableMetaInfo->pVgroupTables = result;
-    info.itemList = vgTableIdItem;
-    taosArrayPush(pGroup, &info);
-  }
-  
-  pTableMetaInfo->pVgroupTables = pGroup;
 }
 
 static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@@ -602,21 +572,6 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
   
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   
-//  if (pSupporter->pState->code != TSDB_CODE_SUCCESS) {
-//    tscError("%p abort query due to other subquery failure. code:%d, global code:%s", pSql, numOfRows,
-//             tstrerror(pSupporter->pState->code));
-//
-//    quitAllSubquery(pParentSql, pSupporter);
-//    return;
-//  }
-//
-//  if (numOfRows < 0) {
-//    tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
-//    pSupporter->pState->code = numOfRows;
-//    quitAllSubquery(pParentSql, pSupporter);
-//    return;
-//  }
-  
   // response of tag retrieve
   if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
     if (numOfRows == 0 || pSql->res.completed) {
@@ -646,8 +601,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
       SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
       SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
   
-      qsort(p1->pIdTagList, p1->num, p1->tagSize, tagsOrderCompar);
+      qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
-      qsort(p2->pIdTagList, p2->num, p2->tagSize, tagsOrderCompar);
+      qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
       
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
@@ -687,11 +642,11 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         
         SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0);
         STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0);
-        doBuildVgroupTableInfo(s1, pTableMetaInfo1);
+        tscBuildVgroupTableInfo(pTableMetaInfo1, s1);
         
         SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0);
         STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
-        doBuildVgroupTableInfo(s2, pTableMetaInfo2);
+        tscBuildVgroupTableInfo(pTableMetaInfo2, s2);
   
         pSupporter->pState->numOfCompleted = 0;
         pSupporter->pState->code = 0;
@@ -743,7 +698,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
     if (pBuf == NULL) {
       tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows);
 
-      pSupporter->pState->code = TSDB_CODE_APP_ERROR;  // todo set the informative code
+      pSupporter->pState->code = TSDB_CODE_TSC_APP_ERROR;  // todo set the informative code
       quitAllSubquery(pParentSql, pSupporter);
       return;
     }
@@ -816,7 +771,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         }
     
         SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
-        pRes1->numOfTotalInCurrentClause += pRes1->numOfRows;
+        pRes1->numOfClauseTotal += pRes1->numOfRows;
       }
   
       // data has retrieved to client, build the join results
@@ -1064,13 +1019,13 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
   if (pSql->pSubs == NULL) {
     pSql->pSubs = calloc(pSupporter->pState->numOfTotal, POINTER_BYTES);
     if (pSql->pSubs == NULL) {
-      return TSDB_CODE_CLI_OUT_OF_MEMORY;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
   }
   
   SSqlObj *pNew = createSubqueryObj(pSql, tableIndex, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL);
   if (pNew == NULL) {
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
   
   pSql->pSubs[pSql->numOfSubs++] = pNew;
@@ -1115,7 +1070,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     tscInitQueryInfo(pNewQueryInfo);
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
     
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // return the tableId & tag
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
       SSchema s = {0};
       SColumnIndex index = {0};
   
@@ -1129,7 +1084,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
         
         int16_t bytes = 0;
         int16_t type = 0;
-        int16_t inter = 0;
+        int32_t inter = 0;
         
         getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
         
@@ -1208,7 +1163,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
     if (pSupporter == NULL) {  // failed to create support struct, abort current query
       tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i);
       pState->numOfCompleted = pQueryInfo->numOfTables - i - 1;
-      pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       
       return pSql->res.code;
     }
@@ -1216,13 +1171,13 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
     int32_t code = tscLaunchJoinSubquery(pSql, i, pSupporter);
     if (code != TSDB_CODE_SUCCESS) {  // failed to create subquery object, quit query
       tscDestroyJoinSupporter(pSupporter);
-      pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       
       break;
     }
   }
   
-  pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_METRIC_JOIN_RETRIEVE;
+  pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_TABLE_JOIN_RETRIEVE;
   
   return TSDB_CODE_SUCCESS;
 }
@@ -1254,7 +1209,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   SSqlCmd *pCmd = &pSql->cmd;
   
   // pRes->code check only serves in launching metric sub-queries
-  if (pRes->code == TSDB_CODE_QUERY_CANCELLED) {
+  if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) {
     pCmd->command = TSDB_SQL_RETRIEVE_LOCALMERGE;  // enable the abort of kill super table function.
     return pRes->code;
   }
@@ -1275,7 +1230,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   
   int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
   if (ret != 0) {
-    pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tscQueueAsyncRes(pSql);
     return ret;
   }
@@ -1336,14 +1291,14 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   
   if (i < pSql->numOfSubs) {
     tscError("%p failed to prepare subquery structure and launch subqueries", pSql);
-    pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     
     tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs);
     doCleanupSubqueries(pSql, i, pState);
     return pRes->code;   // free all allocated resource
   }
   
-  if (pRes->code == TSDB_CODE_QUERY_CANCELLED) {
+  if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) {
     tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs);
     doCleanupSubqueries(pSql, i, pState);
     return pRes->code;
@@ -1414,7 +1369,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
     
     /*
      * kill current sub-query connection, which may retrieve data from vnodes;
-     * Here we get: pPObj->res.code == TSDB_CODE_QUERY_CANCELLED
+     * Here we get: pPObj->res.code == TSDB_CODE_TSC_QUERY_CANCELLED
      */
     pSql->res.numOfRows = 0;
     trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;  // disable retry efforts
@@ -1435,7 +1390,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);
       
       // clear local saved number of results
-      trsupport->localBuffer->numOfElems = 0;
+      trsupport->localBuffer->num = 0;
       pthread_mutex_unlock(&trsupport->queryMutex);
       
       tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql,
@@ -1446,7 +1401,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
         tscError("%p sub:%p failed to create new subquery sqlObj due to out of memory, abort retry",
                  trsupport->pParentSqlObj, pSql);
         
-        pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+        pState->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
         trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
         return;
       }
@@ -1455,8 +1410,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       return;
     } else {  // reach the maximum retry count, abort
       atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows);
-      tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql,
+      tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pPObj, pSql,
-               numOfRows, subqueryIndex, tstrerror(pState->code));
+               tstrerror(numOfRows), subqueryIndex, tstrerror(pState->code));
     }
   }
   
@@ -1469,7 +1424,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   }
   
   // all subqueries are failed
-  tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code);
+  tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pPObj, pState->numOfTotal, tstrerror(pState->code));
   pPObj->res.code = pState->code;
   
   // release allocated resource
@@ -1502,7 +1457,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
   
   // data in from current vnode is stored in cache and disk
-  uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
+  uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num;
     tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
         pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
         numOfRowsFromSubquery, idx);
@@ -1510,17 +1465,17 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
 
 #ifdef _DEBUG_VIEW
-  printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
+  printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->num);
     SSrcColumnInfo colInfo[256] = {0};
     tscGetSrcColumnInfo(colInfo, pQueryInfo);
-    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
+    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->num,
-                       trsupport->localBuffer->numOfElems, colInfo);
+                       trsupport->localBuffer->num, colInfo);
 #endif
   
   if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
     tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
              tsAvailTmpDirGB, tsMinimalTmpDirGB);
-    tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
+    tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE);
     return;
   }
   
@@ -1529,7 +1484,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
                                   pQueryInfo->groupbyExpr.orderType);
   if (ret != 0) { // set no disk space error info, and abort retry
-    return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
+    return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE);
   }
   
   // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
@@ -1552,8 +1507,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
   tscClearInterpInfo(pPQueryInfo);
   
-  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
+  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pPObj);
-                        &pPObj->cmd, &pPObj->res);
   tscTrace("%p build loser tree completed", pPObj);
   
   pPObj->res.precision = pSql->res.precision;
@@ -1603,13 +1557,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
     assert(pRes->numOfRows == numOfRows);
     int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
     
-//    tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
+    tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%s, orderOfSub:%d", pPObj, pSql,
-//             pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
+             pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx);
     
     if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
       tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64,
                pPObj, pSql, tsMaxNumOfOrderedResults, num);
-      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_SORTED_RES_TOO_MANY);
+      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY);
       return;
     }
 
@@ -1624,14 +1578,14 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
     if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
       tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
                tsAvailTmpDirGB, tsMinimalTmpDirGB);
-      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
+      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE);
       return;
     }
     
     int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data,
                                pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
     if (ret < 0) { // set no disk space error info, and abort retry
-      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
+      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE);
       
     } else if (pRes->completed) {
       tscAllDataRetrievedFromDnode(trsupport, pSql);
@@ -1718,7 +1672,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
         tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
                  trsupport->pParentSqlObj, pSql, pVgroup->vgId, trsupport->subqueryIndex);
         
-        pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+        pState->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
         trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
       } else {
         SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
@@ -1759,8 +1713,14 @@ static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) {
   // increase the total inserted rows
   if (numOfRows > 0) {
     pParentObj->res.numOfRows += numOfRows;
+  } else {
+    SSqlObj* pSql = (SSqlObj*) tres;
+    assert(pSql != NULL && pSql->res.code == numOfRows);
+    
+    pParentObj->res.code = pSql->res.code;
   }
   
+  taos_free_result(tres);
   int32_t completed = atomic_add_fetch_32(&pState->numOfCompleted, 1);
   if (completed < total) {
     return;
@@ -1778,7 +1738,7 @@ static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) {
   pParentObj->fp = pParentObj->fetchFp;
   
   // all data has been sent to vnode, call user function
-  (*pParentObj->fp)(pParentObj->param, tres, numOfRows);
+  (*pParentObj->fp)(pParentObj->param, pParentObj, numOfRows);
 }
 
 int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
@@ -1810,13 +1770,20 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
       break;
     }
   
+    /*
+     * assign the callback function to fetchFp to make sure that the error process function can restore
+     * the callback function (multiVnodeInsertMerge) correctly.
+     */
+    pNew->fetchFp = pNew->fp;
     pSql->pSubs[i] = pNew;
+    pNew->fetchFp = pNew->fp;
+    
     tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, i);
   }
   
   if (i < pSql->numOfSubs) {
     tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql);
-    pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+    pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return pRes->code;  // free all allocated resource
   }
   
@@ -1880,7 +1847,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
         pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex];
       }
       
-      pRes->numOfTotalInCurrentClause++;
+      pRes->numOfClauseTotal++;
       break;
     } else {  // continue retrieve data from vnode
       if (!tscHashRemainDataInSubqueryResultSet(pSql)) {
@@ -1925,9 +1892,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
 static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
   SSqlRes *pRes = &pSql->res;
   
-  if (pRes->tsrow[columnIndex] != NULL && isNull(pRes->tsrow[columnIndex], pField->type)) {
+  if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
-    pRes->tsrow[columnIndex] = NULL;
-  } else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
     // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
     if (pRes->buffer[columnIndex] == NULL) {
       pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
@@ -1936,11 +1901,14 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
     /* string terminated char for binary data*/
     memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE);
     
-    if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) {
+    int32_t length = taosUcs4ToMbs(pRes->tsrow[columnIndex], pRes->length[columnIndex], pRes->buffer[columnIndex]);
+    if ( length >= 0 ) {
       pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
+      pRes->length[columnIndex] = length;
     } else {
-      tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);
+      tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow[columnIndex]);
       pRes->tsrow[columnIndex] = NULL;
+      pRes->length[columnIndex] = 0;
     }
   }
 }
@@ -1953,7 +1921,7 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId)
   
   for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
     pExpr = taosArrayGetP(pSupport->exprList, i);
-    if (strncmp(name, pExpr->aliasName, TSDB_COL_NAME_LEN) == 0) {
+    if (strncmp(name, pExpr->aliasName, sizeof(pExpr->aliasName) - 1) == 0) {
       index = i;
       break;
     }
@@ -1969,7 +1937,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
   
   assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
   
-  if(pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+  if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     if (pRes->completed) {
       tfree(pRes->tsrow);
     }
@@ -1984,7 +1952,8 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
   
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   
-  for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) {
+  size_t size = tscNumOfFields(pQueryInfo);
+  for (int i = 0; i < size; ++i) {
     SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i);
     if (pSup->pSqlExpr != NULL) {
       tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);

src/client/src/tscSystem.c

@@ -23,6 +23,7 @@
 #include "tutil.h"
 #include "tsched.h"
 #include "tscLog.h"
+#include "tscUtil.h"
 #include "tsclient.h"
 #include "tglobal.h"
 #include "tconfig.h"
@@ -56,19 +57,22 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) {
     memset(&rpcInit, 0, sizeof(rpcInit));
     rpcInit.localPort = 0;
     rpcInit.label = "TSC";
-    rpcInit.numOfThreads = tscNumOfThreads;
+    rpcInit.numOfThreads = 1;  // every DB connection has only one thread
     rpcInit.cfp = tscProcessMsgFromServer;
-    rpcInit.sessions = tsMaxVnodeConnections;
+    rpcInit.sessions = tsMaxConnections;
     rpcInit.connType = TAOS_CONN_CLIENT;
     rpcInit.user = (char*)user;
     rpcInit.idleTime = 2000;
     rpcInit.ckey = "key";
+    rpcInit.spi = 1;
     rpcInit.secret = secretEncrypt;
 
     *pDnodeConn = rpcOpen(&rpcInit);
     if (*pDnodeConn == NULL) {
       tscError("failed to init connection to TDengine");
       return -1;
+    } else {
+      tscTrace("dnodeConn:%p is created, user:%s", *pDnodeConn, user);
     }
   }
 
@@ -77,7 +81,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) {
 
 void taos_init_imp() {
   char temp[128];
-  struct stat dirstat;
   
   errno = TSDB_CODE_SUCCESS;
   srand(taosGetTimestampSec());
@@ -90,7 +93,9 @@ void taos_init_imp() {
     taosReadGlobalLogCfg();
 
     // For log directory
-    if (stat(tsLogDir, &dirstat) < 0) mkdir(tsLogDir, 0755);
+    if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
+      printf("failed to create log dir:%s\n", tsLogDir);
+    }
 
     sprintf(temp, "%s/taoslog", tsLogDir);
     if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) {
@@ -111,17 +116,13 @@ void taos_init_imp() {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
   }
 
-  tscMgmtIpSet.inUse = 0;
+  if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) {
-  tscMgmtIpSet.numOfIps = 1;
+    tscError("failed to init mnode IP list");
-  taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+    return;
-
-  if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) {
-    tscMgmtIpSet.numOfIps = 2;
-    taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
   } 
 
   tscInitMsgsFp();
-  int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections;
+  int queueSize = tsMaxConnections*2;
 
   if (tscEmbedded == 0) {
     tscNumOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 2.0;
@@ -137,17 +138,17 @@ void taos_init_imp() {
     return;
   }
 
-  tscTmr = taosTmrInit(tsMaxMgmtConnections * 2, 200, 60000, "TSC");
+  tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC");
   if(0 == tscEmbedded){
     taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr);      
   }
   
   int64_t refreshTime = tsTableMetaKeepTimer;
-  refreshTime = refreshTime > 2 ? 2 : refreshTime;
+  refreshTime = refreshTime > 10 ? 10 : refreshTime;
-  refreshTime = refreshTime < 1 ? 1 : refreshTime;
+  refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
   if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
+    tscCacheHandle = taosCacheInit(refreshTime);
   }
 
   tscTrace("client is initialized successfully");
@@ -179,7 +180,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        strncpy(configDir, pStr, TSDB_FILENAME_LEN);
+        tstrncpy(configDir, pStr, TSDB_FILENAME_LEN);
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscPrint("set config file directory:%s", pStr);
       } else {
@@ -200,7 +201,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
         tscPrint("set shellActivityTimer:%d", tsShellActivityTimer);
       } else {
         tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr,
-                tsCfgStatusStr[cfg->cfgStatus], (int32_t *)cfg->ptr);
+                tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
       }
       break;
 
@@ -233,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
           tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale);
         }
 
-        strncpy(tsLocale, locale, tListLen(tsLocale));
+        tstrncpy(tsLocale, locale, sizeof(tsLocale));
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -248,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
               tscPrint("charset changed from %s to %s", tsCharset, charset);
             }
 
-            strncpy(tsCharset, charset, tListLen(tsCharset));
+            tstrncpy(tsCharset, charset, sizeof(tsCharset));
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -285,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
             tscPrint("charset changed from %s to %s", tsCharset, pStr);
           }
 
-          strncpy(tsCharset, pStr, tListLen(tsCharset));
+          tstrncpy(tsCharset, pStr, sizeof(tsCharset));
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscPrint("charset:%s not valid", pStr);
@@ -323,7 +324,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 //          return -1;
 //        }
 
-        strncpy(tsSocketType, pStr, tListLen(tsSocketType));
+        tstrncpy(tsSocketType, pStr, sizeof(tsSocketType));
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscPrint("socket type is set:%s", tsSocketType);
       }

src/client/src/tscUtil.c

@@ -79,7 +79,13 @@ bool tscQueryOnSTable(SSqlCmd* pCmd) {
 
 bool tscQueryTags(SQueryInfo* pQueryInfo) {
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId;
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    int32_t functId = pExpr->functionId;
+
+    // "select count(tbname)" query
+    if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
+      continue;
+    }
 
     if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG) {
       return false;
@@ -157,7 +163,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   }
   
   // for select query super table, the super table vgroup list can not be null in any cases.
-  if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     assert(pTableMetaInfo->vgroupList != NULL);
   }
   
@@ -172,7 +178,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
 
   if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) &&
       pQueryInfo->command == TSDB_SQL_SELECT) {
-    return UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
+    return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
   }
 
   return false;
@@ -183,20 +189,15 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   
   /*
    * In following cases, return false for non ordered project query on super table
-   * 1. failed to get metermeta from server; 2. not a super table; 3. limitation is 0;
+   * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0;
    * 4. show queries, instead of a select query
    */
   size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
-  if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) ||
+  if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) ||
       pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) {
     return false;
   }
   
-  // only query on tag, a project query
-  if (tscQueryTags(pQueryInfo)) {
-    return true;
-  }
-  
   for (int32_t i = 0; i < numOfExprs; ++i) {
     int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
     if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG &&
@@ -208,6 +209,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   return true;
 }
 
+// not order by timestamp projection query on super table
 bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
     return false;
@@ -280,8 +282,8 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
     return;
   }
 
-  pQueryInfo->interpoType = TSDB_INTERPO_NONE;
+  pQueryInfo->fillType = TSDB_FILL_NONE;
-  tfree(pQueryInfo->defaultVal);
+  tfree(pQueryInfo->fillVal);
 }
 
 int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@@ -299,7 +301,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
       tfree(pRes->buffer);
       tfree(pRes->length);
     
-      pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+      pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
       return pRes->code;
     }
   }
@@ -386,14 +388,16 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
 
   int32_t cmd = pCmd->command;
   if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      cmd == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     tscRemoveFromSqlList(pSql);
   }
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
+  if (pObj->signature == pObj) {
     pthread_mutex_lock(&pObj->mutex);
     tfree(pSql->sqlstr);
     pthread_mutex_unlock(&pObj->mutex);
+  }
   
   tscFreeSqlResult(pSql);
   
@@ -402,12 +406,12 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   pSql->numOfSubs = 0;
   
   tscResetSqlCmdObj(pCmd);
-  
-  tscTrace("%p partially free sqlObj completed", pSql);
 }
 
 void tscFreeSqlObj(SSqlObj* pSql) {
-  if (pSql == NULL || pSql->signature != pSql) return;
+  if (pSql == NULL || pSql->signature != pSql) {
+    return;
+  }
   
   tscTrace("%p start to free sql object", pSql);
   tscPartiallyFreeSqlObj(pSql);
@@ -419,8 +423,10 @@ void tscFreeSqlObj(SSqlObj* pSql) {
 
   memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
-
   pCmd->allocSize = 0;
+  
+  tfree(pSql->sqlstr);
+  sem_destroy(&pSql->rspSem);
   free(pSql);
 }
 
@@ -570,7 +576,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
   STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks));
   if (dataBuf == NULL) {
     tscError("failed to allocated memory, reason:%s", strerror(errno));
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
   dataBuf->nAllocSize = (uint32_t)initialSize;
@@ -587,7 +593,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
   dataBuf->size = startOffset;
   dataBuf->tsSource = -1;
 
-  strncpy(dataBuf->tableId, name, TSDB_TABLE_ID_LEN);
+  tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId));
 
   /*
    * The table meta may be released since the table meta cache are completed clean by other thread
@@ -648,6 +654,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
   for (int32_t i = 0; i < numOfRows; ++i) {
     SDataRow trow = (SDataRow)pDataBlock;
     dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen);
+    dataRowSetVersion(trow, pTableMeta->sversion);
 
     int toffset = 0;
     for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
@@ -704,7 +711,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
         tscDestroyBlockArrayList(pVnodeDataBlockList);
         tfree(dataBuf->pData);
 
-        return TSDB_CODE_CLI_OUT_OF_MEMORY;
+        return TSDB_CODE_TSC_OUT_OF_MEMORY;
       }
     }
 
@@ -746,27 +753,14 @@ void tscCloseTscObj(STscObj* pObj) {
   assert(pObj != NULL);
   
   pObj->signature = NULL;
-  SSqlObj* pSql = pObj->pSql;
-  
-  if (pSql) {
-    terrno = pSql->res.code;
-    sem_destroy(&pSql->rspSem);
-  }
-  
   taosTmrStopA(&(pObj->pTimer));
-  tscFreeSqlObj(pSql);
-
-  if (pSql) {
-    sem_destroy(&pSql->rspSem);
-  }
-  
   pthread_mutex_destroy(&pObj->mutex);
   
   if (pObj->pDnodeConn != NULL) {
     rpcClose(pObj->pDnodeConn);
   }
   
-  tscTrace("%p DB connection is closed", pObj);
+  tscTrace("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
   tfree(pObj);
 }
 
@@ -788,12 +782,12 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) {
     assert(pCmd->allocSize == 0);
 
     pCmd->payload = (char*)calloc(1, size);
-    if (pCmd->payload == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    if (pCmd->payload == NULL) return TSDB_CODE_TSC_OUT_OF_MEMORY;
     pCmd->allocSize = size;
   } else {
     if (pCmd->allocSize < size) {
       char* b = realloc(pCmd->payload, size);
-      if (b == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY;
+      if (b == NULL) return TSDB_CODE_TSC_OUT_OF_MEMORY;
       pCmd->payload = b;
       pCmd->allocSize = size;
     }
@@ -807,7 +801,7 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) {
 
 TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
   TAOS_FIELD f = { .type = type, .bytes = bytes, };
-  strncpy(f.name, name, TSDB_COL_NAME_LEN);
+  tstrncpy(f.name, name, sizeof(f.name));
   return f;
 }
 
@@ -972,11 +966,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
     if (isTagCol) {
       SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
       pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
-      strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN);
+      tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name));
-    } else {
+    } else if (pTableMetaInfo->pTableMeta != NULL) {
+      // in handling select database/version/server_status(), the pTableMeta is NULL
       SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
       pExpr->colInfo.colId = pSchema->colId;
-      strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN);
+      tstrncpy(pExpr->colInfo.name, pSchema->name, sizeof(pExpr->colInfo.name));
     }
   }
   
@@ -986,7 +981,10 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
   pExpr->resType       = type;
   pExpr->resBytes      = size;
   pExpr->interBytes    = interSize;
+  
+  if (pTableMetaInfo->pTableMeta) {
     pExpr->uid = pTableMetaInfo->pTableMeta->uid;
+  }
   
   return pExpr;
 }
@@ -1029,7 +1027,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
   return pExpr;
 }
 
-int32_t  tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
+size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
   return taosArrayGetSize(pQueryInfo->exprList);
 }
 
@@ -1144,23 +1142,21 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
 }
 
 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) {
-  SColumnFilterInfo* pFilter = NULL;
+  if (numOfFilters == 0) {
-  if (numOfFilters > 0) {
-    pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo));
-  } else {
     assert(src == NULL);
     return NULL;
   }
   
+  SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo));
+  
   memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters);
   for (int32_t j = 0; j < numOfFilters; ++j) {
+    
     if (pFilter[j].filterstr) {
-      size_t len = (size_t) pFilter[j].len + 1;
+      size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE;
+      pFilter[j].pz = (int64_t) calloc(1, len);
       
-      char*  pTmp   = calloc(1, len);
+      memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len);
-      pFilter[j].pz = (int64_t) pTmp;
-      
-      memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len);
     }
   }
   
@@ -1211,18 +1207,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) {
   }
 }
 
-void tscColumnListDestroy(SArray* pColumnBaseInfo) {
+void tscColumnListDestroy(SArray* pColumnList) {
-  if (pColumnBaseInfo == NULL) {
+  if (pColumnList == NULL) {
     return;
   }
 
-  size_t num = taosArrayGetSize(pColumnBaseInfo);
+  size_t num = taosArrayGetSize(pColumnList);
   for (int32_t i = 0; i < num; ++i) {
-    SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i);
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
     tscColumnDestroy(pCol);
   }
 
-  taosArrayDestroy(pColumnBaseInfo);
+  taosArrayDestroy(pColumnList);
 }
 
 /*
@@ -1240,9 +1236,8 @@ void tscColumnListDestroy(SArray* pColumnBaseInfo) {
  *
  */
 static int32_t validateQuoteToken(SSQLToken* pToken) {
-  pToken->n = strdequote(pToken->z);
+  strdequote(pToken->z);
-  strtrim(pToken->z);
+  pToken->n = strtrim(pToken->z);
-  pToken->n = (uint32_t)strlen(pToken->z);
 
   int32_t k = tSQLGetToken(pToken->z, &pToken->type);
 
@@ -1251,22 +1246,21 @@ static int32_t validateQuoteToken(SSQLToken* pToken) {
   }
 
   if (k != pToken->n || pToken->type != TK_ID) {
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t tscValidateName(SSQLToken* pToken) {
   if (pToken->type != TK_STRING && pToken->type != TK_ID) {
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
 
   char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true);
   if (sep == NULL) {  // single part
     if (pToken->type == TK_STRING) {
-      pToken->n = strdequote(pToken->z);
+      strdequote(pToken->z);
-      strtrim(pToken->z);
+      pToken->n = strtrim(pToken->z);
-      pToken->n = (uint32_t)strlen(pToken->z);
 
       int len = tSQLGetToken(pToken->z, &pToken->type);
 
@@ -1276,14 +1270,14 @@ int32_t tscValidateName(SSQLToken* pToken) {
       } else {
         sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true);
         if (sep == NULL) {
-          return TSDB_CODE_INVALID_SQL;
+          return TSDB_CODE_TSC_INVALID_SQL;
         }
 
         return tscValidateName(pToken);
       }
     } else {
       if (isNumber(pToken)) {
-        return TSDB_CODE_INVALID_SQL;
+        return TSDB_CODE_TSC_INVALID_SQL;
       }
     }
   } else {  // two part
@@ -1291,21 +1285,20 @@ int32_t tscValidateName(SSQLToken* pToken) {
     char*   pStr = pToken->z;
 
     if (pToken->type == TK_SPACE) {
-      strtrim(pToken->z);
+      pToken->n = strtrim(pToken->z);
-      pToken->n = (uint32_t)strlen(pToken->z);
     }
 
     pToken->n = tSQLGetToken(pToken->z, &pToken->type);
     if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     if (pToken->type != TK_STRING && pToken->type != TK_ID) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     int32_t firstPartLen = pToken->n;
@@ -1314,11 +1307,11 @@ int32_t tscValidateName(SSQLToken* pToken) {
     pToken->n = oldLen - (sep - pStr) - 1;
     int32_t len = tSQLGetToken(pToken->z, &pToken->type);
     if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_INVALID_SQL;
+      return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     // re-build the whole name string
@@ -1350,7 +1343,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) {
     return false;
   }
 
-  if (colId == -1 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (colId == TSDB_TBNAME_COLUMN_INDEX) {
     return true;
   }
 
@@ -1461,10 +1454,11 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) {
 }
 
 /*
- * the following three kinds of SqlObj should not be freed
+ * the following four kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
  * 2. main SqlObj
  * 3. heartbeat SqlObj
+ * 4. SqlObj for subscription
  *
  * If res code is error and SqlObj does not belong to above types, it should be
  * automatically freed for async query, ignoring that connection should be kept.
@@ -1472,22 +1466,24 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) {
  * If connection need to be recycled, the SqlObj also should be freed.
  */
 bool tscShouldBeFreed(SSqlObj* pSql) {
-  if (pSql == NULL || pSql->signature != pSql || pSql->fp == NULL) {
+  if (pSql == NULL || pSql->signature != pSql) {
     return false;
   }
   
+  assert(pSql->fp != NULL);
+  
   STscObj* pTscObj = pSql->pTscObj;
-  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql) {
+  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
     return false;
   }
 
+  // only the table meta and super table vgroup query will free resource automatically
   int32_t command = pSql->cmd.command;
-  if (command == TSDB_SQL_CONNECT || command == TSDB_SQL_INSERT) {
+  if (command == TSDB_SQL_META || command == TSDB_SQL_STABLEVGROUP) {
     return true;
-  } else {
-    return tscKeepConn[command] == 0 ||
-           (pSql->res.code != TSDB_CODE_ACTION_IN_PROGRESS && pSql->res.code != TSDB_CODE_SUCCESS);
   }
+
+  return false;
 }
 
 /**
@@ -1584,7 +1580,7 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
   size_t s = pCmd->numOfClause + 1;
   char*  tmp = realloc(pCmd->pQueryInfo, s * POINTER_BYTES);
   if (tmp == NULL) {
-    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
   pCmd->pQueryInfo = (SQueryInfo**)tmp;
@@ -1615,7 +1611,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
   
   pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf);
 
-  tfree(pQueryInfo->defaultVal);
+  tfree(pQueryInfo->fillVal);
 }
 
 void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@@ -1646,9 +1642,11 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro
 void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
   tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
   
-  int32_t index = pQueryInfo->numOfTables;
+  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-  while (index >= 0) {
+    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
-    doRemoveTableMetaInfo(pQueryInfo, --index, removeFromCache);
+  
+    tscClearTableMetaInfo(pTableMetaInfo, removeFromCache);
+    free(pTableMetaInfo);
   }
   
   tfree(pQueryInfo->pTableMetaInfo);
@@ -1668,25 +1666,20 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   assert(pTableMetaInfo != NULL);
 
   if (name != NULL) {
-    assert(strlen(name) <= TSDB_TABLE_ID_LEN);
+    tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name));
-    strcpy(pTableMetaInfo->name, name);
   }
 
   pTableMetaInfo->pTableMeta = pTableMeta;
   
   if (vgroupList != NULL) {
-    assert(vgroupList->numOfVgroups == 1);  // todo fix me
-    
     size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
-    
     pTableMetaInfo->vgroupList = malloc(size);
     memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
   }
 
-  if (pTagCols == NULL) {
   pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
-  } else {
+  if (pTagCols != NULL) {
-    pTableMetaInfo->tagColList = taosArrayClone(pTagCols);
+    tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
   }
   
   pQueryInfo->numOfTables += 1;
@@ -1705,11 +1698,9 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
   taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
   tfree(pTableMetaInfo->vgroupList);
   
-  if (pTableMetaInfo->tagColList != NULL) {
+  tscColumnListDestroy(pTableMetaInfo->tagColList);
-    taosArrayDestroy(pTableMetaInfo->tagColList);
   pTableMetaInfo->tagColList = NULL;
 }
-}
 
 void tscResetForNextRetrieve(SSqlRes* pRes) {
   if (pRes == NULL) {
@@ -1770,11 +1761,12 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pNewQueryInfo->limit  = pQueryInfo->limit;
   pNewQueryInfo->slimit = pQueryInfo->slimit;
   pNewQueryInfo->order  = pQueryInfo->order;
-  pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit;
-  pNewQueryInfo->pTableMetaInfo = NULL;
-  pNewQueryInfo->defaultVal  = NULL;
-  pNewQueryInfo->numOfTables = 0;
   pNewQueryInfo->tsBuf  = NULL;
+  pNewQueryInfo->fillType = pQueryInfo->fillType;
+  pNewQueryInfo->fillVal  = NULL;
+  pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit;
+  pNewQueryInfo->numOfTables = 0;
+  pNewQueryInfo->pTableMetaInfo = NULL;
   
   pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr;
   if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
@@ -1783,9 +1775,9 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   
   tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond);
 
-  if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-    pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
+    pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
-    memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
+    memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
   }
 
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {
@@ -1846,13 +1838,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   pNew->fp = fp;
   pNew->param = param;
+  pNew->maxRetry = TSDB_MAX_REPLICA_NUM;
 
   char* name = pTableMetaInfo->name;
   STableMetaInfo* pFinalInfo = NULL;
 
   if (pPrevSql == NULL) {
-    STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name);
+    STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
-
+    assert(pTableMeta != NULL);
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.
     STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
@@ -1864,8 +1857,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
   }
 
-  assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1);
+  if (pFinalInfo->pTableMeta == NULL) {
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    tscError("%p new subquery failed for get tableMeta is NULL from cache", pSql);
+    tscFreeSqlObj(pNew);
+    return NULL;
+  }
+  
+  assert(pNewQueryInfo->numOfTables == 1);
+  
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     assert(pFinalInfo->vgroupList != NULL);
   }
   
@@ -1946,15 +1946,14 @@ int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid) {
   }
 }
 
-bool tscIsUpdateQuery(STscObj* pObj) {
+bool tscIsUpdateQuery(SSqlObj* pSql) {
-  if (pObj == NULL || pObj->signature != pObj) {
+  if (pSql == NULL || pSql->signature != pSql) {
-    terrno = TSDB_CODE_DISCONNECTED;
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return TSDB_CODE_DISCONNECTED;
+    return TSDB_CODE_TSC_DISCONNECTED;
   }
 
-  SSqlCmd* pCmd = &pObj->pSql->cmd;
+  SSqlCmd* pCmd = &pSql->cmd;
-  return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) ||
+  return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command);
-          TSDB_SQL_USE_DB == pCmd->command);
 }
 
 int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) {
@@ -1967,7 +1966,7 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s
   if (sql == NULL) {
     assert(additionalInfo != NULL);
     sprintf(msg, msgFormat1, additionalInfo);
-    return TSDB_CODE_INVALID_SQL;
+    return TSDB_CODE_TSC_INVALID_SQL;
   }
 
   char buf[64] = {0};  // only extract part of sql string
@@ -1979,34 +1978,40 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s
     sprintf(msg, msgFormat3, buf);  // no additional information for invalid sql error
   }
 
-  return TSDB_CODE_INVALID_SQL;
+  return TSDB_CODE_TSC_INVALID_SQL;
 }
 
 bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
   assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0);
-  return (pQueryInfo->clauseLimit > 0 && pRes->numOfTotalInCurrentClause >= pQueryInfo->clauseLimit);
+  return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
 }
 
 char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
 
 /**
  *  If current vnode query does not return results anymore (pRes->numOfRows == 0), try the next vnode if exists,
- *  in case of multi-vnode super table projection query and the result does not reach the limitation.
+ *  while multi-vnode super table projection query and the result does not reach the limitation.
  */
 bool hasMoreVnodesToTry(SSqlObj* pSql) {
-//  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
+  SSqlRes* pRes = &pSql->res;
-
+  if (pCmd->command != TSDB_SQL_FETCH) {
-//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  
-//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-//  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
     return false;
-//  }
+  }
 
-//  int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-//  return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
+  
-//         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < totalVnode - 1);
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  assert(pRes->completed);
+  
+  // for normal table, no need to try any more if results are all retrieved from one vnode
+  if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) {
+    return false;
+  }
+  
+  int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+  return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
+         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1);
 }
 
 void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
@@ -2022,12 +2027,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
   assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes));
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  int32_t totalVnode = 0;
-//  int32_t         totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
   
-  while (++pTableMetaInfo->vgroupIndex < totalVnode) {
+  int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
-    tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
+  while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
-             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotalInCurrentClause);
+    tscTrace("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%d", pSql,
+             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
 
     /*
      * update the limit and offset value for the query on the next vnode,
@@ -2035,18 +2039,18 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
      *
      * NOTE:
      * if the pRes->offset is larger than 0, the start returned position has not reached yet.
-     * Therefore, the pRes->numOfRows, as well as pRes->numOfTotalInCurrentClause, must be 0.
+     * Therefore, the pRes->numOfRows, as well as pRes->numOfClauseTotal, must be 0.
      * The pRes->offset value will be updated by virtual node, during query execution.
      */
     if (pQueryInfo->clauseLimit >= 0) {
-      pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfTotalInCurrentClause;
+      pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfClauseTotal;
     }
 
     pQueryInfo->limit.offset = pRes->offset;
-
     assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0));
-    tscTrace("%p new query to next vnode, vnode index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, pSql,
+    
-             pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
+    tscTrace("%p new query to next vgroup, index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64,
+        pSql, pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
 
     /*
      * For project query with super table join, the numOfSub is equalled to the number of all subqueries.
@@ -2059,43 +2063,13 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
 
     tscResetForNextRetrieve(pRes);
 
-    // in case of async query, set the callback function
+    // set the callback function
-    void* fp1 = pSql->fp;
     pSql->fp = fp;
-
+    int32_t ret = tscProcessSql(pSql);
-    if (fp1 != NULL) {
+    if (ret == TSDB_CODE_SUCCESS) {
-      assert(fp != NULL);
-    }
-
-    int32_t ret = tscProcessSql(pSql);  // todo check for failure
-
-    // in case of async query, return now
-    if (fp != NULL) {
       return;
+    } else {// todo check for failure
     }
-
-    if (ret != TSDB_CODE_SUCCESS) {
-      pSql->res.code = ret;
-      return;
-    }
-
-    // retrieve data
-    assert(pCmd->command == TSDB_SQL_SELECT);
-    pCmd->command = TSDB_SQL_FETCH;
-
-    if ((ret = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) {
-      pSql->res.code = ret;
-      return;
-    }
-
-    // if the result from current virtual node are empty, try next if exists. otherwise, return the results.
-    if (pRes->numOfRows > 0) {
-      break;
-    }
-  }
-
-  if (pRes->numOfRows == 0) {
-    tscTrace("%p all vnodes exhausted, prj query completed. total res:%d", pSql, totalVnode, pRes->numOfTotal);
   }
 }
 
@@ -2112,7 +2086,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
   pSql->cmd.command = pQueryInfo->command;
 
   //backup the total number of result first
-  int64_t num = pRes->numOfTotal + pRes->numOfTotalInCurrentClause;
+  int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal;
   tscFreeSqlResult(pSql);
   
   pRes->numOfTotal = num;
@@ -2134,29 +2108,92 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
 }
 
 void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
-  SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, columnIndex);
+  SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex);//tscFieldInfoGetSupp(pFieldInfo, columnIndex);
   assert(pInfo->pSqlExpr != NULL);
 
   int32_t type = pInfo->pSqlExpr->resType;
   int32_t bytes = pInfo->pSqlExpr->resBytes;
   
-  char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
+  char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
   
   if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
     int32_t realLen = varDataLen(pData);
     assert(realLen <= bytes - VARSTR_HEADER_SIZE);
     
-    if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
+    if (isNull(pData, type)) {
-      *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0;
+      pRes->tsrow[columnIndex] = NULL;
+    } else {
+      pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
+    }
+  
+    if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
+      *(pData + realLen + VARSTR_HEADER_SIZE) = 0;
     }
     
-    pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
     pRes->length[columnIndex] = realLen;
   } else {
     assert(bytes == tDataTypeDesc[type].nSize);
     
+    if (isNull(pData, type)) {
+      pRes->tsrow[columnIndex] = NULL;
+    } else {
       pRes->tsrow[columnIndex] = pData;
+    }
+    
     pRes->length[columnIndex] = bytes;
   }
 }
 
+void* malloc_throw(size_t size) {
+  void* p = malloc(size);
+  if (p == NULL) {
+    THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  return p;
+}
+
+void* calloc_throw(size_t nmemb, size_t size) {
+  void* p = calloc(nmemb, size);
+  if (p == NULL) {
+    THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  return p;
+}
+
+char* strdup_throw(const char* str) {
+  char* p = strdup(str);
+  if (p == NULL) {
+    THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  return p;
+}
+
+int tscSetMgmtIpListFromCfg(const char *first, const char *second) {
+  tscMgmtIpSet.numOfIps = 0;
+  tscMgmtIpSet.inUse = 0;
+
+  if (first && first[0] != 0) {
+    if (strlen(first) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return -1;
+    }
+    taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+    tscMgmtIpSet.numOfIps++;
+  }
+
+  if (second && second[0] != 0) {
+    if (strlen(second) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return -1;
+    }
+    taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+    tscMgmtIpSet.numOfIps++;
+  }
+
+  if ( tscMgmtIpSet.numOfIps == 0) {
+    terrno = TSDB_CODE_TSC_INVALID_FQDN;
+    return -1;
+  }
+
+  return 0;
+}

src/common/inc/qsqltype.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_QSQLCMD_H
+#define TDENGINE_QSQLCMD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// sql type
+
+#ifdef TSDB_SQL_C
+#define TSDB_DEFINE_SQL_TYPE( name, msg ) msg, 
+char *sqlCmd[] = {
+  "null",
+#else
+#define TSDB_DEFINE_SQL_TYPE( name, msg ) name,
+enum {
+  TSDB_SQL_NULL = 0,
+#endif
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" )
+  
+  // the SQL below is for mgmt node
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" ) 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_USER, "drop-user" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_USER, "alter-user" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_ACCT, "alter-acct" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DNODE, "drop-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" )
+
+  // SQL below is for read operation
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_READ, "read" )  
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CONNECT, "connect" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_USE_DB, "use-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_META, "meta" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" )
+
+  // SQL below for client local 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" ) 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DESCRIBE_TABLE, "describe-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_LOCALMERGE, "retrieve-localmerge" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_TABLE_JOIN_RETRIEVE, "join-retrieve" )
+
+  /*
+   * build empty result instead of accessing dnode to fetch result
+   * reset the client cache
+   */
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_EMPTY_RESULT, "retrieve-empty-result" )
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RESET_CACHE, "reset-cache" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_STATUS, "serv-status" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_DB, "current-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_VERSION, "serv-version" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CLI_VERSION, "cli-version" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_USER, "current-user ")
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_LOCAL, "cfg-local" )
+
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" )
+};
+
+// create table operation type
+enum TSQL_TYPE {
+  TSQL_CREATE_TABLE = 0x1,
+  TSQL_CREATE_STABLE = 0x2,
+  TSQL_CREATE_TABLE_FROM_STABLE = 0x3,
+  TSQL_CREATE_STREAM = 0x4,
+};
+
+extern char *sqlCmd[];
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_QSQLCMD_H

src/common/inc/tdataformat.h

@@ -19,6 +19,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "talgo.h"
 #include "taosdef.h"
 #include "tutil.h"
 
@@ -26,18 +27,23 @@
 extern "C" {
 #endif
 
-#define STR_TO_VARSTR(x, str) do {VarDataLenT __len = strlen(str); \
+#define STR_TO_VARSTR(x, str)             \
+  do {                                    \
+    VarDataLenT __len = strlen(str);      \
     *(VarDataLenT *)(x) = __len;          \
-  strncpy(varDataVal(x), (str), __len);} while(0);
+    memcpy(varDataVal(x), (str), __len); \
+  } while (0);
 
-#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\
+#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)      \
+  do {                                                 \
     char *_e = stpncpy(varDataVal(x), (str), (_maxs)); \
     varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE));   \
   } while (0)
 
-#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\
+#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
+  do {                                         \
     *(VarDataLenT *)(x) = (_size);             \
-  strncpy(varDataVal(x), (str), (_size));\
+    memcpy(varDataVal(x), (str), (_size));    \
   } while (0);
 
 // ----------------- TSDB COLUMN DEFINITION
@@ -60,65 +66,117 @@ typedef struct {
 
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
-  int      totalCols;  // Total columns allocated
+  int      version;    // version
   int      numOfCols;  // Number of columns appended
   int      tlen;       // maximum length of a SDataRow without the header part
-  int      flen;       // First part length in a SDataRow after the header part
+  int16_t  flen;       // First part length in a SDataRow after the header part
+  int16_t  vlen;       // pure value part length, excluded the overhead
   STColumn columns[];
 } STSchema;
 
 #define schemaNCols(s) ((s)->numOfCols)
-#define schemaTotalCols(s) ((s)->totalCols)
+#define schemaVersion(s) ((s)->version)
 #define schemaTLen(s) ((s)->tlen)
 #define schemaFLen(s) ((s)->flen)
+#define schemaVLen(s) ((s)->vlen)
 #define schemaColAt(s, i) ((s)->columns + i)
-
-STSchema *tdNewSchema(int32_t nCols);
 #define tdFreeSchema(s) tfree((s))
-int       tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
+
 STSchema *tdDupSchema(STSchema *pSchema);
-int       tdGetSchemaEncodeSize(STSchema *pSchema);
 void *    tdEncodeSchema(void *dst, STSchema *pSchema);
 STSchema *tdDecodeSchema(void **psrc);
 
+static FORCE_INLINE int comparColId(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
+    return 1;
+  } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
+    return -1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
+  void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId);
+  if (ptr == NULL) return NULL;
+  return (STColumn *)ptr;
+}
+
+// ----------------- SCHEMA BUILDER DEFINITION
+typedef struct {
+  int       tCols;
+  int       nCols;
+  int       tlen;
+  int16_t   flen;
+  int16_t   vlen;
+  int       version;
+  STColumn *columns;
+} STSchemaBuilder;
+
+int       tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
+void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
+void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
+int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int32_t bytes);
+STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
+
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * |<------------------------------------- len ---------------------------------->|
+ * |<--------------------+--------------------------- len ---------------------------------->|
- * |<--Head ->|<---------   flen -------------->|                                 |
+ * |<--     Head      -->|<---------   flen -------------->|                                 |
- * +----------+---------------------------------+---------------------------------+
+ * +---------------------+---------------------------------+---------------------------------+
- * | int32_t  |                                 |                                 |
+ * | int16_t  |  int16_t |                                 |                                 |
- * +----------+---------------------------------+---------------------------------+
+ * +----------+----------+---------------------------------+---------------------------------+
- * |   len    |           First part            |             Second part         |
+ * |   len    | sversion |           First part            |             Second part         |
- * +----------+---------------------------------+---------------------------------+
+ * +----------+----------+---------------------------------+---------------------------------+
  */
 typedef void *SDataRow;
 
-#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
+#define TD_DATA_ROW_HEAD_SIZE sizeof(int16_t)*2
 
-#define dataRowLen(r) (*(int32_t *)(r))
+#define dataRowLen(r) (*(int16_t *)(r))
+#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
+#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
 #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
 
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
 void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-int      tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
 SDataRow tdDataRowDup(SDataRow row);
 
+// offset here not include dataRow header length
+static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
+  ASSERT(value != NULL);
+  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
+  char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
+
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+      memcpy(ptr, value, varDataTLen(value));
+      dataRowLen(row) += varDataTLen(value);
+      break;
+    default:
+      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
+      break;
+  }
+
+  return 0;
+}
+
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
   switch (type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-      break;
     default:
       return POINTER_SHIFT(row, offset);
-      break;
   }
 }
 
@@ -137,8 +195,8 @@ typedef struct SDataCol {
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
@@ -171,14 +229,13 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
   }
 }
 
-
 typedef struct {
   int maxRowSize;
   int maxCols;    // max number of columns
   int maxPoints;  // max number of points
   int bufSize;
 
-  int      numOfPoints;
+  int      numOfRows;
   int      numOfCols;  // Total number of cols
   int      sversion;   // TODO: set sversion
   void *   buf;
@@ -188,17 +245,111 @@ typedef struct {
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
 #define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
 #define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
-#define dataColsKeyLast(pCols) ((pCols->numOfPoints == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfPoints - 1))
+#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 void       tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 void       tdFreeDataCols(SDataCols *pCols);
-void       tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
+void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
+void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows);
+
+// ----------------- K-V data row structure
+/*
+ * +----------+----------+---------------------------------+---------------------------------+
+ * |  int16_t |  int16_t |                                 |                                 |
+ * +----------+----------+---------------------------------+---------------------------------+
+ * |    len   |   ncols  |           cols index            |             data part           |
+ * +----------+----------+---------------------------------+---------------------------------+
+ */
+typedef void *SKVRow;
+
+typedef struct {
+  int16_t colId;
+  int16_t offset;
+} SColIdx;
+
+#define TD_KV_ROW_HEAD_SIZE 2 * sizeof(int16_t)
+
+#define kvRowLen(r) (*(int16_t *)(r))
+#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
+#define kvRowSetLen(r, len) kvRowLen(r) = (len)
+#define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
+#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
+#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r))
+#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
+#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
+#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
+#define kvRowFree(r) tfree(r)
+#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
+
+SKVRow tdKVRowDup(SKVRow row);
+int    tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
+void * tdEncodeKVRow(void *buf, SKVRow row);
+void * tdDecodeKVRow(void *buf, SKVRow *row);
+
+static FORCE_INLINE int comparTagId(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
+    return 1;
+  } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
+    return -1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) {
+  void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
+  if (ret == NULL) return NULL;
+  return kvRowColVal(row, (SColIdx *)ret);
+}
+
+// ----------------- K-V data row builder
+typedef struct {
+  int16_t  tCols;
+  int16_t  nCols;
+  SColIdx *pColIdx;
+  int16_t  alloc;
+  int16_t  size;
+  void *   buf;
+} SKVRowBuilder;
+
+int    tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
+void   tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
+void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
+SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
+
+static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
+  ASSERT(pBuilder->nCols == 0 || colId > pBuilder->pColIdx[pBuilder->nCols - 1].colId);
+
+  if (pBuilder->nCols >= pBuilder->tCols) {
+    pBuilder->tCols *= 2;
+    pBuilder->pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
+    if (pBuilder->pColIdx == NULL) return -1;
+  }
+
+  pBuilder->pColIdx[pBuilder->nCols].colId = colId;
+  pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size;
+
+  pBuilder->nCols++;
+
+  int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
+  if (tlen > pBuilder->alloc - pBuilder->size) {
+    while (tlen > pBuilder->alloc - pBuilder->size) {
+      pBuilder->alloc *= 2;
+    }
+    pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc);
+    if (pBuilder->buf == NULL) return -1;
+  }
+
+  memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen);
+  pBuilder->size += tlen;
+
+  return 0;
+}
 
 #ifdef __cplusplus
 }

src/common/inc/tglobal.h

@@ -49,7 +49,7 @@ extern int32_t tsTotalMemoryMB;
 extern int32_t tsVersion;
 
 extern int32_t tscEmbedded;
-extern int64_t tsMsPerDay[2];
+extern int64_t tsMsPerDay[3];
 
 extern char  tsFirst[];
 extern char  tsSecond[];
@@ -87,22 +87,25 @@ extern int16_t tsWAL;
 extern int32_t tsReplications;
 
 extern int16_t tsAffectedRowsMod;
-extern int32_t tsNumOfMPeers;
+extern int32_t tsNumOfMnodes;
 extern int32_t tsMaxShellConns;
 extern int32_t tsMaxTables;
 
 extern char tsDefaultDB[];
 extern char tsDefaultUser[];
 extern char tsDefaultPass[];
-extern int32_t tsMaxMeterConnections;
+
-extern int32_t tsMaxVnodeConnections;
+extern char tsMqttBrokerAddress[];
-extern int32_t tsMaxMgmtConnections;
+extern char tsMqttBrokerClientId[];
+
+extern int32_t tsMaxConnections;
 
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
-extern int32_t tsMgmtEqualVnodeNum;
+extern int32_t tsMnodeEqualVnodeNum;
 
 extern int32_t tsEnableHttpModule;
+extern int32_t tsEnableMqttModule;
 extern int32_t tsEnableMonitorModule;
 
 extern int32_t tsRestRowLimit;
@@ -141,18 +144,19 @@ extern int32_t tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t dDebugFlag;
 extern int32_t vDebugFlag;
-extern int32_t mdebugFlag;
+extern int32_t mDebugFlag;
-extern int32_t cdebugFlag;
+extern int32_t cDebugFlag;
-extern int32_t jnidebugFlag;
+extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
+extern int32_t mqttDebugFlag;
 extern int32_t monitorDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t debugFlag;
-extern int32_t odbcdebugFlag;
+extern int32_t odbcDebugFlag;
-extern int32_t qdebugFlag;
+extern int32_t qDebugFlag;
 
 extern uint32_t taosMaxTmrCtrl;
 
@@ -175,7 +179,7 @@ void taosInitGlobalCfg();
 bool taosCheckGlobalCfg();
 void taosSetAllDebugFlag();
 bool taosCfgDynamicOptions(char *msg);
-int  taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port);
+int  taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
  
 #ifdef __cplusplus
 }

src/common/inc/tname.h

@@ -23,4 +23,5 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
+
 #endif  // TDENGINE_NAME_H

src/common/inc/tulog.h

@@ -25,31 +25,15 @@ extern "C" {
 extern int32_t uDebugFlag;
 extern int32_t tscEmbedded;
 
-#define uError(...)                                      \
+#define uError(...)       { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR UTL ", uDebugFlag, __VA_ARGS__); }}
-  if (uDebugFlag & DEBUG_ERROR) {                        \
+#define uWarn(...)        { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN UTL ", uDebugFlag, __VA_ARGS__); }}
-    taosPrintLog("ERROR UTL ", uDebugFlag, __VA_ARGS__); \
+#define uTrace(...)       { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
-  }
+#define uDump(x, y)       { if (uDebugFlag & DEBUG_DUMP)  { taosDumpData(x, y); }}
-#define uWarn(...)                                       \
+#define uPrint(...)       { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
-  if (uDebugFlag & DEBUG_WARN) {                         \
+#define uForcePrint(...)  { taosPrintLog("ERROR UTL ", 255, __VA_ARGS__); }
-    taosPrintLog("WARN  UTL ", uDebugFlag, __VA_ARGS__); \
-  }
-#define uTrace(...)                                \
-  if (uDebugFlag & DEBUG_TRACE) {                  \
-    taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); \
-  }
-#define uDump(x, y)              \
-  if (uDebugFlag & DEBUG_DUMP) { \
-    taosDumpData(x, y);          \
-  }
-#define uPrint(...) \
-  { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
-#define uForcePrint(...) \
-  { taosPrintLog("ERROR UTL ", 255, __VA_ARGS__); }
 
-#define pError(...) \
+#define pError(...) { taosPrintLog("ERROR APP ", 255, __VA_ARGS__); }  
-  { taosPrintLog("ERROR APP ", 255, __VA_ARGS__); }  
+#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }  
-#define pPrint(...) \
-  { taosPrintLog("APP ", 255, __VA_ARGS__); }  
 
 #ifdef __cplusplus
 }

src/common/src/sqlcmdstr.c

@@ -13,14 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _sdb_int_hash_header_
+#define TSDB_SQL_C
-#define _sdb_int_hash_header_
 
-void *sdbOpenIntHash(int maxSessions, int dataSize);
+#include "qsqltype.h"
-void sdbCloseIntHash(void *handle);
-void *sdbAddIntHash(void *handle, void *key, void *pData);
-void sdbDeleteIntHash(void *handle, void *key);
-void *sdbGetIntHashData(void *handle, void *key);
-void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta);
-
-#endif

src/common/src/tdataformat.c

@@ -13,95 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "tdataformat.h"
+#include "talgo.h"
 #include "wchar.h"
 
-/**
- * Create a SSchema object with nCols columns
- * ASSUMPTIONS: VALID PARAMETERS
- *
- * @param nCols number of columns the schema has
- *
- * @return a STSchema object for success
- *         NULL for failure
- */
-STSchema *tdNewSchema(int32_t nCols) {
-  int32_t size = sizeof(STSchema) + sizeof(STColumn) * nCols;
-
-  STSchema *pSchema = (STSchema *)calloc(1, size);
-  if (pSchema == NULL) return NULL;
-
-  pSchema->numOfCols = 0;
-  pSchema->totalCols = nCols;
-  pSchema->flen = 0;
-  pSchema->tlen = 0;
-
-  return pSchema;
-}
-
-/**
- * Append a column to the schema
- */
-int tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes) {
-  if (!isValidDataType(type, 0) || pSchema->numOfCols >= pSchema->totalCols) return -1;
-
-  STColumn *pCol = schemaColAt(pSchema, schemaNCols(pSchema));
-  colSetType(pCol, type);
-  colSetColId(pCol, colId);
-  if (schemaNCols(pSchema) == 0) {
-    colSetOffset(pCol, 0);
-  } else {
-    STColumn *pTCol = schemaColAt(pSchema, schemaNCols(pSchema)-1);
-    colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
-  }
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      colSetBytes(pCol, bytes); // Set as maximum bytes
-      pSchema->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes);
-      break;
-    default:
-      colSetBytes(pCol, TYPE_BYTES[type]);
-      pSchema->tlen += TYPE_BYTES[type];
-      break;
-  }
-
-  pSchema->numOfCols++;
-  pSchema->flen += TYPE_BYTES[type];
-
-  ASSERT(pCol->offset < pSchema->flen);
-
-  return 0;
-}
-
 /**
  * Duplicate the schema and return a new object
  */
 STSchema *tdDupSchema(STSchema *pSchema) {
-  STSchema *tSchema = tdNewSchema(schemaNCols(pSchema));
+
+  int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema);
+  STSchema *tSchema = (STSchema *)malloc(tlen);
   if (tSchema == NULL) return NULL;
 
-  int32_t size = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema);
+  memcpy((void *)tSchema, (void *)pSchema, tlen);
-  memcpy((void *)tSchema, (void *)pSchema, size);
 
   return tSchema;
 }
 
-/**
- * Return the size of encoded schema
- */
-int tdGetSchemaEncodeSize(STSchema *pSchema) {
-  return T_MEMBER_SIZE(STSchema, totalCols) +
-         schemaNCols(pSchema) *
-             (T_MEMBER_SIZE(STColumn, type) + T_MEMBER_SIZE(STColumn, colId) + T_MEMBER_SIZE(STColumn, bytes));
-}
-
 /**
  * Encode a schema to dst, and return the next pointer
  */
 void *tdEncodeSchema(void *dst, STSchema *pSchema) {
-  ASSERT(pSchema->numOfCols == pSchema->totalCols);
 
-  T_APPEND_MEMBER(dst, pSchema, STSchema, totalCols);
+  T_APPEND_MEMBER(dst, pSchema, STSchema, version);
+  T_APPEND_MEMBER(dst, pSchema, STSchema, numOfCols);
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     T_APPEND_MEMBER(dst, pCol, STColumn, type);
@@ -117,11 +52,14 @@ void *tdEncodeSchema(void *dst, STSchema *pSchema) {
  */
 STSchema *tdDecodeSchema(void **psrc) {
   int totalCols = 0;
+  int version = 0;
+  STSchemaBuilder schemaBuilder = {0};
 
+  T_READ_MEMBER(*psrc, int, version);
   T_READ_MEMBER(*psrc, int, totalCols);
 
-  STSchema *pSchema = tdNewSchema(totalCols);
+  if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL;
-  if (pSchema == NULL) return NULL;
+
   for (int i = 0; i < totalCols; i++) {
     int8_t  type = 0;
     int16_t colId = 0;
@@ -130,8 +68,94 @@ STSchema *tdDecodeSchema(void **psrc) {
     T_READ_MEMBER(*psrc, int16_t, colId);
     T_READ_MEMBER(*psrc, int32_t, bytes);
 
-    tdSchemaAddCol(pSchema, type, colId, bytes);
+    if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
+      tdDestroyTSchemaBuilder(&schemaBuilder);
+      return NULL;
     }
+  }
+
+  STSchema *pSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+  tdDestroyTSchemaBuilder(&schemaBuilder);
+  return pSchema;
+}
+
+int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
+  if (pBuilder == NULL) return -1;
+
+  pBuilder->tCols = 256;
+  pBuilder->columns = (STColumn *)malloc(sizeof(STColumn) * pBuilder->tCols);
+  if (pBuilder->columns == NULL) return -1;
+
+  tdResetTSchemaBuilder(pBuilder, version);
+  return 0;
+}
+
+void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
+  if (pBuilder) {
+    tfree(pBuilder->columns);
+  }
+}
+
+void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
+  pBuilder->nCols = 0;
+  pBuilder->tlen = 0;
+  pBuilder->flen = 0;
+  pBuilder->vlen = 0;
+  pBuilder->version = version;
+}
+
+int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int32_t bytes) {
+  if (!isValidDataType(type, 0)) return -1;
+
+  if (pBuilder->nCols >= pBuilder->tCols) {
+    pBuilder->tCols *= 2;
+    pBuilder->columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols);
+    if (pBuilder->columns == NULL) return -1;
+  }
+
+  STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
+  colSetType(pCol, type);
+  colSetColId(pCol, colId);
+  if (pBuilder->nCols == 0) {
+    colSetOffset(pCol, 0);
+  } else {
+    STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols-1]);
+    colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
+  }
+
+  if (IS_VAR_DATA_TYPE(type)) {
+    colSetBytes(pCol, bytes);
+    pBuilder->tlen += (TYPE_BYTES[type] + bytes);
+    pBuilder->vlen += bytes - sizeof(VarDataLenT);
+  } else {
+    colSetBytes(pCol, TYPE_BYTES[type]);
+    pBuilder->tlen += TYPE_BYTES[type];
+    pBuilder->vlen += TYPE_BYTES[type];
+  }
+
+  pBuilder->nCols++;
+  pBuilder->flen += TYPE_BYTES[type];
+
+  ASSERT(pCol->offset < pBuilder->flen);
+
+  return 0;
+}
+
+STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
+  if (pBuilder->nCols <= 0) return NULL;
+
+  int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols;
+
+  STSchema *pSchema = (STSchema *)malloc(tlen);
+  if (pSchema == NULL) return NULL;
+
+  schemaVersion(pSchema) = pBuilder->version;
+  schemaNCols(pSchema) = pBuilder->nCols;
+  schemaTLen(pSchema) = pBuilder->tlen;
+  schemaFLen(pSchema) = pBuilder->flen;
+  schemaVLen(pSchema) = pBuilder->vlen;
+
+  memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
 
   return pSchema;
 }
@@ -139,7 +163,10 @@ STSchema *tdDecodeSchema(void **psrc) {
 /**
  * Initialize a data row
  */
-void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); }
+void tdInitDataRow(SDataRow row, STSchema *pSchema) {
+  dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema));
+  dataRowSetVersion(row, schemaVersion(pSchema));
+}
 
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
   int32_t size = dataRowMaxBytesFromSchema(pSchema);
@@ -158,32 +185,6 @@ void tdFreeDataRow(SDataRow row) {
   if (row) free(row);
 }
 
-/**
- * Append a column value to the data row
- * @param type: column type
- * @param bytes: column bytes
- * @param offset: offset in the data row tuple, not including the data row header
- */
-int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
-  ASSERT(value != NULL);
-  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-  char *  ptr = POINTER_SHIFT(row, dataRowLen(row));
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-      memcpy(ptr, value, varDataTLen(value));
-      dataRowLen(row) += varDataTLen(value);
-      break;
-    default:
-      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-      break;
-  }
-
-  return 0;
-}
-
 SDataRow tdDataRowDup(SDataRow row) {
   SDataRow trow = malloc(dataRowLen(row));
   if (trow == NULL) return NULL;
@@ -200,42 +201,41 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
 
   pDataCol->len = 0;
   if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-    pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints;
     pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
-    pDataCol->pData = POINTER_SHIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
+    pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
-    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
+    pDataCol->spaceSize = pDataCol->bytes * maxPoints;
+    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + sizeof(VarDataOffsetT) * maxPoints);
   } else {
     pDataCol->spaceSize = pDataCol->bytes * maxPoints;
     pDataCol->dataOff = NULL;
     pDataCol->pData = *pBuf;
     *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize);
   }
-
 }
 
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints) {
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
   switch (pCol->type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       // set offset
-      pCol->dataOff[numOfPoints] = pCol->len;
+      pCol->dataOff[numOfRows] = pCol->len;
       // Copy data
       memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
       // Update the length
       pCol->len += varDataTLen(value);
       break;
     default:
-      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
       memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
       pCol->len += pCol->bytes;
       break;
   }
 }
 
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
-  int pointsLeft = numOfPoints - pointsToPop;
+  int pointsLeft = numOfRows - pointsToPop;
 
   ASSERT(pointsLeft > 0);
 
@@ -247,7 +247,7 @@ void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
     memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
     dataColSetOffset(pCol, pointsLeft);
   } else {
-    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
     pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
     memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
   }
@@ -269,25 +269,28 @@ bool isNEleNull(SDataCol *pCol, int nEle) {
   }
 }
 
-void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
+void dataColSetNullAt(SDataCol *pCol, int index) {
-  char *ptr = NULL;
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
-  switch (pCol->type) {
+    pCol->dataOff[index] = pCol->len;
-    case TSDB_DATA_TYPE_BINARY:
+    char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
-    case TSDB_DATA_TYPE_NCHAR:
+    setVardataNull(ptr, pCol->type);
-      pCol->len = 0;
-      for (int i = 0; i < nEle; i++) {
-        pCol->dataOff[i] = pCol->len;
-        ptr = (char *)pCol->pData + pCol->len;
-        varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE;
-        setNull(ptr + sizeof(VarDataLenT), pCol->type, pCol->bytes);
     pCol->len += varDataTLen(ptr);
+  } else {
+    setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
+    pCol->len += TYPE_BYTES[pCol->type];
+  }
 }
 
-      break;
+void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
-    default:
+
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->len = 0;
+    for (int i = 0; i < nEle; i++) {
+      dataColSetNullAt(pCol, i);
+    }
+  } else {
     setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
     pCol->len = TYPE_BYTES[pCol->type] * nEle;
-      break;
   }
 }
 
@@ -348,7 +351,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
   pRet->numOfCols = pDataCols->numOfCols;
   pRet->sversion = pDataCols->sversion;
-  if (keepData) pRet->numOfPoints = pDataCols->numOfPoints;
+  if (keepData) pRet->numOfRows = pDataCols->numOfRows;
 
   for (int i = 0; i < pDataCols->numOfCols; i++) {
     pRet->cols[i].type = pDataCols->cols[i].type;
@@ -378,27 +381,45 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 }
 
 void tdResetDataCols(SDataCols *pCols) {
-  pCols->numOfPoints = 0;
+  pCols->numOfRows = 0;
   for (int i = 0; i < pCols->maxCols; i++) {
     dataColReset(pCols->cols + i);
   }
 }
 
-void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) {
+void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
   ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
 
-  for (int i = 0; i < pCols->numOfCols; i++) {
+  int rcol = 0;
-    SDataCol *pCol = pCols->cols + i;
+  int dcol = 0;
-    void *    value = tdGetRowDataOfCol(row, pCol->type, pCol->offset);
 
-    dataColAppendVal(pCol, value, pCols->numOfPoints, pCols->maxPoints);
+  while (dcol < pCols->numOfCols) {
+    SDataCol *pDataCol = &(pCols->cols[dcol]);
+    if (rcol >= schemaNCols(pSchema)) {
+      dataColSetNullAt(pDataCol, pCols->numOfRows);
+      dcol++;
+      continue;
     }
-  pCols->numOfPoints++;
+
+    STColumn *pRowCol = schemaColAt(pSchema, rcol);
+    if (pRowCol->colId == pDataCol->colId) {
+      void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
+      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+      dcol++;
+      rcol++;
+    } else if (pRowCol->colId < pDataCol->colId) {
+      rcol++;
+    } else {
+      dataColSetNullAt(pDataCol, pCols->numOfRows);
+      dcol++;
+    }
+  }
+  pCols->numOfRows++;
 }
 
 // Pop pointsToPop points from the SDataCols
 void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
-  int pointsLeft = pCols->numOfPoints - pointsToPop;
+  int pointsLeft = pCols->numOfRows - pointsToPop;
   if (pointsLeft <= 0) {
     tdResetDataCols(pCols);
     return;
@@ -406,14 +427,14 @@ void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
 
   for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
     SDataCol *pCol = pCols->cols + iCol;
-    dataColPopPoints(pCol, pointsToPop, pCols->numOfPoints);
+    dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
   }
-  pCols->numOfPoints = pointsLeft;
+  pCols->numOfRows = pointsLeft;
 }
 
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
-  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
+  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
-  ASSERT(target->numOfPoints + rowsToMerge <= target->maxPoints);
+  ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
   ASSERT(target->numOfCols == source->numOfCols);
 
   SDataCols *pTarget = NULL;
@@ -421,10 +442,10 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
   if (dataColsKeyLast(target) < dataColsKeyFirst(source)) {  // No overlap
     for (int i = 0; i < rowsToMerge; i++) {
       for (int j = 0; j < source->numOfCols; j++) {
-        dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfPoints,
+        dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfRows,
                          target->maxPoints);
       }
-      target->numOfPoints++;
+      target->numOfRows++;
     }
   } else {
     pTarget = tdDupDataCols(target, true);
@@ -432,7 +453,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
 
     int iter1 = 0;
     int iter2 = 0;
-    tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
+    tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, &iter2, source->numOfRows,
+                       pTarget->numOfRows + rowsToMerge);
   }
 
   tdFreeDataCols(pTarget);
@@ -443,37 +465,196 @@ _err:
   return -1;
 }
 
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
+void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
-  // TODO: add resolve duplicate key here
   tdResetDataCols(target);
+  ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
 
-  while (target->numOfPoints < tRows) {
+  while (target->numOfRows < tRows) {
-    if (*iter1 >= src1->numOfPoints && *iter2 >= src2->numOfPoints) break;
+    if (*iter1 >= limit1 && *iter2 >= limit2) break;
 
-    TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
+    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
-    TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
 
-    if (key1 < key2) {
+    if (key1 <= key2) {
       for (int i = 0; i < src1->numOfCols; i++) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
                          target->maxPoints);
       }
 
-      target->numOfPoints++;
+      target->numOfRows++;
       (*iter1)++;
-    } else if (key1 > key2) {
+      if (key1 == key2) (*iter2)++;
+    } else {
       for (int i = 0; i < src2->numOfCols; i++) {
         ASSERT(target->cols[i].type == src2->cols[i].type);
-        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
+        dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
                          target->maxPoints);
       }
 
-      target->numOfPoints++;
+      target->numOfRows++;
       (*iter2)++;
+    }
+  }
+}
+
+SKVRow tdKVRowDup(SKVRow row) {
+  SKVRow trow = malloc(kvRowLen(row));
+  if (trow == NULL) return NULL;
+
+  kvRowCpy(trow, row);
+  return trow;
+}
+
+int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
+  SColIdx *pColIdx = NULL;
+  SKVRow   row = *orow;
+  SKVRow   nrow = NULL;
+  void *   ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE);
+
+  if (ptr == NULL || ((SColIdx *)ptr)->colId < colId) { // need to add a column value to the row
+    int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
+    nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff);
+    if (nrow == NULL) return -1;
+
+    kvRowSetLen(nrow, kvRowLen(row) + sizeof(SColIdx) + diff);
+    kvRowSetNCols(nrow, kvRowNCols(row) + 1);
+
+    if (ptr == NULL) {
+      memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * kvRowNCols(row));
+      memcpy(kvRowValues(nrow), kvRowValues(row), POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row)));
+      int colIdx = kvRowNCols(nrow) - 1;
+      kvRowColIdxAt(nrow, colIdx)->colId = colId;
+      kvRowColIdxAt(nrow, colIdx)->offset = POINTER_DISTANCE(kvRowEnd(row), kvRowValues(row));
+      memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff);
     } else {
-      // TODO: deal with duplicate keys
+      int16_t tlen = POINTER_DISTANCE(ptr, kvRowColIdx(row));
-      ASSERT(false);
+      if (tlen > 0) {
+        memcpy(kvRowColIdx(nrow), kvRowColIdx(row), tlen);
+        memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset);
+      }
+
+      int colIdx = tlen / sizeof(SColIdx);
+      kvRowColIdxAt(nrow, colIdx)->colId = colId;
+      kvRowColIdxAt(nrow, colIdx)->offset = ((SColIdx *)ptr)->offset;
+      memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx)), value, diff);
+
+      for (int i = colIdx; i < kvRowNCols(row); i++) {
+        kvRowColIdxAt(nrow, i + 1)->colId = kvRowColIdxAt(row, i)->colId;
+        kvRowColIdxAt(nrow, i + 1)->offset = kvRowColIdxAt(row, i)->offset + diff;
+      }
+      memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx)),
+             POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx)))
+
+      );
+    }
+
+    *orow = nrow;
+    free(row);
+  } else {
+    ASSERT(((SColIdx *)ptr)->colId == colId);
+    if (IS_VAR_DATA_TYPE(type)) {
+      void *pOldVal = kvRowColVal(row, (SColIdx *)ptr);
+
+      if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place
+        memcpy(pOldVal, value, varDataTLen(value));
+      } else { // need to reallocate the memory
+        int16_t diff = varDataTLen(value) - varDataTLen(pOldVal);
+        int16_t nlen = kvRowLen(row) + diff;
+        ASSERT(nlen > 0);
+        nrow = malloc(nlen);
+        if (nrow == NULL) return -1;
+
+        kvRowSetLen(nrow, nlen);
+        kvRowSetNCols(nrow, kvRowNCols(row));
+
+        // Copy part ahead
+        nlen = POINTER_DISTANCE(ptr, kvRowColIdx(row));
+        ASSERT(nlen % sizeof(SColIdx) == 0);
+        if (nlen > 0) {
+          ASSERT(((SColIdx *)ptr)->offset > 0);
+          memcpy(kvRowColIdx(nrow), kvRowColIdx(row), nlen);
+          memcpy(kvRowValues(nrow), kvRowValues(row), ((SColIdx *)ptr)->offset);
+        }
+
+        // Construct current column value
+        int colIdx = nlen / sizeof(SColIdx);
+        pColIdx = kvRowColIdxAt(nrow, colIdx);
+        pColIdx->colId = ((SColIdx *)ptr)->colId;
+        pColIdx->offset = ((SColIdx *)ptr)->offset;
+        memcpy(kvRowColVal(nrow, pColIdx), value, varDataTLen(value));
+ 
+        // Construct columns after
+        if (kvRowNCols(nrow) - colIdx - 1 > 0) {
+          for (int i = colIdx + 1; i < kvRowNCols(nrow); i++) {
+            kvRowColIdxAt(nrow, i)->colId = kvRowColIdxAt(row, i)->colId;
+            kvRowColIdxAt(nrow, i)->offset += diff;
+          }
+          memcpy(kvRowColVal(nrow, kvRowColIdxAt(nrow, colIdx + 1)), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1)),
+                 POINTER_DISTANCE(kvRowEnd(row), kvRowColVal(row, kvRowColIdxAt(row, colIdx + 1))));
+        }
+
+        *orow = nrow;
+        free(row);
+      }
+    } else {
+      memcpy(kvRowColVal(row, (SColIdx *)ptr), value, TYPE_BYTES[type]);
     }
   }
+
+  return 0;
+}
+
+void *tdEncodeKVRow(void *buf, SKVRow row) {
+  // May change the encode purpose
+  kvRowCpy(buf, row);
+  return POINTER_SHIFT(buf, kvRowLen(row));
+}
+
+void *tdDecodeKVRow(void *buf, SKVRow *row) {
+  *row = tdKVRowDup(buf);
+  return POINTER_SHIFT(buf, kvRowLen(*row));
+}
+
+int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) {
+  pBuilder->tCols = 128;
+  pBuilder->nCols = 0;
+  pBuilder->pColIdx = (SColIdx *)malloc(sizeof(SColIdx) * pBuilder->tCols);
+  if (pBuilder->pColIdx == NULL) return -1;
+  pBuilder->alloc = 1024;
+  pBuilder->size = 0;
+  pBuilder->buf = malloc(pBuilder->alloc);
+  if (pBuilder->buf == NULL) {
+    free(pBuilder->pColIdx);
+    return -1;
+  }
+  return 0;
+}
+
+void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) {
+  tfree(pBuilder->pColIdx);
+  tfree(pBuilder->buf);
+}
+
+void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {
+  pBuilder->nCols = 0;
+  pBuilder->size = 0;
+}
+
+SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
+  int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size;
+  if (tlen == 0) return NULL;
+
+  tlen += TD_KV_ROW_HEAD_SIZE;
+
+  SKVRow row = malloc(tlen);
+  if (row == NULL) return NULL;
+
+  kvRowSetNCols(row, pBuilder->nCols);
+  kvRowSetLen(row, tlen);
+
+  memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols);
+  memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size);
+
+  return row;
 }

src/common/src/tglobal.c

@@ -55,16 +55,17 @@ int32_t tsEnableCoreFile = 0;
 int32_t tscEmbedded = 0;
 
 /*
- * minmum scale for whole system, millisecond by default
+ * minimum scale for whole system, millisecond by default
  * for TSDB_TIME_PRECISION_MILLI: 86400000L
  *     TSDB_TIME_PRECISION_MICRO: 86400000000L
+ *     TSDB_TIME_PRECISION_NANO:  86400000000000L
  */
-int64_t tsMsPerDay[] = {86400000L, 86400000000L};
+int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
 
-char  tsFirst[TSDB_FQDN_LEN] = {0};
+char  tsFirst[TSDB_EP_LEN] = {0};  
-char  tsSecond[TSDB_FQDN_LEN] = {0};
+char  tsSecond[TSDB_EP_LEN] = {0};
-char  tsArbitrator[TSDB_FQDN_LEN] = {0};
+char  tsArbitrator[TSDB_EP_LEN] = {0};
-char  tsLocalEp[TSDB_FQDN_LEN] = {0};  // Local End Point, hostname:port
+char  tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
 uint16_t tsDnodeDnodePort = 6035;   // udp/tcp
@@ -105,42 +106,42 @@ int32_t tsReplications  = TSDB_DEFAULT_REPLICA_NUM;
  * 1: affected rows include those duplicate records
  */
 int16_t tsAffectedRowsMod = 0;
-int32_t tsNumOfMPeers = 3;
+int32_t tsNumOfMnodes = 3;
-int32_t tsMaxShellConns = 2000;
+int32_t tsMaxShellConns = 5000;
-int32_t tsMaxTables = 100000;
 
 char    tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
 char    tsDefaultUser[64] = "root";
 char    tsDefaultPass[64] = "taosdata";
-int32_t tsMaxMeterConnections = 10000;
+int32_t tsMaxConnections = 5000;
-int32_t tsMaxMgmtConnections = 2000;
-int32_t tsMaxVnodeConnections = 10000;
 
 int32_t tsBalanceInterval = 300;  // seconds
 int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
-int32_t tsMgmtEqualVnodeNum = 4;
+int32_t tsMnodeEqualVnodeNum = 4;
 
 int32_t tsEnableHttpModule = 1;
+int32_t tsEnableMqttModule = 0;   // not finished yet, not started it by default
 int32_t tsEnableMonitorModule = 0;
 
 int32_t tsRestRowLimit = 10240;
 int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
 
 int32_t tsNumOfLogLines = 10000000;
-int32_t mdebugFlag = 135;
+int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
 int32_t dDebugFlag = 135;
 int32_t vDebugFlag = 135;
-int32_t cdebugFlag = 135;
+int32_t cDebugFlag = 135;
-int32_t jnidebugFlag = 131;
+int32_t jniDebugFlag = 131;
-int32_t odbcdebugFlag = 131;
+int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
+int32_t mqttDebugFlag = 131;
 int32_t monitorDebugFlag = 131;
-int32_t qdebugFlag = 131;
+int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
 int32_t sDebugFlag = 135;
+int32_t tsdbDebugFlag = 135;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -199,15 +200,30 @@ int32_t tsMonitorInterval = 30;  // seconds
 char tsTimezone[64] = {0};
 char tsLocale[TSDB_LOCALE_LEN] = {0};
 char tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
+char tsMqttBrokerAddress[128] = {0}; 
+char tsMqttBrokerClientId[128] = {0};
+
+int32_t tsMaxBinaryDisplayWidth = 30;
 
 static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 
 void taosSetAllDebugFlag() {
   for (int32_t i = 0; i < tsGlobalConfigNum; ++i) {
-    SGlobalCfg *cfg = &tsGlobalConfig[i];
+    mDebugFlag = debugFlag;
-    if ((cfg->cfgType & TSDB_CFG_CTYPE_B_LOG) && cfg->cfgType == TAOS_CFG_VTYPE_INT32) {
+    sdbDebugFlag = debugFlag;
-      *((int32_t*)cfg->ptr) = debugFlag;
+    dDebugFlag = debugFlag;
-    }
+    vDebugFlag = debugFlag;
+    cDebugFlag = debugFlag;
+    jniDebugFlag = debugFlag;
+    odbcDebugFlag = debugFlag;
+    httpDebugFlag = debugFlag;
+    mqttDebugFlag = debugFlag;
+    monitorDebugFlag = debugFlag;
+    rpcDebugFlag = debugFlag;
+    uDebugFlag = debugFlag;
+    sDebugFlag = debugFlag;
+    tsdbDebugFlag = debugFlag;
+    qDebugFlag = debugFlag;    
   }
   uPrint("all debug flag are set to %d", debugFlag);
 }
@@ -218,7 +234,7 @@ bool taosCfgDynamicOptions(char *msg) {
   int32_t   vint = 0;
 
   paGetToken(msg, &option, &olen);
-  if (olen == 0) return TSDB_CODE_INVALID_MSG_CONTENT;
+  if (olen == 0) return TSDB_CODE_COM_INVALID_CFG_MSG;
 
   paGetToken(option + olen + 1, &value, &vlen);
   if (vlen == 0)
@@ -275,7 +291,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -285,7 +301,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -347,7 +363,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.ptrLength = TSDB_EP_LEN;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -392,8 +408,8 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "numOfMPeers";
+  cfg.option = "numOfMnodes";
-  cfg.ptr = &tsNumOfMPeers;
+  cfg.ptr = &tsNumOfMnodes;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 1;
@@ -412,7 +428,7 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  // 0-any; 1-mgmt; 2-dnode
+  // 0-any; 1-mnode; 2-vnode
   cfg.option = "alternativeRole";
   cfg.ptr = &tsAlternativeRole;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -592,7 +608,7 @@ static void doInitGlobalConfig() {
   cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE;
   cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE;
   cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_BYTE;
+  cfg.unitType = TAOS_CFG_UTYPE_Mb;
   taosInitConfigOption(cfg);
 
   cfg.option = "blocks";
@@ -602,7 +618,7 @@ static void doInitGlobalConfig() {
   cfg.minValue = TSDB_MIN_TOTAL_BLOCKS;
   cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS;
   cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_BYTE;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
   cfg.option = "days";
@@ -665,7 +681,7 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "wallevel";
+  cfg.option = "walLevel";
   cfg.ptr = &tsWAL;
   cfg.valType = TAOS_CFG_VTYPE_INT16;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
@@ -692,7 +708,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN;
+  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -716,6 +732,26 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "mqttBrokerAddress";
+  cfg.ptr = tsMqttBrokerAddress;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = 126;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "mqttBrokerClientId";
+  cfg.ptr = tsMqttBrokerClientId;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = 126;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+ 
   // socket type; udp by default
   cfg.option = "sockettype";
   cfg.ptr = tsSocketType;
@@ -799,32 +835,12 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "maxMeterConnections";
+  cfg.option = "maxConnections";
-  cfg.ptr = &tsMaxMeterConnections;
+  cfg.ptr = &tsMaxConnections;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 10;
+  cfg.minValue = 1;
-  cfg.maxValue = 50000000;
+  cfg.maxValue = 100000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
-  cfg.option = "maxMgmtConnections";
-  cfg.ptr = &tsMaxMgmtConnections;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 10;
-  cfg.maxValue = 50000000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
-  cfg.option = "maxVnodeConnections";
-  cfg.ptr = &tsMaxVnodeConnections;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 10;
-  cfg.maxValue = 50000000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
@@ -860,8 +876,8 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   // module configs
-  cfg.option = "mgmtEqualVnodeNum";
+  cfg.option = "mnodeEqualVnodeNum";
-  cfg.ptr = &tsMgmtEqualVnodeNum;
+  cfg.ptr = &tsMnodeEqualVnodeNum;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;
@@ -880,6 +896,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  
+  cfg.option = "mqtt";
+  cfg.ptr = &tsEnableMqttModule;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "monitor";
   cfg.ptr = &tsEnableMonitorModule;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -896,7 +923,7 @@ static void doInitGlobalConfig() {
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;
   cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_DB_NAME_LEN;
+  cfg.ptrLength = TSDB_DB_NAME_LEN - 1;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
@@ -993,7 +1020,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "mDebugFlag";
-  cfg.ptr = &mdebugFlag;
+  cfg.ptr = &mDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
@@ -1053,7 +1080,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "cDebugFlag";
-  cfg.ptr = &cdebugFlag;
+  cfg.ptr = &cDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1063,7 +1090,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "jniDebugFlag";
-  cfg.ptr = &jnidebugFlag;
+  cfg.ptr = &jniDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1073,7 +1100,7 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "odbcDebugFlag";
-  cfg.ptr = &odbcdebugFlag;
+  cfg.ptr = &odbcDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1102,6 +1129,17 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "mqttDebugFlag";
+  cfg.ptr = &mqttDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+
   cfg.option = "monitorDebugFlag";
   cfg.ptr = &monitorDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1113,7 +1151,17 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   cfg.option = "qDebugFlag";
-  cfg.ptr = &qdebugFlag;
+  cfg.ptr = &qDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "tsdbDebugFlag";
+  cfg.ptr = &tsdbDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
@@ -1182,6 +1230,16 @@ static void doInitGlobalConfig() {
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
+
+  cfg.option = "maxBinaryDisplayWidth";
+  cfg.ptr = &tsMaxBinaryDisplayWidth;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 1;
+  cfg.maxValue = 0x7fffffff;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
 }
 
 void taosInitGlobalCfg() {
@@ -1189,6 +1247,10 @@ void taosInitGlobalCfg() {
 }
 
 bool taosCheckGlobalCfg() {
+  if (debugFlag == 135 || debugFlag == 199) {
+    taosSetAllDebugFlag();
+  }
+  
   taosGetFqdn(tsLocalEp);
   sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
   uPrint("localEp is %s", tsLocalEp);
@@ -1239,7 +1301,7 @@ bool taosCheckGlobalCfg() {
   return true;
 }
 
-int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) {
+int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
   *port = 0;
   strcpy(fqdn, ep);
 

src/common/src/tlocale.c

@@ -31,16 +31,16 @@ void tsSetLocale() {
 
   // default locale or user specified locale is not valid, abort launch
   if (locale == NULL) {
-    uForcePrint("Invalid locale:%s, please set the valid locale in config file", tsLocale);
+    uError("Invalid locale:%s, please set the valid locale in config file", tsLocale);
   }
 
   if (strlen(tsCharset) == 0) {
-    uForcePrint("failed to get charset, please set the valid charset in config file");
+    uError("failed to get charset, please set the valid charset in config file");
     exit(-1);
   }
 
   if (!taosValidateEncodec(tsCharset)) {
-    uForcePrint("Invalid charset:%s, please set the valid charset in config file", tsCharset);
+    uError("Invalid charset:%s, please set the valid charset in config file", tsCharset);
     exit(-1);
   }
 }

src/common/src/tname.c

@@ -28,7 +28,7 @@ void extractTableName(const char* tableId, char* name) {
   size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
   size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]);
   
-  strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
+  tstrncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN);
 }
 
 char* extractDBName(const char* tableId, char* name) {

src/common/src/ttypes.c

@@ -32,18 +32,319 @@ const int32_t TYPE_BYTES[11] = {
     sizeof(VarDataOffsetT)   // TSDB_DATA_TYPE_NCHAR
 };
 
+static void getStatics_bool(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                            int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int8_t *data = (int8_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((char *)&data[i], TSDB_DATA_TYPE_BOOL)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+  }
+}
+
+static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                          int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int8_t *data = (int8_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+  }
+}
+
+static void getStatics_i16(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int16_t *data = (int16_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  //  int64_t lastKey = 0;
+  //  int16_t lastVal = TSDB_DATA_SMALLINT_NULL;
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+}
+
+static void getStatics_i32(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int32_t *data = (int32_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  //  int64_t lastKey = 0;
+  //  int32_t lastVal = TSDB_DATA_INT_NULL;
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+  }
+}
+
+static void getStatics_i64(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  int64_t *data = (int64_t *)pData;
+  *min = INT64_MAX;
+  *max = INT64_MIN;
+  *minIndex = 0;
+  *maxIndex = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    *sum += data[i];
+    if (*min > data[i]) {
+      *min = data[i];
+      *minIndex = i;
+    }
+    
+    if (*max < data[i]) {
+      *max = data[i];
+      *maxIndex = i;
+    }
+    
+    //    if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) {
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    } else {
+    //      *wsum = lastVal * (primaryKey[i] - lastKey);
+    //      lastKey = primaryKey[i];
+    //      lastVal = data[i];
+    //    }
+  }
+}
+
+static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  float *data = (float *)pData;
+  float fmin      = DBL_MAX;
+  float fmax      = -DBL_MAX;
+  double dsum     = 0;
+  *minIndex       = 0;
+  *maxIndex       = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    float fv = 0;
+    fv = GET_FLOAT_VAL(&(data[i]));
+    dsum += fv;
+    if (fmin > fv) {
+      fmin = fv;
+      *minIndex = i;
+    }
+    
+    if (fmax < fv) {
+      fmax = fv;
+      *maxIndex = i;
+    }
+  }
+  
+  double csum = 0;
+  csum = GET_DOUBLE_VAL(sum);
+  csum += dsum;
+#ifdef _TD_ARM_32_
+  SET_DOUBLE_VAL_ALIGN(sum, &csum);
+  SET_DOUBLE_VAL_ALIGN(max, &fmax);
+  SET_DOUBLE_VAL_ALIGN(min, &fmin);
+#else
+  *(double*)sum = csum;
+  *(double*)max = fmax;
+  *(double*)min = fmin;
+#endif
+}
+
+static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  double *data = (double *)pData;
+  double dmin      = DBL_MAX;
+  double dmax      = -DBL_MAX;
+  double dsum      = 0;
+  *minIndex        = 0;
+  *maxIndex        = 0;
+  
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) {
+      (*numOfNull) += 1;
+      continue;
+    }
+    
+    double dv = 0;
+    dv = GET_DOUBLE_VAL(&(data[i]));
+    dsum += dv;
+    if (dmin > dv) {
+      dmin = dv;
+      *minIndex = i;
+    }
+    
+    if (dmax < dv) {
+      dmax = dv;
+      *maxIndex = i;
+    }
+  }
+  
+  double csum = 0;
+  csum = GET_DOUBLE_VAL(sum);
+  csum += dsum;
+
+
+#ifdef _TD_ARM_32_
+  SET_DOUBLE_VAL_ALIGN(sum, &csum);
+  SET_DOUBLE_VAL_ALIGN(max, &dmax);
+  SET_DOUBLE_VAL_ALIGN(min, &dmin);
+#else
+  *(double*) sum = csum;
+  *(double*) max = dmax;
+  *(double*) min = dmin;
+#endif
+}
+
+static void getStatics_bin(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                         int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  const char* data = pData;
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
+      (*numOfNull) += 1;
+    }
+    
+    data += varDataTLen(data);
+  }
+  
+  *sum = 0;
+  *max = 0;
+  *min = 0;
+  *minIndex = 0;
+  *maxIndex = 0;
+}
+
+static void getStatics_nchr(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max,
+                           int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+  const char* data = pData;
+  ASSERT(numOfRow <= INT16_MAX);
+  
+  for (int32_t i = 0; i < numOfRow; ++i) {
+    if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
+      (*numOfNull) += 1;
+    }
+    
+    data += varDataTLen(data);
+  }
+  
+  *sum = 0;
+  *max = 0;
+  *min = 0;
+  *minIndex = 0;
+  *maxIndex = 0;
+}
+
 tDataTypeDescriptor tDataTypeDesc[11] = {
-  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL},
+  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL,                  NULL},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      getStatics_bool},
-  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint},
+  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
-  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint},
+  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
-  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt},
+  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt,       getStatics_i32},
-  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint},
+  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
-  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat},
+  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat,     getStatics_f},
-  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble},
+  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble,    getStatics_d},
-  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString},
+  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString,    getStatics_bin},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
-  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString},
+  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString,    getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {
@@ -63,7 +364,7 @@ char tTokenTypeSwitcher[13] = {
 };
 
 bool isValidDataType(int32_t type, int32_t length) {
-  if (type < TSDB_DATA_TYPE_BOOL || type > TSDB_DATA_TYPE_NCHAR) {
+  if (type < TSDB_DATA_TYPE_NULL || type > TSDB_DATA_TYPE_NCHAR) {
     return false;
   }
 
@@ -100,6 +401,18 @@ bool isNull(const char *val, int32_t type) {
   };
 }
 
+void setVardataNull(char* val, int32_t type) {
+  if (type == TSDB_DATA_TYPE_BINARY) {
+    varDataSetLen(val, sizeof(int8_t));
+    *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL;
+  } else if (type == TSDB_DATA_TYPE_NCHAR) {
+    varDataSetLen(val, sizeof(int32_t));
+    *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL;
+  } else {
+    assert(0);
+  }
+}
+
 void setNull(char *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); }
 
 void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
@@ -140,7 +453,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
         *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
       }
       break;
-    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_NCHAR: // todo : without length?
       for (int32_t i = 0; i < numOfElems; ++i) {
         *(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
       }
@@ -202,7 +515,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
       break;
     };
     case TSDB_DATA_TYPE_NCHAR: {
-      wcsncpy((wchar_t*)val, (wchar_t*)src, len / TSDB_NCHAR_SIZE);
+      varDataCopy(val, src);
       break;
     };
     default: {

src/connector/grafana/tdengine/package-lock.json

@@ -3724,9 +3724,9 @@
       }
     },
     "websocket-extensions": {
-      "version": "0.1.3",
+      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz",
-      "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==",
+      "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==",
       "dev": true
     },
     "whatwg-encoding": {

src/connector/grafana/tdengine/yarn.lock

@@ -2839,8 +2839,9 @@ websocket-driver@>=0.5.1:
     websocket-extensions ">=0.1.1"
 
 websocket-extensions@>=0.1.1:
-  version "0.1.3"
+  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29"
+  resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42"
+  integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==
 
 whatwg-encoding@^1.0.1:
   version "1.0.3"

src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java

@@ -18,8 +18,8 @@ import java.io.InputStream;
 import java.io.Reader;
 import java.math.BigDecimal;
 import java.net.URL;
-import java.sql.*;
 import java.sql.Date;
+import java.sql.*;
 import java.util.*;
 
 /*
@@ -102,41 +102,49 @@ public class DatabaseMetaDataResultSet implements ResultSet {
 
     @Override
     public byte getByte(int columnIndex) throws SQLException {
+        columnIndex--;
         return (byte) rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public short getShort(int columnIndex) throws SQLException {
+        columnIndex--;
         return (short) rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public int getInt(int columnIndex) throws SQLException {
+        columnIndex--;
         return rowCursor.getInt(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public long getLong(int columnIndex) throws SQLException {
+        columnIndex--;
         return rowCursor.getLong(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public float getFloat(int columnIndex) throws SQLException {
+        columnIndex--;
         return rowCursor.getFloat(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public double getDouble(int columnIndex) throws SQLException {
+        columnIndex--;
         return rowCursor.getDouble(columnIndex, columnMetaDataList.get(columnIndex).getColType());
     }
 
     @Override
     public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException {
+        columnIndex--;
         return new BigDecimal(rowCursor.getDouble(columnIndex, columnMetaDataList.get(columnIndex).getColType()));
     }
 
     @Override
     public byte[] getBytes(int columnIndex) throws SQLException {
+        columnIndex--;
         return (rowCursor.getString(columnIndex, columnMetaDataList.get(columnIndex).getColType())).getBytes();
     }
 
@@ -152,6 +160,7 @@ public class DatabaseMetaDataResultSet implements ResultSet {
 
     @Override
     public Timestamp getTimestamp(int columnIndex) throws SQLException {
+        columnIndex--;
         return rowCursor.getTimestamp(columnIndex);
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java

@@ -84,6 +84,14 @@ public class TSDBConnection implements Connection {
 		}
 	}
 
+	public TSDBSubscribe createSubscribe() throws SQLException {
+		if (!this.connector.isClosed()) {
+			return new TSDBSubscribe(this.connector);
+		} else {
+			throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+		}
+	}
+
 	public PreparedStatement prepareStatement(String sql) throws SQLException {
 		if (!this.connector.isClosed()) {
 			return new TSDBPreparedStatement(this.connector, sql);

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -188,7 +188,7 @@ public class TSDBDriver implements java.sql.Driver {
 	}
 
 	public boolean acceptsURL(String url) throws SQLException {
-		return true;
+		return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX);
 	}
 
 	public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -99,7 +99,7 @@ public class TSDBJNIConnector {
 
         this.taos = this.connectImp(host, port, dbName, user, password);
         if (this.taos == TSDBConstants.JNI_NULL_POINTER) {
-            throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg()), "", this.getErrCode());
+            throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(null)), "", this.getErrCode(null));
         }
 
         return true;
@@ -117,52 +117,57 @@ public class TSDBJNIConnector {
             freeResultSet(taosResultSetPointer);
         }
 
-        int code;
+        long pSql = 0l;
         try {
-            code = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
+            pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
         } catch (Exception e) {
             e.printStackTrace();
+            this.freeResultSet(pSql);
             throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
         }
+        int code = this.getErrCode(pSql);
+
         affectedRows = code;
         if (code < 0) {
             affectedRows = -1;
             if (code == TSDBConstants.JNI_TDENGINE_ERROR) {
-                throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg()), "", this.getErrCode());
+                this.freeResultSet(pSql);
+                throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(pSql)), "", this.getErrCode(pSql));
             } else {
-                throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode());
+                this.freeResultSet(pSql);
+                throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(pSql));
             }
         }
 
         // Try retrieving result set for the executed SQL using the current connection pointer. If the executed
         // SQL is a DML/DDL which doesn't return a result set, then taosResultSetPointer should be 0L. Otherwise,
         // taosResultSetPointer should be a non-zero value.
-        taosResultSetPointer = this.getResultSetImp(this.taos);
+        taosResultSetPointer = this.getResultSetImp(this.taos, pSql);
         if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) {
             isResultsetClosed = false;
         }
         return code;
     }
 
-    private native int executeQueryImp(byte[] sqlBytes, long connection);
+    private native long executeQueryImp(byte[] sqlBytes, long connection);
 
     /**
      * Get recent error code by connection
      */
-    public int getErrCode() {
+    public int getErrCode(Long pSql) {
-        return Math.abs(this.getErrCodeImp(this.taos));
+        return Math.abs(this.getErrCodeImp(this.taos, pSql));
     }
 
-    private native int getErrCodeImp(long connection);
+    private native int getErrCodeImp(long connection, Long pSql);
 
     /**
      * Get recent error message by connection
      */
-    public String getErrMsg() {
+    public String getErrMsg(Long pSql) {
-        return this.getErrMsgImp(this.taos);
+        return this.getErrMsgImp(this.taos, pSql);
     }
 
-    private native String getErrMsgImp(long connection);
+    private native String getErrMsgImp(long connection, Long pSql);
 
     /**
      * Get resultset pointer
@@ -172,7 +177,7 @@ public class TSDBJNIConnector {
         return taosResultSetPointer;
     }
 
-    private native long getResultSetImp(long connection);
+    private native long getResultSetImp(long connection, long pSql);
 
     /**
      * Free resultset operation from C to release resultset pointer by JNI
@@ -212,15 +217,15 @@ public class TSDBJNIConnector {
     /**
      * Get affected rows count
      */
-    public int getAffectedRows() {
+    public int getAffectedRows(Long pSql) {
         int affectedRows = this.affectedRows;
         if (affectedRows < 0) {
-            affectedRows = this.getAffectedRowsImp(this.taos);
+            affectedRows = this.getAffectedRowsImp(this.taos, pSql);
         }
         return affectedRows;
     }
 
-    private native int getAffectedRowsImp(long connection);
+    private native int getAffectedRowsImp(long connection, Long pSql);
 
     /**
      * Get schema metadata
@@ -248,7 +253,7 @@ public class TSDBJNIConnector {
     public void closeConnection() throws SQLException {
         int code = this.closeConnectionImp(this.taos);
         if (code < 0) {
-            throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode());
+            throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(null));
         } else if (code == 0) {
             this.taos = TSDBConstants.JNI_NULL_POINTER;
         } else {
@@ -261,31 +266,31 @@ public class TSDBJNIConnector {
     /**
      * Subscribe to a table in TSDB
      */
-    public long subscribe(String host, String user, String password, String database, String table, long time, int period) {
+    public long subscribe(String topic, String sql, boolean restart, int period) {
-        return subscribeImp(host, user, password, database, table, time, period);
+        return subscribeImp(this.taos, restart, topic, sql, period);
     }
 
-    private native long subscribeImp(String host, String user, String password, String database, String table, long time, int period);
+    public native long subscribeImp(long connection, boolean restart, String topic, String sql, int period);
 
     /**
      * Consume a subscribed table
      */
-    public TSDBResultSetRowData consume(long subscription) {
+    public long consume(long subscription) {
         return this.consumeImp(subscription);
     }
 
-    private native TSDBResultSetRowData consumeImp(long subscription);
+    private native long consumeImp(long subscription);
 
     /**
      * Unsubscribe a table
      *
      * @param subscription
      */
-    public void unsubscribe(long subscription) {
+    public void unsubscribe(long subscription, boolean isKeep) {
-        unsubscribeImp(subscription);
+        unsubscribeImp(subscription, isKeep);
     }
 
-    private native void unsubscribeImp(long subscription);
+    private native void unsubscribeImp(long subscription, boolean isKeep);
 
     /**
      * Validate if a <I>create table</I> sql statement is correct without actually creating that table

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -27,6 +27,8 @@ public class TSDBStatement implements Statement {
 	/** Timeout for a query */
 	protected int queryTimeout = 0;
 
+	private Long pSql = 0l;
+
     /**
      * Status of current statement
      */
@@ -66,21 +68,23 @@ public class TSDBStatement implements Statement {
         if (isClosed) {
             throw new SQLException("Invalid method call on a closed statement.");
         }
-		int res = this.connecter.executeQuery(sql);
+		long res = this.connecter.executeQuery(sql);
 		long resultSetPointer = this.connecter.getResultSet();
 
 		if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
+			this.connecter.freeResultSet(res);
 			throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
 		} else if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) {
 			this.connecter.freeResultSet();
 			throw new SQLException("The executed SQL is not a DML or a DDL");
 		} else {
-			return res;
+			int num = this.connecter.getAffectedRows(res);
+			return num;
 		}
 	}
 
-	public String getErrorMsg() {
+	public String getErrorMsg(long pSql) {
-		return this.connecter.getErrMsg();
+		return this.connecter.getErrMsg(pSql);
 	}
 
 	public void close() throws SQLException {
@@ -170,7 +174,7 @@ public class TSDBStatement implements Statement {
         if (isClosed) {
             throw new SQLException("Invalid method call on a closed statement.");
         }
-		return this.connecter.getAffectedRows();
+		return this.connecter.getAffectedRows(this.pSql);
 	}
 
 	public boolean getMoreResults() throws SQLException {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java

@@ -0,0 +1,185 @@
+/***************************************************************************
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *****************************************************************************/
+package com.taosdata.jdbc;
+
+import javax.management.OperationsException;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.TimerTask;
+import java.util.concurrent.*;
+
+public class TSDBSubscribe {
+    private TSDBJNIConnector connecter = null;
+    private static ScheduledExecutorService pool;
+    private static Map<Long, TSDBTimerTask> timerTaskMap = new ConcurrentHashMap<>();
+    private static Map<Long, ScheduledFuture> scheduledMap = new ConcurrentHashMap();
+
+    private static class TimerInstance {
+        private static final ScheduledExecutorService instance = Executors.newScheduledThreadPool(1);
+    }
+
+    public static ScheduledExecutorService getTimerInstance() {
+        return TimerInstance.instance;
+    }
+
+    public TSDBSubscribe(TSDBJNIConnector connecter) throws SQLException {
+        if (null != connecter) {
+            this.connecter = connecter;
+        } else {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        }
+    }
+
+    /**
+     * sync subscribe
+     *
+     * @param topic
+     * @param sql
+     * @param restart
+     * @param period
+     * @throws SQLException
+     */
+    public long subscribe(String topic, String sql, boolean restart, int period) throws SQLException {
+        if (this.connecter.isClosed()) {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        }
+        if (period < 1000) {
+            throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.INVALID_VARIABLES));
+        }
+        return this.connecter.subscribe(topic, sql, restart, period);
+    }
+
+    /**
+     * async subscribe
+     *
+     * @param topic
+     * @param sql
+     * @param restart
+     * @param period
+     * @param callBack
+     * @throws SQLException
+     */
+    public long subscribe(String topic, String sql, boolean restart, int period, TSDBSubscribeCallBack callBack) throws SQLException {
+        if (this.connecter.isClosed()) {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        }
+        final long subscription = this.connecter.subscribe(topic, sql, restart, period);
+        if (null != callBack) {
+            pool = getTimerInstance();
+
+            TSDBTimerTask timerTask = new TSDBTimerTask(subscription, callBack);
+
+            timerTaskMap.put(subscription, timerTask);
+
+            ScheduledFuture scheduledFuture = pool.scheduleAtFixedRate(timerTask, 1, 1000, TimeUnit.MILLISECONDS);
+            scheduledMap.put(subscription, scheduledFuture);
+        }
+        return subscription;
+    }
+
+    public TSDBResultSet consume(long subscription) throws OperationsException, SQLException {
+        if (this.connecter.isClosed()) {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        }
+        if (0 == subscription) {
+            throw new OperationsException("Invalid use of consume");
+        }
+        long resultSetPointer = this.connecter.consume(subscription);
+
+        if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) {
+            return null;
+        } else {
+            return new TSDBResultSet(this.connecter, resultSetPointer);
+        }
+    }
+
+    /**
+     * cancel subscribe
+     *
+     * @param subscription
+     * @param isKeep
+     * @throws SQLException
+     */
+    public void unsubscribe(long subscription, boolean isKeep) throws SQLException {
+        if (this.connecter.isClosed()) {
+            throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
+        }
+
+        if (null != timerTaskMap.get(subscription)) {
+            synchronized (timerTaskMap.get(subscription)) {
+                while (1 == timerTaskMap.get(subscription).getState()) {
+                    try {
+                        Thread.sleep(10);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                timerTaskMap.get(subscription).setState(2);
+                if (!timerTaskMap.isEmpty() && timerTaskMap.containsKey(subscription)) {
+                    timerTaskMap.get(subscription).cancel();
+                    timerTaskMap.remove(subscription);
+                    scheduledMap.get(subscription).cancel(false);
+                    scheduledMap.remove(subscription);
+                }
+                this.connecter.unsubscribe(subscription, isKeep);
+            }
+        } else {
+            this.connecter.unsubscribe(subscription, isKeep);
+        }
+    }
+
+    class TSDBTimerTask extends TimerTask {
+        private long subscription;
+        private TSDBSubscribeCallBack callBack;
+        // 0: not running 1: running 2: cancel
+        private int state = 0;
+
+        public TSDBTimerTask(long subscription, TSDBSubscribeCallBack callBack) {
+            this.subscription = subscription;
+            this.callBack = callBack;
+        }
+
+        public int getState() {
+            return this.state;
+        }
+
+        public void setState(int state) {
+            this.state = state;
+        }
+
+        @Override
+        public void run() {
+            synchronized (this) {
+                if (2 == state) {
+                    return;
+                }
+
+                state = 1;
+
+                try {
+                    TSDBResultSet resultSet = consume(subscription);
+                    callBack.invoke(resultSet);
+                } catch (Exception e) {
+                    this.cancel();
+                    throw new RuntimeException(e);
+                }
+                state = 0;
+            }
+        }
+    }
+}
+

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java

@@ -0,0 +1,19 @@
+/***************************************************************************
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *****************************************************************************/
+package com.taosdata.jdbc;
+
+public interface TSDBSubscribeCallBack {
+    void invoke(TSDBResultSet resultSet);
+}

src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java

@@ -0,0 +1,83 @@
+import com.taosdata.jdbc.*;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Properties;
+
+public class TestAsyncTSDBSubscribe {
+    public static void main(String[] args) {
+        String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName -topic topicName " +
+                "-tname tableName -h host";
+        if (args.length < 2) {
+            System.err.println(usage);
+            return;
+        }
+
+        String dbName = "";
+        String tName = "";
+        String host = "localhost";
+        String topic = "";
+        for (int i = 0; i < args.length; i++) {
+            if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                dbName = args[++i];
+            }
+            if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                tName = args[++i];
+            }
+            if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                host = args[++i];
+            }
+            if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                topic = args[++i];
+            }
+        }
+        if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) {
+            System.err.println(usage);
+            return;
+        }
+
+        Connection connection = null;
+        TSDBSubscribe subscribe = null;
+        long subscribId = 0;
+        try {
+            Class.forName("com.taosdata.jdbc.TSDBDriver");
+            Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+            connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties);
+            String rawSql = "select * from " + tName + ";";
+            subscribe = ((TSDBConnection) connection).createSubscribe();
+            subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first"));
+            long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second"));
+            int a = 0;
+            Thread.sleep(2000);
+            subscribe.unsubscribe(subscribId, true);
+            System.err.println("cancel subscribe");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static class CallBack implements TSDBSubscribeCallBack {
+        private String name = "";
+
+        public CallBack(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public void invoke(TSDBResultSet resultSet) {
+            try {
+                while (null !=resultSet && resultSet.next()) {
+                    System.out.print("callback_" + name + ": ");
+                    for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
+                        System.out.printf(i + ": " + resultSet.getString(i) + "\t");
+                    }
+                    System.out.println();
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+    }
+}

src/connector/jdbc/src/test/java/TestPreparedStatement.java

@@ -10,9 +10,9 @@ public class TestPreparedStatement {
         try {
             Class.forName("com.taosdata.jdbc.TSDBDriver");
             Properties properties = new Properties();
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.117");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
-            Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.117:0/?user=root&password=taosdata", properties);
+            Connection connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties);
-            String rawSql = "SELECT ts, c1 FROM (select c1, ts from db.tb1) SUB_QRY";
+            String rawSql = "select * from test.log0601";
 //            String[] params = new String[]{"ts", "c1"};
             PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql);
             ResultSet resSet = pstmt.executeQuery();

src/connector/jdbc/src/test/java/TestTSDBSubscribe.java

@@ -0,0 +1,83 @@
+import com.taosdata.jdbc.TSDBConnection;
+import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.TSDBResultSet;
+import com.taosdata.jdbc.TSDBSubscribe;
+import org.apache.commons.lang3.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Properties;
+
+public class TestTSDBSubscribe {
+    public static void main(String[] args) throws Exception {
+        String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName " +
+                "-topic topicName -tname tableName -h host";
+        if (args.length < 2) {
+            System.err.println(usage);
+            return;
+        }
+
+        String dbName = "";
+        String tName = "";
+        String host = "localhost";
+        String topic = "";
+        for (int i = 0; i < args.length; i++) {
+            if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                dbName = args[++i];
+            }
+            if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                tName = args[++i];
+            }
+            if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                host = args[++i];
+            }
+            if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+                topic = args[++i];
+            }
+        }
+        if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) {
+            System.err.println(usage);
+            return;
+        }
+
+        Connection connection = null;
+        TSDBSubscribe subscribe = null;
+        long subscribId = 0;
+        try {
+            Class.forName("com.taosdata.jdbc.TSDBDriver");
+            Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
+            connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata"
+                    , properties);
+            String rawSql = "select * from " + tName + ";";
+            subscribe = ((TSDBConnection) connection).createSubscribe();
+            subscribId = subscribe.subscribe(topic, rawSql, false, 1000);
+            int a = 0;
+            while (true) {
+                Thread.sleep(900);
+                TSDBResultSet resSet = subscribe.consume(subscribId);
+
+                while (resSet.next()) {
+                    for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
+                        System.out.printf(i + ": " + resSet.getString(i) + "\t");
+                    }
+                    System.out.println("\n======" + a + "==========");
+                }
+
+                a++;
+                if (a >= 10) {
+                    break;
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (null != subscribe && 0 != subscribId) {
+                subscribe.unsubscribe(subscribId, true);
+            }
+            if (null != connection) {
+                connection.close();
+            }
+        }
+    }
+}

src/connector/nodejs/nodetaos/cinterface.js

@@ -241,17 +241,12 @@ function CTaosInterface (config = null, pass = false) {
     'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]],
 
     // Subscription
-    //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, long time, int mseconds)
+    //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval)
-    ////TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds);
+    'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ],
-    'taos_subscribe': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int64, ref.types.int] ],
+    // TAOS_RES *taos_consume(TAOS_SUB *tsub)
-    //TAOS_ROW taos_consume(TAOS_SUB *tsub);
+    'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ],
-    'taos_consume': [ ref.refType(ref.types.void_ptr2), [ref.types.void_ptr] ],
     //void taos_unsubscribe(TAOS_SUB *tsub);
     'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ],
-    //int taos_subfields_count(TAOS_SUB *tsub);
-    'taos_subfields_count': [ ref.types.int, [ref.types.void_ptr ] ],
-    //TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub);
-    'taos_fetch_subfields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ],
 
     // Continuous Query
     //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
@@ -362,7 +357,7 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
   blocks.fill(null);
   num_of_rows = Math.abs(num_of_rows);
   let offset = 0;
-  pblock = pblock.deref()
+  pblock = pblock.deref();
   for (let i = 0; i < fields.length; i++) {
 
     if (!convertFunctions[fields[i]['type']] ) {
@@ -472,64 +467,40 @@ CTaosInterface.prototype.getClientInfo = function getClientInfo() {
 }
 
 // Subscription
-CTaosInterface.prototype.subscribe = function subscribe(host=null, user="root", password="taosdata", db=null, table=null, time=null, mseconds=null) {
+CTaosInterface.prototype.subscribe = function subscribe(connection, restart, topic, sql, interval) {
-  let dbOrig = db;
+  let topicOrig = topic;
-  let tableOrig = table;
+  let sqlOrig = sql;
   try {
-    host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL);
+    sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL);
   }
   catch(err) {
-    throw "Attribute Error: host is expected as a str";
+    throw "Attribute Error: sql is expected as a str";
   }
   try {
-    user = ref.allocCString(user)
+    topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL);
   }
   catch(err) {
-    throw "Attribute Error: user is expected as a str";
+    throw TypeError("topic is expected as a str");
   }
-  try {
+
-    password = ref.allocCString(password);
+  restart = ref.alloc(ref.types.int, restart);
-  }
+
-  catch(err) {
+  let subscription = this.libtaos.taos_subscribe(connection, restart, topic, sql, null, null, interval);
-    throw "Attribute Error: password is expected as a str";
-  }
-  try {
-    db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL);
-  }
-  catch(err) {
-    throw "Attribute Error: db is expected as a str";
-  }
-  try {
-    table = table != null ? ref.allocCString(table) : ref.alloc(ref.types.char_ptr, ref.NULL);
-  }
-  catch(err) {
-    throw TypeError("table is expected as a str");
-  }
-  try {
-    mseconds = ref.alloc(ref.types.int, mseconds);
-  }
-  catch(err) {
-    throw TypeError("mseconds is expected as an int");
-  }
-  //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds);
-  let subscription = this.libtaos.taos_subscribe(host, user, password, db, table, time, mseconds);
   if (ref.isNull(subscription)) {
     throw new errors.TDError('Failed to subscribe to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig);
   }
   else {
-    console.log('Successfully subscribed to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig);
+    console.log('Successfully subscribed to TDengine - Topic: ' + topicOrig);
   }
   return subscription;
 }
-CTaosInterface.prototype.subFieldsCount = function subFieldsCount(subscription) {
+
-  return this.libtaos.taos_subfields_count(subscription);
+CTaosInterface.prototype.consume = function consume(subscription) {
-}
+  let result = this.libtaos.taos_consume(subscription);
-CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription) {
-  let pfields = this.libtaos.taos_fetch_subfields(subscription);
-  let pfieldscount = this.subFieldsCount(subscription);
   let fields = [];
+  let pfields = this.fetchFields(result);
   if (ref.isNull(pfields) == false) {
-    pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0);
+    pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0);
     for (let i = 0; i < pfields.length; i += 68) {
       //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type
       fields.push( {
@@ -539,27 +510,23 @@ CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription)
       })
     }
   }
-  return fields;
+
+  let data = [];
+  while(true) {
+    let { blocks, num_of_rows } = this.fetchBlock(result, fields);
+    if (num_of_rows == 0) {
+      break;
     }
-CTaosInterface.prototype.consume = function consume(subscription) {
+    for (let i = 0; i < num_of_rows; i++) {
-  let row = this.libtaos.taos_consume(subscription);
+      data.push([]);
-  let fields = this.fetchSubFields(subscription);
+      let rowBlock = new Array(fields.length);
-  //let isMicro = (cti.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
+      for (let j = 0; j < fields.length; j++) {
-  let isMicro = false; //no supported function for determining precision?
+        rowBlock[j] = blocks[j][i];
-  let blocks = new Array(fields.length);
-  blocks.fill(null);
-  let numOfRows2 = 1; //Math.abs(numOfRows2);
-  let offset = 0;
-  if (numOfRows2 > 0){
-    for (let i = 0; i < fields.length; i++) {
-      if (!convertFunctions[fields[i]['type']] ) {
-        throw new errors.DatabaseError("Invalid data type returned from database");
       }
-      blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro);
+      data[data.length-1] = (rowBlock);
-      offset += fields[i]['bytes'] * numOfRows2;
     }
   }
-  return {blocks:blocks, fields:fields};
+  return { data: data, fields: fields, result: result };
 }
 CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) {
   //void taos_unsubscribe(TAOS_SUB *tsub);

src/connector/nodejs/nodetaos/cursor.js

@@ -405,18 +405,16 @@ TDengineCursor.prototype.getClientInfo = function getClientInfo() {
 /**
  * Subscribe to a table from a database in TDengine.
  * @param {Object} config - A configuration object containing the configuration options for the subscription
- * @param {string} config.host - The host to subscribe to
+ * @param {string} config.restart - whether or not to continue a subscription if it already exits, otherwise start from beginning
- * @param {string} config.user - The user to subscribe as
+ * @param {string} config.topic - The unique identifier of a subscription
- * @param {string} config.password - The password for the said user
+ * @param {string} config.sql - A sql statement for data query
- * @param {string} config.db - The db containing the table to subscribe to
+ * @param {string} config.interval - The pulling interval
- * @param {string} config.table - The name of the table to subscribe to
- * @param {number} config.time - The start time to start a subscription session
- * @param {number} config.mseconds - The pulling period of the subscription session
  * @return {Buffer} A buffer pointing to the subscription session handle
  * @since 1.3.0
  */
 TDengineCursor.prototype.subscribe = function subscribe(config) {
-  return this._chandle.subscribe(config.host, config.user, config.password, config.db, config.table, config.time, config.mseconds);
+  let restart = config.restart ? 1 : 0;
+  return this._chandle.subscribe(this._connection._conn, restart, config.topic, config.sql, config.interval);
 };
 /**
  * An infinite loop that consumes the latest data and calls a callback function that is provided.
@@ -426,18 +424,8 @@ TDengineCursor.prototype.subscribe = function subscribe(config) {
  */
 TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) {
   while (true) {
-    let res = this._chandle.consume(subscription);
+    let { data, fields, result} = this._chandle.consume(subscription);
-    let data = [];
+    callback(data, fields, result);
-    let num_of_rows = res.blocks[0].length;
-    for (let j = 0; j < num_of_rows; j++) {
-      data.push([]);
-      let rowBlock = new Array(res.fields.length);
-      for (let k = 0; k < res.fields.length; k++) {
-        rowBlock[k] = res.blocks[k][j];
-      }
-      data[data.length-1] = rowBlock;
-    }
-    callback(data, res.fields, subscription);
   }
 }
 /**

src/connector/nodejs/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "td-connector",
-  "version": "1.5.0",
+  "version": "1.6.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

src/connector/nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "td-connector",
-  "version": "1.5.0",
+  "version": "1.6.1",
   "description": "A Node.js connector for TDengine.",
   "main": "tdengine.js",
   "scripts": {

src/connector/nodejs/test/test.js

@@ -33,12 +33,12 @@ for (let i = 0; i < 10000; i++) {
                     parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // Int
                     parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // BigInt
                     parseFloat( R(-3.4E38, 3.4E38) ), // Float
-                    parseFloat( R(-1.7E308, 1.7E308) ), // Double
+                    parseFloat( R(-1.7E30, 1.7E30) ), // Double
                     "\"Long Binary\"", // Binary
                     parseInt( R(-32767, 32767) ), // Small Int
                     parseInt( R(-127, 127) ), // Tiny Int
                     randomBool(),
-                    "\"Nchars 一些中文字幕\""]; // Bool
+                    "\"Nchars\""]; // Bool
   c1.execute('insert into td_connector_test.all_types values(' + insertData.join(',') + ' );', {quiet:true});
   if (i % 1000 == 0) {
     console.log("Insert # " , i);

src/connector/nodejs/test/testSubscribe.js

@@ -0,0 +1,16 @@
+const taos = require('../tdengine');
+var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:10});
+var c1 = conn.cursor();
+let stime = new Date();
+let interval = 1000;
+c1.execute('use td_connector_test');
+let sub = c1.subscribe({
+  restart: true,
+  sql: "select AVG(_int) from td_connector_test.all_Types;",
+  topic: 'all_Types',
+  interval: 1000
+});
+
+c1.consumeData(sub, (data, fields) => {
+  console.log(data);
+});

src/connector/python/linux/python2/taos/cinterface.py

@@ -142,12 +142,15 @@ class CTaosInterface(object):
     libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField)
     libtaos.taos_init.restype = None
     libtaos.taos_connect.restype = ctypes.c_void_p
-    libtaos.taos_use_result.restype = ctypes.c_void_p
+    #libtaos.taos_use_result.restype = ctypes.c_void_p
     libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p)
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
     libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
+    libtaos.taos_free_result.restype = None
+    libtaos.taos_errno.restype = ctypes.c_int
+    libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p)
 
     def __init__(self, config=None):
         '''
@@ -251,10 +254,10 @@ class CTaosInterface(object):
         #     CTaosInterface.libtaos.close(connection)
       
     @staticmethod
-    def affectedRows(connection):
+    def affectedRows(result):
         """The affected rows after runing query
         """
-        return CTaosInterface.libtaos.taos_affected_rows(connection)
+        return CTaosInterface.libtaos.taos_affected_rows(result)
 
     @staticmethod
     def subscribe(connection, restart, topic, sql, interval):
@@ -292,18 +295,17 @@ class CTaosInterface(object):
         CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0)
 
     @staticmethod
-    def useResult(connection):
+    def useResult(result):
         '''Use result after calling self.query
         '''
-        result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection))
         fields = []
         pfields = CTaosInterface.fetchFields(result)
-        for i in range(CTaosInterface.fieldsCount(connection)):
+        for i in range(CTaosInterface.fieldsCount(result)):
             fields.append({'name': pfields[i].name.decode('utf-8'),
                            'bytes': pfields[i].bytes,
                            'type': ord(pfields[i].type)})
 
-        return result, fields
+        return fields
 
     @staticmethod
     def fetchBlock(result, fields):
@@ -337,8 +339,8 @@ class CTaosInterface(object):
         result.value = None
 
     @staticmethod
-    def fieldsCount(connection):
+    def fieldsCount(result):
-        return CTaosInterface.libtaos.taos_field_count(connection)
+        return CTaosInterface.libtaos.taos_field_count(result)
 
     @staticmethod
     def fetchFields(result):
@@ -386,29 +388,30 @@ class CTaosInterface(object):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 
     @staticmethod
-    def errno(connection):
+    def errno(result):
         """Return the error number.
         """
-        return CTaosInterface.libtaos.taos_errno(connection)
+        return CTaosInterface.libtaos.taos_errno(result)
 
     @staticmethod
-    def errStr(connection):
+    def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(connection)
+        return CTaosInterface.libtaos.taos_errstr(result)
 
 
 if __name__ == '__main__':
     cinter = CTaosInterface()
     conn = cinter.connect()
+    result = cinter.query(conn, 'show databases')
 
-    print('Query return value: {}'.format(cinter.query(conn, 'show databases')))
+    print('Query Affected rows: {}'.format(cinter.affectedRows(result)))
-    print('Affected rows: {}'.format(cinter.affectedRows(conn)))
 
-    result, des = CTaosInterface.useResult(conn)
+    fields = CTaosInterface.useResult(result)
 
-    data, num_of_rows = CTaosInterface.fetchBlock(result, des)
+    data, num_of_rows = CTaosInterface.fetchBlock(result, fields)
 
     print(data)
 
+    cinter.freeResult(result)
     cinter.close(conn)

src/connector/python/linux/python2/taos/connection.py

@@ -78,9 +78,7 @@ class TDengineConnection(object):
     def clear_result_set(self):
         """Clear unused result set on this connection.
         """
-        result = self._chandle.useResult(self._conn)[0]
+        pass
-        if result:
-            self._chandle.freeResult(result)
 
 if __name__ == "__main__":
     conn = TDengineConnection(host='192.168.1.107')

src/connector/python/linux/python2/taos/constants.py

@@ -28,6 +28,6 @@ class FieldType(object):
     C_FLOAT_NULL = float('nan')
     C_DOUBLE_NULL = float('nan')
     C_BINARY_NULL = bytearray([int('0xff', 16)])
-    # Time precision definition
+    # Timestamp precision definition
     C_TIMESTAMP_MILLI = 0
     C_TIMESTAMP_MICRO = 1

src/connector/python/linux/python2/taos/cursor.py

@@ -1,5 +1,7 @@
 from .cinterface import CTaosInterface
 from .error import *
+from .constants import FieldType
+
 
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
@@ -32,6 +34,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -44,7 +47,8 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchBlock(
+                self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -80,13 +84,15 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
         if self._connection is None:
             return False
 
-        self._connection.clear_result_set()
         self._reset_result()
         self._connection = None
 
@@ -102,23 +108,36 @@ class TDengineCursor(object):
             # TODO : change the exception raised here
             raise ProgrammingError("Cursor is not connected")
 
-        self._connection.clear_result_set()
         self._reset_result()
 
         stmt = operation
         if params is not None:
             pass
 
-        res = CTaosInterface.query(self._connection._conn, stmt)
+        # global querySeqNum
-        if res == 0:
+        # querySeqNum += 1
-            if CTaosInterface.fieldsCount(self._connection._conn) == 0:
+        # localSeqNum = querySeqNum # avoid raice condition
-                self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
+        # print("   >> Exec Query ({}): {}".format(localSeqNum, str(stmt)))
-                return CTaosInterface.affectedRows(self._connection._conn)
+        self._result = CTaosInterface.query(self._connection._conn, stmt)
+        # print("   << Query ({}) Exec Done".format(localSeqNum))
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
+        errno = CTaosInterface.libtaos.taos_errno(self._result)
+        if errno == 0:
+            if CTaosInterface.fieldsCount(self._result) == 0:
+                self._affected_rows += CTaosInterface.affectedRows(
+                    self._result )
+                return CTaosInterface.affectedRows(self._result )
             else:
-                self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
+                self._fields = CTaosInterface.useResult(
+                    self._result)
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
+            raise ProgrammingError(
+                CTaosInterface.errStr(
+                    self._result ), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@@ -130,6 +149,37 @@ class TDengineCursor(object):
         """
         pass
 
+    def istype(self, col, dataType):
+        if (dataType.upper() == "BOOL"):
+            if (self._description[col][1] == FieldType.C_BOOL):
+                return True
+        if (dataType.upper() == "TINYINT"):
+            if (self._description[col][1] == FieldType.C_TINYINT):
+                return True
+        if (dataType.upper() == "INT"):
+            if (self._description[col][1] == FieldType.C_INT):
+                return True
+        if (dataType.upper() == "BIGINT"):
+            if (self._description[col][1] == FieldType.C_INT):
+                return True
+        if (dataType.upper() == "FLOAT"):
+            if (self._description[col][1] == FieldType.C_FLOAT):
+                return True
+        if (dataType.upper() == "DOUBLE"):
+            if (self._description[col][1] == FieldType.C_DOUBLE):
+                return True
+        if (dataType.upper() == "BINARY"):
+            if (self._description[col][1] == FieldType.C_BINARY):
+                return True
+        if (dataType.upper() == "TIMESTAMP"):
+            if (self._description[col][1] == FieldType.C_TIMESTAMP):
+                return True
+        if (dataType.upper() == "NCHAR"):
+            if (self._description[col][1] == FieldType.C_NCHAR):
+                return True
+
+        return False
+
     def fetchmany(self):
         pass
 
@@ -142,18 +192,17 @@ class TDengineCursor(object):
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
-            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, num_of_fields = CTaosInterface.fetchBlock(
-            if num_of_fields == 0: break
+                self._result, self._fields)
+            if num_of_fields == 0:
+                break
             self._rowcount += num_of_fields
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
 
-        self._connection.clear_result_set()
 
         return list(map(tuple, zip(*buffer)))
 
-
-
     def nextset(self):
         """
         """
@@ -170,6 +219,8 @@ class TDengineCursor(object):
         """
         self._description = None
         self._rowcount = -1
+        if self._result is not None:
+            CTaosInterface.freeResult(self._result)
         self._result = None
         self._fields = None
         self._block = None
@@ -182,6 +233,7 @@ class TDengineCursor(object):
         """
         self._description = []
         for ele in self._fields:
-            self._description.append((ele['name'], ele['type'], None, None, None, None, False))
+            self._description.append(
+                (ele['name'], ele['type'], None, None, None, None, False))
 
         return self._result

src/connector/python/linux/python3/taos/cinterface.py

@@ -130,9 +130,9 @@ _CONVERT_FUNC = {
 
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
-    _fields_ = [('name', ctypes.c_char * 64),
+    _fields_ = [('name', ctypes.c_char * 65),
-                ('bytes', ctypes.c_short),
+                ('type', ctypes.c_char),
-                ('type', ctypes.c_char)]
+                ('bytes', ctypes.c_short)]
 
 # C interface class
 class CTaosInterface(object):
@@ -142,12 +142,15 @@ class CTaosInterface(object):
     libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField)
     libtaos.taos_init.restype = None
     libtaos.taos_connect.restype = ctypes.c_void_p
-    libtaos.taos_use_result.restype = ctypes.c_void_p
+    #libtaos.taos_use_result.restype = ctypes.c_void_p
     libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p)
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
     libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
+    libtaos.taos_free_result.restype = None
+    libtaos.taos_errno.restype = ctypes.c_int
+    libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p)
 
     def __init__(self, config=None):
         '''
@@ -251,10 +254,10 @@ class CTaosInterface(object):
         #     CTaosInterface.libtaos.close(connection)
       
     @staticmethod
-    def affectedRows(connection):
+    def affectedRows(result):
         """The affected rows after runing query
         """
-        return CTaosInterface.libtaos.taos_affected_rows(connection)
+        return CTaosInterface.libtaos.taos_affected_rows(result)
 
     @staticmethod
     def subscribe(connection, restart, topic, sql, interval):
@@ -292,18 +295,17 @@ class CTaosInterface(object):
         CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0)
 
     @staticmethod
-    def useResult(connection):
+    def useResult(result):
         '''Use result after calling self.query
         '''
-        result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection))
         fields = []
         pfields = CTaosInterface.fetchFields(result)
-        for i in range(CTaosInterface.fieldsCount(connection)):
+        for i in range(CTaosInterface.fieldsCount(result)):
             fields.append({'name': pfields[i].name.decode('utf-8'),
                            'bytes': pfields[i].bytes,
                            'type': ord(pfields[i].type)})
 
-        return result, fields
+        return fields
 
     @staticmethod
     def fetchBlock(result, fields):
@@ -337,8 +339,8 @@ class CTaosInterface(object):
         result.value = None
 
     @staticmethod
-    def fieldsCount(connection):
+    def fieldsCount(result):
-        return CTaosInterface.libtaos.taos_field_count(connection)
+        return CTaosInterface.libtaos.taos_field_count(result)
 
     @staticmethod
     def fetchFields(result):
@@ -386,29 +388,30 @@ class CTaosInterface(object):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 
     @staticmethod
-    def errno(connection):
+    def errno(result):
         """Return the error number.
         """
-        return CTaosInterface.libtaos.taos_errno(connection)
+        return CTaosInterface.libtaos.taos_errno(result)
 
     @staticmethod
-    def errStr(connection):
+    def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(connection).decode('utf-8')
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':
     cinter = CTaosInterface()
     conn = cinter.connect()
+    result = cinter.query(conn, 'show databases')
 
-    print('Query return value: {}'.format(cinter.query(conn, 'show databases')))
+    print('Query Affected rows: {}'.format(cinter.affectedRows(result)))
-    print('Affected rows: {}'.format(cinter.affectedRows(conn)))
 
-    result, des = CTaosInterface.useResult(conn)
+    fields = CTaosInterface.useResult(result)
 
-    data, num_of_rows = CTaosInterface.fetchBlock(result, des)
+    data, num_of_rows = CTaosInterface.fetchBlock(result, fields)
 
     print(data)
 
+    cinter.freeResult(result)
     cinter.close(conn)

src/connector/python/linux/python3/taos/connection.py

@@ -78,9 +78,7 @@ class TDengineConnection(object):
     def clear_result_set(self):
         """Clear unused result set on this connection.
         """
-        result = self._chandle.useResult(self._conn)[0]
+        pass
-        if result:
-            self._chandle.freeResult(result)
 
 if __name__ == "__main__":
     conn = TDengineConnection(host='192.168.1.107')

src/connector/python/linux/python3/taos/cursor.py

@@ -1,5 +1,9 @@
 from .cinterface import CTaosInterface
 from .error import *
+from .constants import FieldType
+
+# querySeqNum = 0
+
 
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
@@ -32,6 +36,7 @@ class TDengineCursor(object):
         self._block_rows = -1
         self._block_iter = 0
         self._affected_rows = 0
+        self._logfile = ""
 
         if connection is not None:
             self._connection = connection
@@ -44,7 +49,8 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchBlock(
+                self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -80,13 +86,15 @@ class TDengineCursor(object):
         """
         pass
 
+    def log(self, logfile):
+        self._logfile = logfile
+
     def close(self):
         """Close the cursor.
         """
         if self._connection is None:
             return False
 
-        self._connection.clear_result_set()
         self._reset_result()
         self._connection = None
 
@@ -102,23 +110,36 @@ class TDengineCursor(object):
             # TODO : change the exception raised here
             raise ProgrammingError("Cursor is not connected")
 
-        self._connection.clear_result_set()
         self._reset_result()
 
         stmt = operation
         if params is not None:
             pass
 
-        res = CTaosInterface.query(self._connection._conn, stmt)
+        # global querySeqNum
-        if res == 0:
+        # querySeqNum += 1
-            if CTaosInterface.fieldsCount(self._connection._conn) == 0:
+        # localSeqNum = querySeqNum # avoid raice condition
-                self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
+        # print("   >> Exec Query ({}): {}".format(localSeqNum, str(stmt)))
-                return CTaosInterface.affectedRows(self._connection._conn)
+        self._result = CTaosInterface.query(self._connection._conn, stmt)
+        # print("   << Query ({}) Exec Done".format(localSeqNum))
+        if (self._logfile):
+            with open(self._logfile, "a") as logfile:
+                logfile.write("%s;\n" % operation)
+
+        errno = CTaosInterface.libtaos.taos_errno(self._result)
+        if errno == 0:
+            if CTaosInterface.fieldsCount(self._result) == 0:
+                self._affected_rows += CTaosInterface.affectedRows(
+                    self._result )
+                return CTaosInterface.affectedRows(self._result )
             else:
-                self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
+                self._fields = CTaosInterface.useResult(
+                    self._result)
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
+            raise ProgrammingError(
+                CTaosInterface.errStr(
+                    self._result), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@@ -133,6 +154,37 @@ class TDengineCursor(object):
     def fetchmany(self):
         pass
 
+    def istype(self, col, dataType):
+        if (dataType.upper() == "BOOL"):
+            if (self._description[col][1] == FieldType.C_BOOL):
+                return True
+        if (dataType.upper() == "TINYINT"):
+            if (self._description[col][1] == FieldType.C_TINYINT):
+                return True
+        if (dataType.upper() == "INT"):
+            if (self._description[col][1] == FieldType.C_INT):
+                return True
+        if (dataType.upper() == "BIGINT"):
+            if (self._description[col][1] == FieldType.C_INT):
+                return True
+        if (dataType.upper() == "FLOAT"):
+            if (self._description[col][1] == FieldType.C_FLOAT):
+                return True
+        if (dataType.upper() == "DOUBLE"):
+            if (self._description[col][1] == FieldType.C_DOUBLE):
+                return True
+        if (dataType.upper() == "BINARY"):
+            if (self._description[col][1] == FieldType.C_BINARY):
+                return True
+        if (dataType.upper() == "TIMESTAMP"):
+            if (self._description[col][1] == FieldType.C_TIMESTAMP):
+                return True
+        if (dataType.upper() == "NCHAR"):
+            if (self._description[col][1] == FieldType.C_NCHAR):
+                return True
+
+        return False
+
     def fetchall(self):
         """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.
         """
@@ -142,18 +194,16 @@ class TDengineCursor(object):
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
-            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, num_of_fields = CTaosInterface.fetchBlock(
-            if num_of_fields == 0: break
+                self._result, self._fields)
+            if num_of_fields == 0:
+                break
             self._rowcount += num_of_fields
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
 
-        self._connection.clear_result_set()
-        
         return list(map(tuple, zip(*buffer)))
 
-
-
     def nextset(self):
         """
         """
@@ -170,6 +220,8 @@ class TDengineCursor(object):
         """
         self._description = None
         self._rowcount = -1
+        if self._result is not None:
+            CTaosInterface.freeResult(self._result)
         self._result = None
         self._fields = None
         self._block = None
@@ -182,6 +234,8 @@ class TDengineCursor(object):
         """
         self._description = []
         for ele in self._fields:
-            self._description.append((ele['name'], ele['type'], None, None, None, None, False))
+            self._description.append(
+                (ele['name'], ele['type'], None, None, None, None, False))
 
         return self._result
+

src/connector/python/linux/python3/test.py

@@ -0,0 +1,21 @@
+from taos.cinterface import CTaosInterface
+from taos.error import *
+from taos.subscription import TDengineSubscription
+from taos.connection import TDengineConnection
+
+
+if __name__ == '__main__':
+    conn = TDengineConnection(
+        host="127.0.0.1", user="root", password="taosdata", database="test")
+
+    # Generate a cursor object to run SQL commands
+    sub = conn.subscribe(False, "test", "select * from log0601;", 1000)
+
+    for i in range(100):
+        print(i)
+        data = sub.consume()
+        for d in data:
+            print(d)
+
+    sub.close()
+    conn.close()

src/connector/python/windows/python2/taos/cinterface.py

@@ -142,12 +142,15 @@ class CTaosInterface(object):
     libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField)
     libtaos.taos_init.restype = None
     libtaos.taos_connect.restype = ctypes.c_void_p
-    libtaos.taos_use_result.restype = ctypes.c_void_p
+    #libtaos.taos_use_result.restype = ctypes.c_void_p
     libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p)
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
     libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
+    libtaos.taos_free_result.restype = None
+    libtaos.taos_errno.restype = ctypes.c_int
+    libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p)
 
     def __init__(self, config=None):
         '''
@@ -251,10 +254,10 @@ class CTaosInterface(object):
         #     CTaosInterface.libtaos.close(connection)
 
     @staticmethod
-    def affectedRows(connection):
+    def affectedRows(result):
         """The affected rows after runing query
         """
-        return CTaosInterface.libtaos.taos_affected_rows(connection)
+        return CTaosInterface.libtaos.taos_affected_rows(result)
 
     @staticmethod
     def subscribe(connection, restart, topic, sql, interval):
@@ -292,18 +295,17 @@ class CTaosInterface(object):
         CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0)
 
     @staticmethod
-    def useResult(connection):
+    def useResult(result):
         '''Use result after calling self.query
         '''
-        result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection))
         fields = []
         pfields = CTaosInterface.fetchFields(result)
-        for i in range(CTaosInterface.fieldsCount(connection)):
+        for i in range(CTaosInterface.fieldsCount(result)):
             fields.append({'name': pfields[i].name.decode('utf-8'),
                            'bytes': pfields[i].bytes,
                            'type': ord(pfields[i].type)})
 
-        return result, fields
+        return fields
 
     @staticmethod
     def fetchBlock(result, fields):
@@ -337,8 +339,8 @@ class CTaosInterface(object):
         result.value = None
 
     @staticmethod
-    def fieldsCount(connection):
+    def fieldsCount(result):
-        return CTaosInterface.libtaos.taos_field_count(connection)
+        return CTaosInterface.libtaos.taos_field_count(result)
 
     @staticmethod
     def fetchFields(result):
@@ -386,29 +388,30 @@ class CTaosInterface(object):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 
     @staticmethod
-    def errno(connection):
+    def errno(result):
         """Return the error number.
         """
-        return CTaosInterface.libtaos.taos_errno(connection)
+        return CTaosInterface.libtaos.taos_errno(result)
 
     @staticmethod
-    def errStr(connection):
+    def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(connection)
+        return CTaosInterface.libtaos.taos_errstr(result)
 
 
 if __name__ == '__main__':
     cinter = CTaosInterface()
     conn = cinter.connect()
+    result = cinter.query(conn, 'show databases')
 
-    print('Query return value: {}'.format(cinter.query(conn, 'show databases')))
+    print('Query Affected rows: {}'.format(cinter.affectedRows(result)))
-    print('Affected rows: {}'.format(cinter.affectedRows(conn)))
 
-    result, des = CTaosInterface.useResult(conn)
+    fields = CTaosInterface.useResult(result)
 
-    data, num_of_rows = CTaosInterface.fetchBlock(result, des)
+    data, num_of_rows = CTaosInterface.fetchBlock(result, fields)
 
     print(data)
 
+    cinter.freeResult(result)
     cinter.close(conn)

src/connector/python/windows/python2/taos/connection.py

@@ -79,9 +79,7 @@ class TDengineConnection(object):
     def clear_result_set(self):
         """Clear unused result set on this connection.
         """
-        result = self._chandle.useResult(self._conn)[0]
+        pass
-        if result:
-            self._chandle.freeResult(result)
 
 if __name__ == "__main__":
     conn = TDengineConnection(host='192.168.1.107')

src/connector/python/windows/python2/taos/cursor.py

@@ -86,7 +86,6 @@ class TDengineCursor(object):
         if self._connection is None:
             return False
         
-        self._connection.clear_result_set()
         self._reset_result()
         self._connection = None
 
@@ -102,23 +101,23 @@ class TDengineCursor(object):
             # TODO : change the exception raised here
             raise ProgrammingError("Cursor is not connected")
         
-        self._connection.clear_result_set()
         self._reset_result()
 
         stmt = operation
         if params is not None:
             pass
         
-        res = CTaosInterface.query(self._connection._conn, stmt)
+        self._result = CTaosInterface.query(self._connection._conn, stmt)
-        if res == 0:
+        errno = CTaosInterface.libtaos.taos_errno(self._result)
-            if CTaosInterface.fieldsCount(self._connection._conn) == 0:
+        if errno == 0:
-                self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
+            if CTaosInterface.fieldsCount(self._result) == 0:
-                return CTaosInterface.affectedRows(self._connection._conn)
+                self._affected_rows += CTaosInterface.affectedRows(self._result)
+                return CTaosInterface.affectedRows(self._result )
             else:
-                self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
+                self._fields = CTaosInterface.useResult(self._result)
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
+            raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@@ -148,7 +147,6 @@ class TDengineCursor(object):
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
 
-        self._connection.clear_result_set()
         
         return list(map(tuple, zip(*buffer)))
 
@@ -170,6 +168,8 @@ class TDengineCursor(object):
         """
         self._description = None
         self._rowcount = -1
+        if self._result is not None:
+            CTaosInterface.freeResult(self._result)        
         self._result = None
         self._fields = None
         self._block = None

src/connector/python/windows/python3/taos/cinterface.py

@@ -142,12 +142,15 @@ class CTaosInterface(object):
     libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField)
     libtaos.taos_init.restype = None
     libtaos.taos_connect.restype = ctypes.c_void_p
-    libtaos.taos_use_result.restype = ctypes.c_void_p
+    #libtaos.taos_use_result.restype = ctypes.c_void_p
     libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p)
     libtaos.taos_errstr.restype = ctypes.c_char_p
     libtaos.taos_subscribe.restype = ctypes.c_void_p
     libtaos.taos_consume.restype = ctypes.c_void_p
     libtaos.taos_fetch_lengths.restype = ctypes.c_void_p
+    libtaos.taos_free_result.restype = None
+    libtaos.taos_errno.restype = ctypes.c_int
+    libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p)
 
     def __init__(self, config=None):
         '''
@@ -251,10 +254,10 @@ class CTaosInterface(object):
         #     CTaosInterface.libtaos.close(connection)
 
     @staticmethod
-    def affectedRows(connection):
+    def affectedRows(result):
         """The affected rows after runing query
         """
-        return CTaosInterface.libtaos.taos_affected_rows(connection)
+        return CTaosInterface.libtaos.taos_affected_rows(result)
 
     @staticmethod
     def subscribe(connection, restart, topic, sql, interval):
@@ -292,18 +295,17 @@ class CTaosInterface(object):
         CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0)
 
     @staticmethod
-    def useResult(connection):
+    def useResult(result):
         '''Use result after calling self.query
         '''
-        result = ctypes.c_void_p(CTaosInterface.libtaos.taos_use_result(connection))
         fields = []
         pfields = CTaosInterface.fetchFields(result)
-        for i in range(CTaosInterface.fieldsCount(connection)):
+        for i in range(CTaosInterface.fieldsCount(result)):
             fields.append({'name': pfields[i].name.decode('utf-8'),
                            'bytes': pfields[i].bytes,
                            'type': ord(pfields[i].type)})
 
-        return result, fields
+        return fields
 
     @staticmethod
     def fetchBlock(result, fields):
@@ -337,8 +339,8 @@ class CTaosInterface(object):
         result.value = None
 
     @staticmethod
-    def fieldsCount(connection):
+    def fieldsCount(result):
-        return CTaosInterface.libtaos.taos_field_count(connection)
+        return CTaosInterface.libtaos.taos_field_count(result)
 
     @staticmethod
     def fetchFields(result):
@@ -386,29 +388,30 @@ class CTaosInterface(object):
     #         return (ctypes.cast(data,  ctypes.c_char_p).value).rstrip('\x00')
 
     @staticmethod
-    def errno(connection):
+    def errno(result):
         """Return the error number.
         """
-        return CTaosInterface.libtaos.taos_errno(connection)
+        return CTaosInterface.libtaos.taos_errno(result)
 
     @staticmethod
-    def errStr(connection):
+    def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(connection).decode('utf-8')
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':
     cinter = CTaosInterface()
     conn = cinter.connect()
+    result = cinter.query(conn, 'show databases')
 
-    print('Query return value: {}'.format(cinter.query(conn, 'show databases')))
+    print('Query Affected rows: {}'.format(cinter.affectedRows(result)))
-    print('Affected rows: {}'.format(cinter.affectedRows(conn)))
 
-    result, des = CTaosInterface.useResult(conn)
+    fields = CTaosInterface.useResult(result)
 
-    data, num_of_rows = CTaosInterface.fetchBlock(result, des)
+    data, num_of_rows = CTaosInterface.fetchBlock(result, fields)
 
     print(data)
 
+    cinter.freeResult(result)
     cinter.close(conn)

src/connector/python/windows/python3/taos/connection.py

@@ -79,9 +79,7 @@ class TDengineConnection(object):
     def clear_result_set(self):
         """Clear unused result set on this connection.
         """
-        result = self._chandle.useResult(self._conn)[0]
+        pass
-        if result:
-            self._chandle.freeResult(result)
 
 if __name__ == "__main__":
     conn = TDengineConnection(host='192.168.1.107')

src/connector/python/windows/python3/taos/cursor.py

@@ -86,7 +86,6 @@ class TDengineCursor(object):
         if self._connection is None:
             return False
         
-        self._connection.clear_result_set()
         self._reset_result()
         self._connection = None
 
@@ -102,23 +101,23 @@ class TDengineCursor(object):
             # TODO : change the exception raised here
             raise ProgrammingError("Cursor is not connected")
         
-        self._connection.clear_result_set()
         self._reset_result()
 
         stmt = operation
         if params is not None:
             pass
         
-        res = CTaosInterface.query(self._connection._conn, stmt)
+        self._result = CTaosInterface.query(self._connection._conn, stmt)
-        if res == 0:
+        errno = CTaosInterface.libtaos.taos_errno(self._result)
-            if CTaosInterface.fieldsCount(self._connection._conn) == 0:
+        if errno == 0:
-                self._affected_rows += CTaosInterface.affectedRows(self._connection._conn)
+            if CTaosInterface.fieldsCount(self._result) == 0:
-                return CTaosInterface.affectedRows(self._connection._conn)
+                self._affected_rows += CTaosInterface.affectedRows(self._result )
+                return CTaosInterface.affectedRows(self._result )
             else:
-                self._result, self._fields = CTaosInterface.useResult(self._connection._conn)
+                self._fields = CTaosInterface.useResult(self._result )
                 return self._handle_result()
         else:
-            raise ProgrammingError(CTaosInterface.errStr(self._connection._conn))
+            raise ProgrammingError(CTaosInterface.errStr(self._result ), errno)
 
     def executemany(self, operation, seq_of_parameters):
         """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.
@@ -148,7 +147,6 @@ class TDengineCursor(object):
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
 
-        self._connection.clear_result_set()
         
         return list(map(tuple, zip(*buffer)))
 
@@ -170,6 +168,8 @@ class TDengineCursor(object):
         """
         self._description = None
         self._rowcount = -1
+        if self._result is not None:
+            CTaosInterface.freeResult(self._result)
         self._result = None
         self._fields = None
         self._block = None