refactor:merge from 3.0

.devcontainer/Dockerfile

@@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
 # [Optional] Uncomment this section to install additional packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
-RUN apt-get update && apt-get -y install tree vim tmux
+RUN apt-get update && apt-get -y install tree vim tmux python3-pip

.gitignore

@@ -40,6 +40,7 @@ taosdalipu/
 Target/
 *.failed
 *.sql
+sim
 sim/
 psim/
 pysim/
@@ -107,3 +108,4 @@ TAGS
 contrib/*
 !contrib/CMakeLists.txt
 !contrib/test
+sql

CMakeLists.txt

@@ -7,7 +7,7 @@ project(
 )
 
 if (NOT DEFINED TD_SOURCE_DIR)
-  set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} )
+  set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
 endif()
 
 set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")

Jenkinsfile

@@ -113,7 +113,7 @@ pipeline {
               '''
               sh'''
               cd ${WKC}/debug
-              ctest
+              ctest -VV
               '''
             }
           }

Jenkinsfile2

@@ -88,6 +88,12 @@ def pre_test(){
     cmake .. > /dev/null
     make -j4> /dev/null
     '''
+    sh'''
+    cd ${WKPY}
+    git reset --hard
+    git pull
+    pip3 install .
+    '''
     return 1
 }
 
@@ -97,6 +103,7 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
+      WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
   }
   stages {
       stage('pre_build'){
@@ -113,12 +120,17 @@ pipeline {
             timeout(time: 45, unit: 'MINUTES'){
               pre_test()
               sh'''
-              cd ${WKC}/tests
+              cd ${WKC}/debug
-              ./test-all.sh b1fq
+              ctest -VV
               '''
               sh'''
-              cd ${WKC}/debug
+              export LD_LIBRARY_PATH=${WKC}/debug/build/lib
-              ctest
+              cd ${WKC}/tests/system-test
+              ./fulltest.sh
+              '''
+              sh'''
+              cd ${WKC}/tests
+              ./test-all.sh b1fq
               '''
             }
           }

cmake/cmake.define

@@ -1,5 +1,9 @@
 cmake_minimum_required(VERSION 3.16)
 
+set(CMAKE_VERBOSE_MAKEFILE OFF)
+
+SET(BUILD_SHARED_LIBS "OFF")
+
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
@@ -42,12 +46,14 @@ ENDIF ()
 
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+    SET(COMMON_FLAGS "/W3 /D_WIN32")
-    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
 
-    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+    # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
-      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
-    ENDIF ()
+    # ENDIF ()
+
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
 
 ELSE ()
     IF (${SANITIZER} MATCHES "true")
@@ -63,7 +69,13 @@ ELSE ()
     IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
         ADD_DEFINITIONS("-D_TD_ARM_")
     ELSE ()
-        ADD_DEFINITIONS("-msse4.2 -mfma")
+        ADD_DEFINITIONS("-msse4.2")
+        IF("${FMA_SUPPORT}" MATCHES "true")
+            MESSAGE(STATUS "turn fma function support on")
+            ADD_DEFINITIONS("-mfma")
+        ELSE ()
+            MESSAGE(STATUS "turn fma function support off")
+        ENDIF()
     ENDIF ()
 
 ENDIF ()

cmake/cmake.options

@@ -25,9 +25,28 @@ IF(${TD_WINDOWS})
             ON
           )
 
-ENDIF ()
+    MESSAGE("build msvcregex Win32")
+    option(
+            BUILD_MSVCREGEX
+            "If build msvcregex on Windows"
+            ON
+          )
 
-IF(${TD_LINUX} MATCHES TRUE)
+    MESSAGE("build wcwidth Win32")
+    option(
+            BUILD_WCWIDTH
+            "If build wcwidth on Windows"
+            ON
+          )
+
+    MESSAGE("build wingetopt Win32")
+    option(
+            BUILD_WINGETOPT 
+                "If build wingetopt on Windows"
+            ON
+        )
+
+ENDIF ()
 
 option(
     BUILD_TEST 
@@ -35,8 +54,6 @@ option(
     ON
 )
 
-ENDIF ()
-
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 

cmake/msvcregex_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# msvcregex
+ExternalProject_Add(msvcregex
+        GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/pthread_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(pthread
         GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
         GIT_TAG v3.0.3.1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/wcwidth_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# wcwidth
+ExternalProject_Add(wcwidth
+        GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/wingetopt_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# wingetopt
+ExternalProject_Add(wingetopt
+        GIT_REPOSITORY https://github.com/alex85k/wingetopt.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

contrib/CMakeLists.txt

@@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
     cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
-# gnu regex
+# iconv
-if(${BUILD_GNUREGEX})
+if(${BUILD_WITH_ICONV})
-    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# msvc regex
+if(${BUILD_MSVCREGEX})
+    cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wcwidth
+if(${BUILD_WCWIDTH})
+    cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
@@ -95,12 +110,31 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 # ================================================================================================
 # googletest
 if(${BUILD_TEST})
-    add_subdirectory(googletest)
+    add_subdirectory(googletest EXCLUDE_FROM_ALL)
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
-        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
     )
+    if(${TD_WINDOWS})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
+        )
+    endif(${TD_WINDOWS})
+    if(${TD_LINUX})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
+        )
+    endif(${TD_LINUX})
+    if(${TD_DARWIN})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
+        )
+    endif(${TD_DARWIN})
+    
+    
 endif(${BUILD_TEST})
 
 # cJson
@@ -109,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
 option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
 option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
-add_subdirectory(cJson)
+add_subdirectory(cJson EXCLUDE_FROM_ALL)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
@@ -118,7 +152,7 @@ target_include_directories(
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
 # lz4
-add_subdirectory(lz4/build/cmake)
+add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
@@ -126,7 +160,7 @@ target_include_directories(
 
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
-add_subdirectory(zlib)
+add_subdirectory(zlib EXCLUDE_FROM_ALL)
 target_include_directories(
     zlibstatic
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@@ -142,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 # leveldb
 if(${BUILD_WITH_LEVELDB})
     option(LEVELDB_BUILD_TESTS "" OFF)
-    add_subdirectory(leveldb)
+    add_subdirectory(leveldb EXCLUDE_FROM_ALL)
     target_include_directories(
         leveldb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
@@ -158,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB})
     option(WITH_TOOLS "" OFF)
     option(WITH_LIBURING "" OFF)
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    add_subdirectory(rocksdb)
+    add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
     target_include_directories(
         rocksdb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
@@ -169,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB})
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
-    add_subdirectory(lucene)
+    add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
       lucene++ 
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
@@ -179,9 +213,56 @@ endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
 if(${BUILD_WITH_NURAFT})
-    add_subdirectory(nuraft)
+    add_subdirectory(nuraft EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_NURAFT})
 
+# pthread
+if(${BUILD_PTHREAD})
+    set(CMAKE_BUILD_TYPE release)
+    add_definitions(-DPTW32_STATIC_LIB)
+    add_subdirectory(pthread)
+    set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
+    add_library(pthread STATIC IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+endif()
+
+# iconv
+if(${BUILD_WITH_ICONV})
+    add_library(iconv STATIC iconv/win_iconv.c)
+endif(${BUILD_WITH_ICONV})
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
+endif(${BUILD_WINGETOPT})
+
+# msvcregex
+if(${BUILD_MSVCREGEX})
+    add_library(msvcregex STATIC "")
+    target_sources(msvcregex
+        PRIVATE "msvcregex/regex.c"
+    )
+    target_include_directories(msvcregex
+        PRIVATE "msvcregex"
+    )
+    target_link_libraries(msvcregex 
+        INTERFACE Shell32
+    )
+    SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
+endif(${BUILD_MSVCREGEX})
+
+# msvcregex
+if(${BUILD_WCWIDTH})
+    add_library(wcwidth STATIC "")
+    target_sources(wcwidth
+        PRIVATE "wcwidth/wcwidth.c"
+    )
+    target_include_directories(wcwidth
+        PRIVATE "wcwidth"
+    )
+    SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
+endif(${BUILD_WCWIDTH})
+
 # CRAFT
 if(${BUILD_WITH_CRAFT})
     add_library(craft STATIC IMPORTED GLOBAL)
@@ -212,7 +293,7 @@ if(${BUILD_WITH_UV})
         MESSAGE("Windows need set no-sign-compare")
         add_compile_options(-Wno-sign-compare)
     endif ()
-    add_subdirectory(libuv)
+    add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})
 
 # BDB
@@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
     target_link_libraries(sqlite 
         INTERFACE m 
         INTERFACE pthread 
-        INTERFACE dl
     )
+    if(NOT TD_WINDOWS)
+        target_link_libraries(sqlite 
+            INTERFACE dl
+        )
+    endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
 # pthread
@@ -249,5 +334,5 @@ endif(${BUILD_WITH_SQLITE})
 # Build test
 # ================================================================================================
 if(${BUILD_DEPENDENCY_TESTS})
-    add_subdirectory(test)
+    add_subdirectory(test EXCLUDE_FROM_ALL)
 endif(${BUILD_DEPENDENCY_TESTS})

contrib/test/craft/CMakeLists.txt

@@ -1,2 +1,9 @@
 add_executable(simulate_vnode "simulate_vnode.c")
 target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
+if(${BUILD_WINGETOPT})
+    target_link_libraries(simulate_vnode PUBLIC wingetopt)
+    target_include_directories(
+        simulate_vnode
+        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+    )
+endif()

contrib/test/tdev/src/main.c

@@ -6,43 +6,39 @@
 #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
 #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
 
-#define tPutA(buf, val)              \
+static inline void tPutA(void **buf, uint64_t val) {
-  ({                                 \
+  memcpy(buf, &val, sizeof(val));
-    memcpy(buf, &val, sizeof(val));  \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val)); \
+}
-  })
 
-#define tPutB(buf, val)                         \
+static inline void tPutB(void **buf, uint64_t val) {
-  ({                                            \
+  ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
-    ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
+  ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
-    ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
+  ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
-    ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
+  ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
-    ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
+  ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
-    ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
+  ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
-    ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
+  ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
-    ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;  \
+  ((uint8_t *)buf)[0] = (val)&0xff;
-    ((uint8_t *)buf)[0] = (val)&0xff;           \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val));            \
+}
-  })
 
-#define tPutC(buf, val)                \
+static inline void tPutC(void **buf, uint64_t val) {
-  ({                                   \
+  if (buf) {
-    if (buf) {                         \
+    ((uint64_t *)buf)[0] = (val);
-      ((uint64_t *)buf)[0] = (val);    \
+    POINTER_SHIFT(buf, sizeof(val));
-      POINTER_SHIFT(buf, sizeof(val)); \
+  }
-    }                                  \
+  *buf = NULL;
-    NULL;                                 \
+}
-  })
 
-#define tPutD(buf, val)                        \
+static inline void tPutD(void **buf, uint64_t val) {
-  ({                                           \
+  uint64_t tmp = val;
-    uint64_t tmp = val;                        \
+  for (size_t i = 0; i < sizeof(val); i++) {
-    for (size_t i = 0; i < sizeof(val); i++) { \
+    ((uint8_t *)buf)[i] = tmp & 0xff;
-      ((uint8_t *)buf)[i] = tmp & 0xff;        \
+    tmp >>= 8;
-      tmp >>= 8;                               \
+  }
-    }                                          \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val));           \
+}
-  })
 
 static inline void tPutE(void **buf, uint64_t val) {
   if (buf) {
@@ -61,7 +57,7 @@ static void func(T t) {
   switch (t) {
     case A:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutA(pBuf, val);
+        tPutA(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -69,7 +65,7 @@ static void func(T t) {
       break;
     case B:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutB(pBuf, val);
+        tPutB(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -77,7 +73,7 @@ static void func(T t) {
       break;
     case C:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutC(pBuf, val);
+        tPutC(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -85,7 +81,7 @@ static void func(T t) {
       break;
     case D:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutD(pBuf, val);
+        tPutD(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }

example/CMakeLists.txt

@@ -30,6 +30,7 @@ target_link_libraries(demoapi
 )
 
 target_include_directories(tmq
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
@@ -39,6 +40,7 @@ target_include_directories(tstream
 
 target_include_directories(demoapi
     PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

example/src/demoapi.c

@@ -4,10 +4,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
+// #include <unistd.h>
 #include <inttypes.h>
+#ifndef WINDOWS
 #include <argp.h>
-
+#endif
+#include "osSleep.h"
 #include "taos.h"
 
 #define debugPrint(fmt, ...) \
@@ -32,6 +34,7 @@
 int64_t g_num_of_tb = 2;
 int64_t g_num_of_rec = 3;
 
+#ifndef WINDOWS
 static struct argp_option options[] = {
     {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
     {"records", 'n', "NUMBER", 0,
@@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 }
 
 static struct argp argp = {options, parse_opt, "", ""};
-
+#endif
 static void prepare_data(TAOS* taos) {
     TAOS_RES *res;
     res = taos_query(taos, "drop database if exists test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
 
     res = taos_query(taos, "create database test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
     taos_select_db(taos, "test");
 
     char command[1024] = {0};
@@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) {
     taos_free_result(res);
 
     for (int64_t i = 0; i < g_num_of_tb; i ++) {
+        // sprintf(command, "create table t%"PRId64" using meters "
+        //         "tags(%"PRId64", '%s', '%s', '%s');",
+        //         i, i, (i%2)?"beijing":"shanghai",
+        //         (i%2)?"朝阳区":"黄浦区",
+        //         (i%2)?"长安街":"中山路");
         sprintf(command, "create table t%"PRId64" using meters "
-                "tags(%"PRId64", '%s', '%s', '%s');",
-                i, i, (i%2)?"beijing":"shanghai",
-                (i%2)?"朝阳区":"黄浦区",
-                (i%2)?"长安街":"中山路");
-/*        sprintf(command, "create table t%"PRId64" using meters "
                 "tags(%"PRId64", '%s', '%s', '%s');",
                 i, i,
                 (i%2)?"beijing":"shanghai",
                 (i%2)?"chaoyang":"huangpu",
                 (i%2?"changan street":"jianguo rd"));
-                */
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             okPrint("t%" PRId64 " created\n", i);
@@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) {
                     "'%c%d', '%s%c%d', '%c%d')",
                     i, 1650000000000+j, (float)j, j,
                     'a'+(int)j%25, rand(),
-                    "涛思", 'z' - (int)j%25, rand(),
+                    // "涛思", 'z' - (int)j%25, rand(),
+                    "TAOS", 'z' - (int)j%25, rand(),
                     'b' - (int)j%25, rand()
                     );
             res = taos_query(taos,  command);
@@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %"PRId64"\n",
                                             f, c,
-                                            *(int64_t*)(row[f]+c*sizeof(int64_t)));
+                                            *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
                                 }
                                 break;
 
@@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %d\n",
                                             f, c,
-                                            *(int32_t*)(row[f]+c*sizeof(int32_t)));
+                                            *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
                                 }
                                 break;
 
@@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %f\n",
                                             f, c,
-                                            *(float*)(row[f]+c*sizeof(float)));
+                                            *(float*)((char*)(row[f])+c*sizeof(float)));
                                 }
                                 break;
 
@@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     if (offsets) {
                         for (int c = 0; c < rows; c++) {
                             if (offsets[c] != -1) {
-                                int length = *(int16_t*)(row[f] + offsets[c]);
+                                int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
                                 char *buf = calloc(1, length + 1);
-                                strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
+                                strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
                                 printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
                                         c, f, offsets[c], length, buf);
                                 free(buf);
@@ -314,8 +317,9 @@ int main(int argc, char *argv[]) {
     const char* host = "127.0.0.1";
     const char* user = "root";
     const char* passwd = "taosdata";
-
+#ifndef WINDOWS
     argp_parse(&argp, argc, argv, 0, 0, NULL);
+#endif
     TAOS* taos = taos_connect(host, user, passwd, "", 0);
     if (taos == NULL) {
         printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));

example/src/tmq.c

@@ -24,7 +24,7 @@ static void msg_process(TAOS_RES* msg) {
   char buf[1024];
   memset(buf, 0, 1024);
   printf("topic: %s\n", tmq_get_topic_name(msg));
-  printf("vg:%d\n", tmq_get_vgroup_id(msg));
+  printf("vg: %d\n", tmq_get_vgroup_id(msg));
   while (1) {
     TAOS_ROW row = taos_fetch_row(msg);
     if (row == NULL) break;
@@ -101,9 +101,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*const char* sql = "select * from tu1";*/
+  pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
-  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/
-  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -141,8 +140,8 @@ int32_t create_topic() {
   return 0;
 }
 
-void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) {
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
-  printf("commit %d\n", resp);
+  printf("commit %d tmq %p offsets %p param %p\n", resp, tmq, offsets, param);
 }
 
 tmq_t* build_consumer() {
@@ -161,9 +160,10 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "group.id", "tg2");
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  tmq_conf_set(conf, "td.connect.db", "abc1");
+  /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+  assert(tmq);
   return tmq;
 }
 
@@ -215,12 +215,24 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     return;
   }
 
+  tmq_list_t* subList = NULL;
+  tmq_subscription(tmq, &subList);
+  char**  subTopics = tmq_list_to_c_array(subList);
+  int32_t sz = tmq_list_get_size(subList);
+  printf("subscribed topics: ");
+  for (int32_t i = 0; i < sz; i++) {
+    printf("%s, ", subTopics[i]);
+  }
+  printf("\n");
+  tmq_list_destroy(subList);
+
   while (running) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
       msg_process(tmqmessage);
       taos_free_result(tmqmessage);
 
+      tmq_commit(tmq, NULL, 1);
       /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
@@ -266,10 +278,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 int main(int argc, char* argv[]) {
-  int code;
   if (argc > 1) {
     printf("env init\n");
-    code = init_env();
+    if (init_env() < 0) {
+      return -1;
+    }
     create_topic();
   }
   tmq_t*      tmq = build_consumer();

example/src/tstream.c

@@ -25,7 +25,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -78,11 +78,15 @@ int32_t create_stream() {
   taos_free_result(pRes);
 
   /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
-  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
-  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
+  pRes =
+      taos_query(pConn,
+                 "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
+                 "from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);

examples/c/CMakeLists.txt

@@ -1,7 +1,7 @@
 PROJECT(TDengine)
 
 IF (TD_LINUX)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo apitest.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
@@ -13,7 +13,7 @@ IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()
 IF (TD_DARWIN)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo demo.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)

examples/c/asyncdemo.c

@@ -45,7 +45,7 @@ typedef struct {
 
 void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
 void taos_select_call_back(void *param, TAOS_RES *tres, int code);
-void taos_error(TAOS *taos);
+void shellPrintError(TAOS *taos);
 
 static void queryDB(TAOS *taos, char *command) {
   int i;
@@ -102,7 +102,7 @@ int main(int argc, char *argv[])
 
   taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
   if (taos == NULL)
-    taos_error(taos);
+    shellPrintError(taos);
 
   printf("success to connect to server\n");
 
@@ -193,7 +193,7 @@ int main(int argc, char *argv[])
   return 0;
 }
 
-void taos_error(TAOS *con)
+void shellPrintError(TAOS *con)
 {
   fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
   taos_close(con);

include/client/taos.h

@@ -27,10 +27,7 @@ typedef void   TAOS;
 typedef void   TAOS_STMT;
 typedef void   TAOS_RES;
 typedef void **TAOS_ROW;
-#if 0
+typedef void   TAOS_SUB;
-typedef void   TAOS_STREAM;
-#endif
-typedef void TAOS_SUB;
 
 // Data type definition
 #define TSDB_DATA_TYPE_NULL       0   // 1 bytes
@@ -88,18 +85,22 @@ typedef struct taosField {
   int32_t bytes;
 } TAOS_FIELD;
 
+#ifdef WINDOWS
+#define DLL_EXPORT __declspec(dllexport)
+#else
 #define DLL_EXPORT
+#endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-typedef struct TAOS_BIND_v2 {
+typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
   void     *buffer;
-  int32_t   buffer_length;
+  uintptr_t buffer_length;
   int32_t  *length;
   char     *is_null;
   int       num;
-} TAOS_BIND_v2;
+} TAOS_MULTI_BIND;
 
 typedef enum {
   SET_CONF_RET_SUCC = 0,
@@ -120,6 +121,7 @@ typedef struct setConfRet {
 DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
+DLL_EXPORT int        taos_init(void);
 DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
@@ -128,35 +130,35 @@ DLL_EXPORT void  taos_close(TAOS *taos);
 
 const char *taos_data_type(int type);
 
-DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
+DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
-DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
+DLL_EXPORT int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int         taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags);
+DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
-DLL_EXPORT int         taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int        taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
-DLL_EXPORT int         taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
-DLL_EXPORT int         taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int         taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int         taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int         taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int         taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int         taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx);
+DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
-DLL_EXPORT int         taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
-DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);
+DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_close(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char       *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
-DLL_EXPORT int         taos_stmt_affected_rows_once(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
-DLL_EXPORT TAOS_RES   *taos_query(TAOS *taos, const char *sql);
+DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
-DLL_EXPORT TAOS_RES   *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
+DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
 
-DLL_EXPORT TAOS_ROW    taos_fetch_row(TAOS_RES *res);
+DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
-DLL_EXPORT int         taos_result_precision(TAOS_RES *res);  // get the time precision of result
+DLL_EXPORT int      taos_result_precision(TAOS_RES *res);  // get the time precision of result
-DLL_EXPORT void        taos_free_result(TAOS_RES *res);
+DLL_EXPORT void     taos_free_result(TAOS_RES *res);
-DLL_EXPORT int         taos_field_count(TAOS_RES *res);
+DLL_EXPORT int      taos_field_count(TAOS_RES *res);
-DLL_EXPORT int         taos_num_fields(TAOS_RES *res);
+DLL_EXPORT int      taos_num_fields(TAOS_RES *res);
-DLL_EXPORT int         taos_affected_rows(TAOS_RES *res);
+DLL_EXPORT int      taos_affected_rows(TAOS_RES *res);
 
 DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
 DLL_EXPORT int         taos_select_db(TAOS *taos, const char *db);
@@ -192,12 +194,6 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-#if 0
-DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
-                                         int64_t stime, void *param, void (*callback)(void *));
-DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
-#endif
-
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat);
 
@@ -216,13 +212,14 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
 
 typedef struct tmq_conf_t tmq_conf_t;
 typedef struct tmq_list_t tmq_list_t;
-// typedef struct tmq_message_t tmq_message_t;
 
-typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
+typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param));
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
+DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
 #if 0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
@@ -233,15 +230,11 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
-DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
+DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
-#if 0
-DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups);
-DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups);
-#endif
 DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
@@ -260,38 +253,26 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
 DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
-DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
+DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
-
-#if 0
-// temporary used function for demo only
-void    tmqShowMsg(tmq_message_t *tmq_message);
-int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
-#endif
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
-DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
-DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
+DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
 // TODO
 #if 0
-DLL_EXPORT char   *tmq_get_block_table_name(TAOS_RES *res);
+DLL_EXPORT char   *tmq_get_table_name(TAOS_RES *res);
 #endif
 
 #if 0
-DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
-DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
-DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
-DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
 #endif
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
-#endif
-
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
-
+#endif
 /* ------------------------------ TMQ END -------------------------------- */
 #if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);

include/common/systable.h

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#ifndef TDENGINE_SYSTABLE_H
+#define TDENGINE_SYSTABLE_H
+
+#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
+#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
+#define TSDB_INS_TABLE_DNODES                 "dnodes"
+#define TSDB_INS_TABLE_MNODES                 "mnodes"
+#define TSDB_INS_TABLE_MODULES                "modules"
+#define TSDB_INS_TABLE_QNODES                 "qnodes"
+#define TSDB_INS_TABLE_BNODES                 "bnodes"
+#define TSDB_INS_TABLE_SNODES                 "snodes"
+#define TSDB_INS_TABLE_CLUSTER                "cluster"
+#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
+#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
+#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
+#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
+#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
+#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
+#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
+#define TSDB_INS_TABLE_USER_USERS             "user_users"
+#define TSDB_INS_TABLE_LICENCES               "grants"
+#define TSDB_INS_TABLE_VGROUPS                "vgroups"
+#define TSDB_INS_TABLE_VNODES                 "vnodes"
+#define TSDB_INS_TABLE_CONFIGS                "configs"
+
+#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
+#define TSDB_PERFS_TABLE_SMAS                 "smas"
+#define TSDB_PERFS_TABLE_SUBSCRIBES           "subscribes"
+#define TSDB_PERFS_TABLE_CONNECTIONS          "connections"
+#define TSDB_PERFS_TABLE_QUERIES              "queries"
+#define TSDB_PERFS_TABLE_TOPICS               "topics"
+#define TSDB_PERFS_TABLE_CONSUMERS            "consumers"
+#define TSDB_PERFS_TABLE_SUBSCRIPTIONS        "subscriptions"
+#define TSDB_PERFS_TABLE_OFFSETS              "offsets"
+#define TSDB_PERFS_TABLE_TRANS                "trans"
+#define TSDB_PERFS_TABLE_STREAMS              "streams"
+
+typedef struct SSysDbTableSchema {
+  const char   *name;
+  const int32_t type;
+  const int32_t bytes;
+} SSysDbTableSchema;
+
+typedef struct SSysTableMeta {
+  const char              *name;
+  const SSysDbTableSchema *schema;
+  const int32_t            colNum;
+} SSysTableMeta;
+
+void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size);
+void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SYSTABLE_H

include/common/taosdef.h

@@ -28,15 +28,16 @@ typedef int64_t tb_uid_t;
 #define TSWINDOW_INITIALIZER       ((STimeWindow){INT64_MIN, INT64_MAX})
 #define TSWINDOW_DESC_INITIALIZER  ((STimeWindow){INT64_MAX, INT64_MIN})
 #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
+#define TSWINDOW_IS_EQUAL(t1, t2)  (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
 
 typedef enum {
-  TSDB_SUPER_TABLE  = 1,    // super table
+  TSDB_SUPER_TABLE = 1,   // super table
-  TSDB_CHILD_TABLE  = 2,    // table created from super table
+  TSDB_CHILD_TABLE = 2,   // table created from super table
-  TSDB_NORMAL_TABLE = 3,    // ordinary table
+  TSDB_NORMAL_TABLE = 3,  // ordinary table
-  TSDB_STREAM_TABLE = 4,    // table created from stream computing
+  TSDB_STREAM_TABLE = 4,  // table created from stream computing
-  TSDB_TEMP_TABLE   = 5,    // temp table created by nest query
+  TSDB_TEMP_TABLE = 5,    // temp table created by nest query
   TSDB_SYSTEM_TABLE = 6,
-  TSDB_TABLE_MAX    = 7
+  TSDB_TABLE_MAX = 7
 } ETableType;
 
 typedef enum {
@@ -71,16 +72,24 @@ typedef enum {
   TSDB_SMA_STAT_DROPPED = 2,   // sma dropped
 } ETsdbSmaStat;                // bit operation
 
-
 typedef enum {
   TSDB_SMA_TYPE_BLOCK = 0,       // Block-wise SMA
   TSDB_SMA_TYPE_TIME_RANGE = 1,  // Time-range-wise SMA
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
+typedef enum {
+  TSDB_RETENTION_L0 = 0,
+  TSDB_RETENTION_L1 = 1,
+  TSDB_RETENTION_L2 = 2,
+  TSDB_RETENTION_MAX = 3
+} ERetentionLevel;
+
 extern char *qtypeStr[];
 
-#define TSDB_PORT_HTTP      11
+#define TSDB_PORT_HTTP 11
+
+#undef TD_DEBUG_PRINT_ROW
 
 #ifdef __cplusplus
 }

include/common/tcommon.h

@@ -65,20 +65,18 @@ typedef struct SDataBlockInfo {
   STimeWindow window;
   int32_t     rows;
   int32_t     rowSize;
-  union {
+  int64_t     uid;      // the uid of table, from which current data block comes
-    int64_t uid;        // from which table of uid, comes from this data block
+  int64_t     blockId;  // block id, generated by physical planner
-    int64_t blockId;
-  };
   uint64_t    groupId;  // no need to serialize
   int16_t     numOfCols;
   int16_t     hasVarCol;
-  int16_t     capacity;
+  int32_t     capacity;
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
-  SColumnDataAgg* pBlockAgg;
+  SColumnDataAgg** pBlockAgg;
-  SArray*         pDataBlock;  // SArray<SColumnInfoData>
+  SArray*          pDataBlock;  // SArray<SColumnInfoData>
-  SDataBlockInfo  info;
+  SDataBlockInfo   info;
 } SSDataBlock;
 
 typedef struct SVarColAttr {
@@ -99,6 +97,15 @@ typedef struct SColumnInfoData {
   };
 } SColumnInfoData;
 
+typedef struct SQueryTableDataCond {
+  STimeWindow  twindow;
+  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} SQueryTableDataCond;
+
 void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
@@ -122,59 +129,6 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
 
 static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
 
-static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
-  int32_t tlen = 0;
-  int32_t sz = 0;
-  // tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
-  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
-  tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return tlen;
-  tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema);
-  if (pRsp->pBlockData) {
-    sz = taosArrayGetSize(pRsp->pBlockData);
-  }
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-    tlen += tEncodeDataBlock(buf, pBlock);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
-  int32_t sz;
-  // buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
-  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
-  buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return buf;
-  pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-  if (pRsp->schema == NULL) return NULL;
-  buf = taosDecodeSSchemaWrapper(buf, pRsp->schema);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock block = {0};
-    tDecodeDataBlock(buf, &block);
-    taosArrayPush(pRsp->pBlockData, &block);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
-  if (pRsp->schema) {
-    if (pRsp->schema->nCols) {
-      taosMemoryFreeClear(pRsp->schema->pSchema);
-    }
-    taosMemoryFree(pRsp->schema);
-  }
-  taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
-  pRsp->pBlockData = NULL;
-}
-
 //======================================================================================================================
 // the following structure shared by parser and executor
 typedef struct SColumn {
@@ -195,22 +149,22 @@ typedef struct SColumn {
 } SColumn;
 
 typedef struct STableBlockDistInfo {
-  uint16_t  rowSize;
+  uint16_t rowSize;
-  uint16_t  numOfFiles;
+  uint16_t numOfFiles;
-  uint32_t  numOfTables;
+  uint32_t numOfTables;
-  uint64_t  totalSize;
+  uint64_t totalSize;
-  uint64_t  totalRows;
+  uint64_t totalRows;
-  int32_t   maxRows;
+  int32_t  maxRows;
-  int32_t   minRows;
+  int32_t  minRows;
-  int32_t   firstSeekTimeUs;
+  int32_t  firstSeekTimeUs;
-  uint32_t  numOfRowsInMemTable;
+  uint32_t numOfRowsInMemTable;
-  uint32_t  numOfSmallBlocks;
+  uint32_t numOfSmallBlocks;
-  SArray   *dataBlockInfos;
+  SArray*  dataBlockInfos;
 } STableBlockDistInfo;
 
 enum {
   FUNC_PARAM_TYPE_VALUE = 0x1,
-  FUNC_PARAM_TYPE_COLUMN= 0x2,
+  FUNC_PARAM_TYPE_COLUMN = 0x2,
 };
 
 typedef struct SFunctParam {
@@ -229,7 +183,6 @@ typedef struct SResSchame {
   char    name[TSDB_COL_NAME_LEN];
 } SResSchema;
 
-// TODO move away to executor.h
 typedef struct SExprBasicInfo {
   SResSchema   resSchema;
   int16_t      numOfParams;  // argument value of each function
@@ -256,7 +209,7 @@ typedef struct {
   int32_t valueLen;
 } SSmlKv;
 
-#define QUERY_ASC_FORWARD_STEP 1
+#define QUERY_ASC_FORWARD_STEP  1
 #define QUERY_DESC_FORWARD_STEP -1
 
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

include/common/tdatablock.h

@@ -54,6 +54,11 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataSetNotNull_f(bm_, r_)                  \
+  do {                                                \
+    BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_)));  \
+  } while (0)
+
 #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
 #define colDataSetNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] = -1)
 
@@ -63,16 +68,15 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 
 #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
 // SColumnInfoData, rowNumber
-#define colDataGetData(p1_, r_)                                        \
+#define colDataGetData(p1_, r_) \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
-                                        :  colDataGetNumData(p1_, r_))
 
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
-  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
+  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
-    if(colDataIsNull_var(pColumnInfoData, row)){
+    if (colDataIsNull_var(pColumnInfoData, row)) {
       return true;
     }
-    char *data = colDataGetVarData(pColumnInfoData, row);
+    char* data = colDataGetVarData(pColumnInfoData, row);
     return (*data == TSDB_DATA_TYPE_NULL);
   }
 
@@ -80,7 +84,7 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData,
     return false;
   }
 
-  if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
+  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
     return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
@@ -132,7 +136,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
 static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     for (int32_t i = start; i < start + nRows; ++i) {
-      colDataSetNull_var(pColumnInfoData,i);  // it is a null value of VAR type.
+      colDataSetNull_var(pColumnInfoData, i);  // it is a null value of VAR type.
     }
   } else {
     for (int32_t i = start; i < start + nRows; ++i) {
@@ -183,7 +187,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
 }
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource,
                         uint32_t numOfRow2);
 int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
@@ -225,6 +229,11 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
 
 void blockDebugShowData(const SArray* dataBlocks);
 
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
+                                 tb_uid_t uid, tb_uid_t suid);
+
+SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema);
+
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
   return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
 }
@@ -238,10 +247,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32
 
 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                                              int8_t needCompress) {
-  int32_t* colSizes = (int32_t*)data;
+  int32_t* actualLen = (int32_t*)data;
+  data += sizeof(int32_t);
 
+  uint64_t* groupId = (uint64_t*)data;
+  data += sizeof(uint64_t);
+
+  int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
-  *dataLen = (numOfCols * sizeof(int32_t));
+
+  *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
@@ -273,6 +288,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
 
     colSizes[col] = htonl(colSizes[col]);
   }
+
+  *actualLen = *dataLen;
+  *groupId = pBlock->info.groupId;
 }
 
 #ifdef __cplusplus

include/common/tdataformat.h

@@ -61,22 +61,22 @@ extern "C" {
 // ----------------- TSDB COLUMN DEFINITION
 #pragma pack(push, 1)
 typedef struct {
-  col_id_t colId;        // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
+  col_id_t colId;   // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
-  int32_t  type : 8;     // column type
+  int8_t   type;    // column type
-  int32_t  bytes : 24;   // column bytes (0~16M)
+  int8_t   flags;   // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON
-  int32_t  sma : 8;      // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
+  int32_t  bytes;   // column bytes (0~16M)
-  int32_t  offset : 24;  // point offset in STpRow after the header part.
+  int32_t  offset;  // point offset in STpRow after the header part.
 } STColumn;
 #pragma pack(pop)
 
 #define colType(col)   ((col)->type)
-#define colSma(col)    ((col)->sma)
+#define colFlags(col)  ((col)->flags)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
-#define colSetSma(col, s)    (colSma(col) = (s))
+#define colSetFlags(col, f)  (colFlags(col) = (f))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
@@ -146,7 +146,7 @@ typedef struct {
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition

include/common/tglobal.h

@@ -34,13 +34,9 @@ extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
 
 // common
-extern int32_t tsRpcTimer;
-extern int32_t tsRpcMaxTime;
-extern bool    tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
 extern int32_t tsMaxConnections;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
-extern int32_t tsMaxTmrCtrl;
 extern int32_t tsCompressMsgSize;
 extern int32_t tsCompressColData;
 extern int32_t tsMaxNumOfDistinctResults;
@@ -94,10 +90,10 @@ extern bool    tsRetrieveBlockingModel;  // retrieve threads will be blocked
 extern bool    tsKeepOriginalColumnName;
 extern bool    tsDeadLockKillQuery;
 
+// query client
+extern int32_t tsQueryPolicy;
+
 // client
-extern int32_t tsMaxWildCardsLen;
-extern int32_t tsMaxRegexStringLen;
-extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
 extern int32_t tsMaxStreamComputDelay;
@@ -125,11 +121,15 @@ extern char     tsCompressor[];
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
+// internal 
+extern int32_t tsTransPullupMs;
+extern int32_t tsMaRebalanceMs;
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
-                      const char *apolloUrl, SArray *pArgs, bool tsc);
+                      char *apolloUrl, SArray *pArgs, bool tsc);
-int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc);
+int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
 void    taosCleanupCfg();
 void    taosCfgDynamicOptions(const char *option, const char *value);
 void    taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);

include/common/tmsgcb.h

@@ -42,6 +42,7 @@ typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueTy
 typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
 typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
 typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
+typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet);
 typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
 typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
 typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc);
@@ -52,6 +53,7 @@ typedef struct {
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
   SendRspFp               sendRspFp;
+  SendRedirectRspFp       sendRedirectRspFp;
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
   ReportStartup           reportStartupFp;
@@ -62,6 +64,7 @@ int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
 int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
 void    tmsgSendRsp(const SRpcMsg* pRsp);
+void    tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet);
 void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
 void    tmsgReleaseHandle(void* handle, int8_t type);
 void    tmsgReportStartup(const char* name, const char* desc);

include/common/tmsgdef.h

@@ -85,6 +85,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
 
   // Requests handled by MNODE
   TD_NEW_MSG_SEG(TDMT_MND_MSG)
@@ -144,10 +145,11 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
@@ -168,18 +170,14 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateTbReq, SVCreateTbRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONSUME, "vnode-mq-consume", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@@ -204,6 +202,22 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp)
+
+  // sync integration
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
   
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)

include/common/trow.h

@@ -23,7 +23,6 @@
 #include "tbuffer.h"
 #include "tdataformat.h"
 #include "tdef.h"
-#include "tschema.h"
 #include "ttypes.h"
 #include "tutil.h"
 
@@ -58,12 +57,12 @@ extern "C" {
 #define TD_ROWS_ALL_NORM  0x00U
 #define TD_ROWS_NULL_NORM 0x01U
 
-#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM)  // all rows of SDataCol/SBlockCol is NORM
+#define TD_COL_ROWS_NORM(c)          ((c)->bitmap == TD_ROWS_ALL_NORM)  // all rows of SDataCol/SBlockCol is NORM
 #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v))
-#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
+#define TD_SET_COL_ROWS_NORM(c)      TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
-#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
+#define TD_SET_COL_ROWS_MISC(c)      TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
 
-#define KvConvertRatio (0.9f)
+#define KvConvertRatio            (0.9f)
 #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio))
 
 #ifdef TD_SUPPORT_BITMAP
@@ -215,6 +214,16 @@ STSRow *tdRowDup(STSRow *row);
 static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
   return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx;
 }
+
+static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
+  ASSERT(idx >= 0);
+  if (idx == 0) {
+    return PRIMARYKEY_TIMESTAMP_COL_ID;
+  }
+
+  return ((SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx - 1)->colId;
+}
+
 static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); }
 
 #define TD_ROW_OFFSET(p) ((p)->toffset);  // During ParseInsert when without STSchema, how to get the offset for STpRow?
@@ -341,7 +350,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx,
   return false;
 }
 
-static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
+                                               int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
@@ -585,6 +595,33 @@ static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols,
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief
+ *
+ * @param pBuilder
+ * @param nCols
+ * @param nBoundCols use -1 if not available
+ * @param flen
+ * @return FORCE_INLINE
+ */
+static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) {
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
 /**
  * @brief To judge row type: STpRow/SKvRow
  *
@@ -668,7 +705,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
     case TD_ROW_KV:
 #ifdef TD_SUPPORT_BITMAP
       pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
-      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
+      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
 #endif
       len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
             pBuilder->nBoundBitmaps;  // add
@@ -720,7 +757,6 @@ static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 /**
  * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数
  *
@@ -1092,7 +1128,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
   STSRow    *pRow = pIter->pRow;
   SKvRowIdx *pKvIdx = NULL;
   bool       colFound = false;
-  col_id_t   kvNCols = tdRowGetNCols(pRow);
+  col_id_t   kvNCols = tdRowGetNCols(pRow) - 1;
   while (*nIdx < kvNCols) {
     pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
     if (pKvIdx->colId == colId) {
@@ -1108,7 +1144,14 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
     }
   }
 
-  if (!colFound) return false;
+  if (!colFound) {
+    if (colId <= pIter->maxColId) {
+      pVal->valType = TD_VTYPE_NONE;
+      return true;
+    } else {
+      return false;
+    }
+  }
 
 #ifdef TD_SUPPORT_BITMAP
   int16_t colIdx = -1;
@@ -1352,14 +1395,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
   }
 }
 
-static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
+static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) {
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pSchema);
   tdSTSRowIterReset(&iter, row);
-  printf(">>>");
+  printf("%s >>>", tag);
   for (int i = 0; i < pSchema->numOfCols; ++i) {
     STColumn *stCol = pSchema->columns + i;
-    SCellVal  sVal = {.valType = 255, .val = NULL};
+    SCellVal  sVal = {255, NULL};
     if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
       break;
     }
@@ -1368,7 +1411,6 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
   }
   printf("\n");
 }
-
 #ifdef TROW_ORIGIN_HZ
 typedef struct {
   uint32_t nRows;

include/common/tschema.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_COMMON_SCHEMA_H_
-#define _TD_COMMON_SCHEMA_H_
-
-#include "os.h"
-#include "tarray.h"
-#include "ttypes.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if 0
-typedef struct STColumn {
-  /// column name
-  char *cname;
-  union {
-    /// for encode purpose
-    uint64_t info;
-    struct {
-      uint64_t sma : 1;
-      /// column data type
-      uint64_t type : 7;
-      /// column id
-      uint64_t cid : 16;
-      /// max bytes of the column
-      uint64_t bytes : 32;
-      /// reserved
-      uint64_t reserve : 8;
-    };
-  };
-  /// comment about the column
-  char *comment;
-} STColumn;
-
-typedef struct STSchema {
-  /// schema version
-  uint16_t sver;
-  /// number of columns
-  uint16_t ncols;
-  /// sma attributes
-  struct {
-    bool    sma;
-    SArray *smaArray;
-  };
-  /// column info
-  STColumn cols[];
-} STSchema;
-
-typedef struct {
-  uint64_t  size;
-  STSchema *pSchema;
-} STShemaBuilder;
-
-#define tSchemaBuilderInit(target, capacity) \
-  { .size = (capacity), .pSchema = (target) }
-void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
-void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
-int32_t  tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
-
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_COMMON_SCHEMA_H_*/

include/common/ttokendef.h

@@ -16,242 +16,244 @@
 #ifndef _TD_COMMON_TOKEN_H_
 #define _TD_COMMON_TOKEN_H_
 
-#define TK_OR                               1
+#define TK_OR            1
-#define TK_AND                              2
+#define TK_AND           2
-#define TK_UNION                            3
+#define TK_UNION         3
-#define TK_ALL                              4
+#define TK_ALL           4
-#define TK_MINUS                            5
+#define TK_MINUS         5
-#define TK_EXCEPT                           6
+#define TK_EXCEPT        6
-#define TK_INTERSECT                        7
+#define TK_INTERSECT     7
-#define TK_NK_BITAND                        8
+#define TK_NK_BITAND     8
-#define TK_NK_BITOR                         9
+#define TK_NK_BITOR      9
-#define TK_NK_LSHIFT                       10
+#define TK_NK_LSHIFT     10
-#define TK_NK_RSHIFT                       11
+#define TK_NK_RSHIFT     11
-#define TK_NK_PLUS                         12
+#define TK_NK_PLUS       12
-#define TK_NK_MINUS                        13
+#define TK_NK_MINUS      13
-#define TK_NK_STAR                         14
+#define TK_NK_STAR       14
-#define TK_NK_SLASH                        15
+#define TK_NK_SLASH      15
-#define TK_NK_REM                          16
+#define TK_NK_REM        16
-#define TK_NK_CONCAT                       17
+#define TK_NK_CONCAT     17
-#define TK_CREATE                          18
+#define TK_CREATE        18
-#define TK_ACCOUNT                         19
+#define TK_ACCOUNT       19
-#define TK_NK_ID                           20
+#define TK_NK_ID         20
-#define TK_PASS                            21
+#define TK_PASS          21
-#define TK_NK_STRING                       22
+#define TK_NK_STRING     22
-#define TK_ALTER                           23
+#define TK_ALTER         23
-#define TK_PPS                             24
+#define TK_PPS           24
-#define TK_TSERIES                         25
+#define TK_TSERIES       25
-#define TK_STORAGE                         26
+#define TK_STORAGE       26
-#define TK_STREAMS                         27
+#define TK_STREAMS       27
-#define TK_QTIME                           28
+#define TK_QTIME         28
-#define TK_DBS                             29
+#define TK_DBS           29
-#define TK_USERS                           30
+#define TK_USERS         30
-#define TK_CONNS                           31
+#define TK_CONNS         31
-#define TK_STATE                           32
+#define TK_STATE         32
-#define TK_USER                            33
+#define TK_USER          33
-#define TK_PRIVILEGE                       34
+#define TK_PRIVILEGE     34
-#define TK_DROP                            35
+#define TK_DROP          35
-#define TK_DNODE                           36
+#define TK_DNODE         36
-#define TK_PORT                            37
+#define TK_PORT          37
-#define TK_NK_INTEGER                      38
+#define TK_NK_INTEGER    38
-#define TK_DNODES                          39
+#define TK_DNODES        39
-#define TK_NK_IPTOKEN                      40
+#define TK_NK_IPTOKEN    40
-#define TK_LOCAL                           41
+#define TK_LOCAL         41
-#define TK_QNODE                           42
+#define TK_QNODE         42
-#define TK_ON                              43
+#define TK_ON            43
-#define TK_BNODE                           44
+#define TK_BNODE         44
-#define TK_SNODE                           45
+#define TK_SNODE         45
-#define TK_MNODE                           46
+#define TK_MNODE         46
-#define TK_DATABASE                        47
+#define TK_DATABASE      47
-#define TK_USE                             48
+#define TK_USE           48
-#define TK_IF                              49
+#define TK_IF            49
-#define TK_NOT                             50
+#define TK_NOT           50
-#define TK_EXISTS                          51
+#define TK_EXISTS        51
-#define TK_BLOCKS                          52
+#define TK_BUFFER        52
-#define TK_CACHE                           53
+#define TK_CACHELAST     53
-#define TK_CACHELAST                       54
+#define TK_COMP          54
-#define TK_COMP                            55
+#define TK_DAYS          55
-#define TK_DAYS                            56
+#define TK_NK_VARIABLE   56
-#define TK_NK_VARIABLE                     57
+#define TK_FSYNC         57
-#define TK_FSYNC                           58
+#define TK_MAXROWS       58
-#define TK_MAXROWS                         59
+#define TK_MINROWS       59
-#define TK_MINROWS                         60
+#define TK_KEEP          60
-#define TK_KEEP                            61
+#define TK_PAGES         61
-#define TK_PRECISION                       62
+#define TK_PAGESIZE      62
-#define TK_QUORUM                          63
+#define TK_PRECISION     63
-#define TK_REPLICA                         64
+#define TK_REPLICA       64
-#define TK_TTL                             65
+#define TK_STRICT        65
-#define TK_WAL                             66
+#define TK_WAL           66
-#define TK_VGROUPS                         67
+#define TK_VGROUPS       67
-#define TK_SINGLE_STABLE                   68
+#define TK_SINGLE_STABLE 68
-#define TK_STREAM_MODE                     69
+#define TK_RETENTIONS    69
-#define TK_RETENTIONS                      70
+#define TK_NK_COMMA      70
-#define TK_STRICT                          71
+#define TK_NK_COLON      71
-#define TK_NK_COMMA                        72
+#define TK_TABLE         72
-#define TK_NK_COLON                        73
+#define TK_NK_LP         73
-#define TK_TABLE                           74
+#define TK_NK_RP         74
-#define TK_NK_LP                           75
+#define TK_STABLE        75
-#define TK_NK_RP                           76
+#define TK_ADD           76
-#define TK_STABLE                          77
+#define TK_COLUMN        77
-#define TK_ADD                             78
+#define TK_MODIFY        78
-#define TK_COLUMN                          79
+#define TK_RENAME        79
-#define TK_MODIFY                          80
+#define TK_TAG           80
-#define TK_RENAME                          81
+#define TK_SET           81
-#define TK_TAG                             82
+#define TK_NK_EQ         82
-#define TK_SET                             83
+#define TK_USING         83
-#define TK_NK_EQ                           84
+#define TK_TAGS          84
-#define TK_USING                           85
+#define TK_NK_DOT        85
-#define TK_TAGS                            86
+#define TK_COMMENT       86
-#define TK_NK_DOT                          87
+#define TK_BOOL          87
-#define TK_COMMENT                         88
+#define TK_TINYINT       88
-#define TK_BOOL                            89
+#define TK_SMALLINT      89
-#define TK_TINYINT                         90
+#define TK_INT           90
-#define TK_SMALLINT                        91
+#define TK_INTEGER       91
-#define TK_INT                             92
+#define TK_BIGINT        92
-#define TK_INTEGER                         93
+#define TK_FLOAT         93
-#define TK_BIGINT                          94
+#define TK_DOUBLE        94
-#define TK_FLOAT                           95
+#define TK_BINARY        95
-#define TK_DOUBLE                          96
+#define TK_TIMESTAMP     96
-#define TK_BINARY                          97
+#define TK_NCHAR         97
-#define TK_TIMESTAMP                       98
+#define TK_UNSIGNED      98
-#define TK_NCHAR                           99
+#define TK_JSON          99
-#define TK_UNSIGNED                       100
+#define TK_VARCHAR       100
-#define TK_JSON                           101
+#define TK_MEDIUMBLOB    101
-#define TK_VARCHAR                        102
+#define TK_BLOB          102
-#define TK_MEDIUMBLOB                     103
+#define TK_VARBINARY     103
-#define TK_BLOB                           104
+#define TK_DECIMAL       104
-#define TK_VARBINARY                      105
+#define TK_DELAY         105
-#define TK_DECIMAL                        106
+#define TK_FILE_FACTOR   106
-#define TK_SMA                            107
+#define TK_NK_FLOAT      107
-#define TK_ROLLUP                         108
+#define TK_ROLLUP        108
-#define TK_FILE_FACTOR                    109
+#define TK_TTL           109
-#define TK_NK_FLOAT                       110
+#define TK_SMA           110
-#define TK_DELAY                          111
+#define TK_SHOW          111
-#define TK_SHOW                           112
+#define TK_DATABASES     112
-#define TK_DATABASES                      113
+#define TK_TABLES        113
-#define TK_TABLES                         114
+#define TK_STABLES       114
-#define TK_STABLES                        115
+#define TK_MNODES        115
-#define TK_MNODES                         116
+#define TK_MODULES       116
-#define TK_MODULES                        117
+#define TK_QNODES        117
-#define TK_QNODES                         118
+#define TK_FUNCTIONS     118
-#define TK_FUNCTIONS                      119
+#define TK_INDEXES       119
-#define TK_INDEXES                        120
+#define TK_FROM          120
-#define TK_FROM                           121
+#define TK_ACCOUNTS      121
-#define TK_ACCOUNTS                       122
+#define TK_APPS          122
-#define TK_APPS                           123
+#define TK_CONNECTIONS   123
-#define TK_CONNECTIONS                    124
+#define TK_LICENCE       124
-#define TK_LICENCE                        125
+#define TK_GRANTS        125
-#define TK_GRANTS                         126
+#define TK_QUERIES       126
-#define TK_QUERIES                        127
+#define TK_SCORES        127
-#define TK_SCORES                         128
+#define TK_TOPICS        128
-#define TK_TOPICS                         129
+#define TK_VARIABLES     129
-#define TK_VARIABLES                      130
+#define TK_BNODES        130
-#define TK_BNODES                         131
+#define TK_SNODES        131
-#define TK_SNODES                         132
+#define TK_CLUSTER       132
-#define TK_CLUSTER                        133
+#define TK_LIKE          133
-#define TK_LIKE                           134
+#define TK_INDEX         134
-#define TK_INDEX                          135
+#define TK_FULLTEXT      135
-#define TK_FULLTEXT                       136
+#define TK_FUNCTION      136
-#define TK_FUNCTION                       137
+#define TK_INTERVAL      137
-#define TK_INTERVAL                       138
+#define TK_TOPIC         138
-#define TK_TOPIC                          139
+#define TK_AS            139
-#define TK_AS                             140
+#define TK_WITH          140
-#define TK_DESC                           141
+#define TK_SCHEMA        141
-#define TK_DESCRIBE                       142
+#define TK_DESC          142
-#define TK_RESET                          143
+#define TK_DESCRIBE      143
-#define TK_QUERY                          144
+#define TK_RESET         144
-#define TK_EXPLAIN                        145
+#define TK_QUERY         145
-#define TK_ANALYZE                        146
+#define TK_CACHE         146
-#define TK_VERBOSE                        147
+#define TK_EXPLAIN       147
-#define TK_NK_BOOL                        148
+#define TK_ANALYZE       148
-#define TK_RATIO                          149
+#define TK_VERBOSE       149
-#define TK_COMPACT                        150
+#define TK_NK_BOOL       150
-#define TK_VNODES                         151
+#define TK_RATIO         151
-#define TK_IN                             152
+#define TK_COMPACT       152
-#define TK_OUTPUTTYPE                     153
+#define TK_VNODES        153
-#define TK_AGGREGATE                      154
+#define TK_IN            154
-#define TK_BUFSIZE                        155
+#define TK_OUTPUTTYPE    155
-#define TK_STREAM                         156
+#define TK_AGGREGATE     156
-#define TK_INTO                           157
+#define TK_BUFSIZE       157
-#define TK_TRIGGER                        158
+#define TK_STREAM        158
-#define TK_AT_ONCE                        159
+#define TK_INTO          159
-#define TK_WINDOW_CLOSE                   160
+#define TK_TRIGGER       160
-#define TK_WATERMARK                      161
+#define TK_AT_ONCE       161
-#define TK_KILL                           162
+#define TK_WINDOW_CLOSE  162
-#define TK_CONNECTION                     163
+#define TK_WATERMARK     163
-#define TK_MERGE                          164
+#define TK_KILL          164
-#define TK_VGROUP                         165
+#define TK_CONNECTION    165
-#define TK_REDISTRIBUTE                   166
+#define TK_MERGE         166
-#define TK_SPLIT                          167
+#define TK_VGROUP        167
-#define TK_SYNCDB                         168
+#define TK_REDISTRIBUTE  168
-#define TK_NULL                           169
+#define TK_SPLIT         169
-#define TK_NK_QUESTION                    170
+#define TK_SYNCDB        170
-#define TK_NK_ARROW                       171
+#define TK_NULL          171
-#define TK_ROWTS                          172
+#define TK_NK_QUESTION   172
-#define TK_TBNAME                         173
+#define TK_NK_ARROW      173
-#define TK_QSTARTTS                       174
+#define TK_ROWTS         174
-#define TK_QENDTS                         175
+#define TK_TBNAME        175
-#define TK_WSTARTTS                       176
+#define TK_QSTARTTS      176
-#define TK_WENDTS                         177
+#define TK_QENDTS        177
-#define TK_WDURATION                      178
+#define TK_WSTARTTS      178
-#define TK_CAST                           179
+#define TK_WENDTS        179
-#define TK_NOW                            180
+#define TK_WDURATION     180
-#define TK_TODAY                          181
+#define TK_CAST          181
-#define TK_TIMEZONE                       182
+#define TK_NOW           182
-#define TK_COUNT                          183
+#define TK_TODAY         183
-#define TK_FIRST                          184
+#define TK_TIMEZONE      184
-#define TK_LAST                           185
+#define TK_COUNT         185
-#define TK_LAST_ROW                       186
+#define TK_FIRST         186
-#define TK_BETWEEN                        187
+#define TK_LAST          187
-#define TK_IS                             188
+#define TK_LAST_ROW      188
-#define TK_NK_LT                          189
+#define TK_BETWEEN       189
-#define TK_NK_GT                          190
+#define TK_IS            190
-#define TK_NK_LE                          191
+#define TK_NK_LT         191
-#define TK_NK_GE                          192
+#define TK_NK_GT         192
-#define TK_NK_NE                          193
+#define TK_NK_LE         193
-#define TK_MATCH                          194
+#define TK_NK_GE         194
-#define TK_NMATCH                         195
+#define TK_NK_NE         195
-#define TK_CONTAINS                       196
+#define TK_MATCH         196
-#define TK_JOIN                           197
+#define TK_NMATCH        197
-#define TK_INNER                          198
+#define TK_CONTAINS      198
-#define TK_SELECT                         199
+#define TK_JOIN          199
-#define TK_DISTINCT                       200
+#define TK_INNER         200
-#define TK_WHERE                          201
+#define TK_SELECT        201
-#define TK_PARTITION                      202
+#define TK_DISTINCT      202
-#define TK_BY                             203
+#define TK_WHERE         203
-#define TK_SESSION                        204
+#define TK_PARTITION     204
-#define TK_STATE_WINDOW                   205
+#define TK_BY            205
-#define TK_SLIDING                        206
+#define TK_SESSION       206
-#define TK_FILL                           207
+#define TK_STATE_WINDOW  207
-#define TK_VALUE                          208
+#define TK_SLIDING       208
-#define TK_NONE                           209
+#define TK_FILL          209
-#define TK_PREV                           210
+#define TK_VALUE         210
-#define TK_LINEAR                         211
+#define TK_NONE          211
-#define TK_NEXT                           212
+#define TK_PREV          212
-#define TK_GROUP                          213
+#define TK_LINEAR        213
-#define TK_HAVING                         214
+#define TK_NEXT          214
-#define TK_ORDER                          215
+#define TK_GROUP         215
-#define TK_SLIMIT                         216
+#define TK_HAVING        216
-#define TK_SOFFSET                        217
+#define TK_ORDER         217
-#define TK_LIMIT                          218
+#define TK_SLIMIT        218
-#define TK_OFFSET                         219
+#define TK_SOFFSET       219
-#define TK_ASC                            220
+#define TK_LIMIT         220
-#define TK_NULLS                          221
+#define TK_OFFSET        221
+#define TK_ASC           222
+#define TK_NULLS         223
+#define TK_ID            224
+#define TK_NK_BITNOT     225
+#define TK_INSERT        226
+#define TK_VALUES        227
+#define TK_IMPORT        228
+#define TK_NK_SEMI       229
+#define TK_FILE          230
 
-#define TK_NK_SPACE                       300
+#define TK_NK_SPACE   300
-#define TK_NK_COMMENT                     301
+#define TK_NK_COMMENT 301
-#define TK_NK_ILLEGAL                     302
+#define TK_NK_ILLEGAL 302
-#define TK_NK_HEX                         303   // hex number  0x123
+#define TK_NK_HEX     303  // hex number  0x123
-#define TK_NK_OCT                         304   // oct number
+#define TK_NK_OCT     304  // oct number
-#define TK_NK_BIN                         305   // bin format data 0b111
+#define TK_NK_BIN     305  // bin format data 0b111
-#define TK_NK_FILE                        306
 
-#define TK_NK_BITNOT                      501
+#define TK_NK_NIL 65535
-#define TK_INSERT                         502
-#define TK_VALUES                         507
-#define TK_IMPORT                         509
-#define TK_NK_SEMI                        508
-
-#define TK_NK_NIL                         65535
 
 #endif /*_TD_COMMON_TOKEN_H_*/

include/dnode/bnode/bnode.h

@@ -26,6 +26,9 @@ extern "C" {
 typedef struct SBnode SBnode;
 
 typedef struct {
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 } SBnodeLoad;
 
 typedef struct {

include/dnode/mnode/sdb/sdb.h

@@ -18,6 +18,11 @@
 
 #include "os.h"
 
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -102,10 +107,9 @@ typedef enum {
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,
-  SDB_STATUS_UPDATING = 2,
+  SDB_STATUS_DROPPING = 2,
-  SDB_STATUS_DROPPING = 3,
+  SDB_STATUS_DROPPED = 3,
   SDB_STATUS_READY = 4,
-  SDB_STATUS_DROPPED = 5
 } ESdbStatus;
 
 typedef enum {
@@ -135,7 +139,7 @@ typedef enum {
 typedef struct SSdb SSdb;
 typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
-typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj);
+typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
@@ -221,7 +225,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw);
  * @param pRaw The raw data.
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw);
+int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw);
 
 /**
  * @brief Acquire a row from sdb
@@ -326,9 +330,28 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
+typedef struct SSdb {
+  SMnode     *pMnode;
+  char       *currDir;
+  char       *syncDir;
+  char       *tmpDir;
+  int64_t     lastCommitVer;
+  int64_t     curVer;
+  int64_t     tableVer[SDB_MAX];
+  int64_t     maxId[SDB_MAX];
+  EKeyType    keyTypes[SDB_MAX];
+  SHashObj   *hashObjs[SDB_MAX];
+  SRWLatch    locks[SDB_MAX];
+  SdbInsertFp insertFps[SDB_MAX];
+  SdbUpdateFp updateFps[SDB_MAX];
+  SdbDeleteFp deleteFps[SDB_MAX];
+  SdbDeployFp deployFps[SDB_MAX];
+  SdbEncodeFp encodeFps[SDB_MAX];
+  SdbDecodeFp decodeFps[SDB_MAX];
+} SSdb;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/catalog/catalog.h

@@ -51,7 +51,7 @@ typedef struct SMetaData {
   SArray    *pTableMeta;  // STableMeta array
   SArray    *pVgroupInfo; // SVgroupInfo list
   SArray    *pUdfList;    // udf info list
-  SArray    *pEpSetList;  // qnode epset list, SArray<SEpSet>
+  SArray    *pQnodeList;  // qnode list, SArray<SQueryNodeAddr>
 } SMetaData;
 
 typedef struct SCatalogCfg {
@@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const
  */
 int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
-
 int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
 
 int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);

include/libs/executor/dataSinkMgt.h

@@ -45,6 +45,7 @@ typedef struct SInputData {
 
 typedef struct SOutputData {
   int32_t numOfRows;
+  int32_t numOfCols;
   int8_t  compressed;
   char*   pData;
   bool    queryEnd;

include/libs/executor/executor.h

@@ -32,6 +32,8 @@ typedef struct SReadHandle {
   void* reader;
   void* meta;
   void* config;
+  void* vnode;
+  void* mnd;
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1

include/libs/function/function.h

@@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
 typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -113,7 +113,7 @@ typedef struct SResultRowEntryInfo {
   bool     initialized:1;     // output buffer has been initialized
   bool     complete:1;        // query has completed
   uint8_t  isNullRes:6;       // the result is null
-  uint8_t  numOfRes;        // num of output result in current buffer
+  uint8_t  numOfRes;          // num of output result in current buffer
 } SResultRowEntryInfo;
 
 // determine the real data need to calculated the result
@@ -141,8 +141,7 @@ struct SResultRowEntryInfo;
 
 //for selectivity query, the corresponding tag value is assigned if the data is qualified
 typedef struct SSubsidiaryResInfo {
-  int16_t                 bufLen;      // keep the tags data for top/bottom query result
+  int16_t num;
-  int16_t                 numOfCols;
   struct SqlFunctionCtx **pCtx;
 } SSubsidiaryResInfo;
 
@@ -166,7 +165,7 @@ typedef struct SInputColumnInfoData {
   SColumnInfoData  *pPTS;           // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;
-  uint64_t          uid;            // table uid
+  uint64_t          uid;            // table uid, used to set the tag value when building the final query result for selectivity functions.
 } SInputColumnInfoData;
 
 // sql function runtime context
@@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx {
   uint8_t          currentStage;  // record current running step, default: 0
   bool             isAggSet;
   int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block, TODO remove it
-  /////////////////////////////////////////////////////////////////
   bool             stableQuery;
+  /////////////////////////////////////////////////////////////////
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
   int32_t          numOfParams;
@@ -198,11 +197,17 @@ typedef struct SqlFunctionCtx {
   int32_t          offset;
   SVariant         tag;
   struct  SResultRowEntryInfo *resultInfo;
-  SSubsidiaryResInfo     subsidiaryRes;
+  SSubsidiaryResInfo     subsidiaries;
-  SPoint1          start;
+  SPoint1                start;
-  SPoint1          end;
+  SPoint1                end;
-  SFuncExecFuncs   fpSet;
+  SFuncExecFuncs         fpSet;
-  SScalarFuncExecFuncs sfp;
+  SScalarFuncExecFuncs   sfp;
+  struct SExprInfo      *pExpr;
+  struct SDiskbasedBuf  *pBuf;
+  struct SSDataBlock    *pSrcBlock;
+  int32_t                curBufPage;
+
+  char                   udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 enum {
@@ -319,6 +324,18 @@ struct SUdfInfo;
 void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 
+/**
+ * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
+ * @return error code
+ */
+int32_t udfcOpen();
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t udfcClose();
+
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "function.h"
+#include "querynodes.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -40,6 +40,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_STDDEV,
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
+  FUNCTION_TYPE_HISTOGRAM,
 
   // nonstandard SQL function
   FUNCTION_TYPE_BOTTOM = 500,
@@ -112,6 +113,9 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_WENDTS,
   FUNCTION_TYPE_WDURATION,
 
+  // internal function
+  FUNCTION_TYPE_SELECT_VALUE,
+
   // user defined funcion
   FUNCTION_TYPE_UDF = 10000
 } EFunctionType;
@@ -123,26 +127,28 @@ struct SCatalog;
 
 typedef struct SFmGetFuncInfoParam {
   struct SCatalog* pCtg;
-  void *pRpc;
+  void*            pRpc;
-  const SEpSet* pMgmtEps;
+  const SEpSet*    pMgmtEps;
+  char*            pErrBuf;
+  int32_t          errBufLen;
 } SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
-
-int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
 
 bool fmIsAggFunc(int32_t funcId);
 bool fmIsScalarFunc(int32_t funcId);
 bool fmIsNonstandardSQLFunc(int32_t funcId);
 bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
+bool fmIsSelectFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
+bool fmIsScanPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);
@@ -161,6 +167,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
+int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

include/libs/function/tudf.h

@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TUDF_H
+#define TDENGINE_TUDF_H
+
+
+#include <stdint.h>
+#include <stdbool.h>
+#include "tmsg.h"
+#include "tcommon.h"
+#include "function.h"
+#include "tdatablock.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UDF_LISTEN_PIPE_NAME_LEN 32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
+#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
+
+//======================================================================================
+//begin API to taosd and qworker
+
+enum {
+  UDFC_CODE_STOPPING = -1,
+  UDFC_CODE_PIPE_READ_ERR = -2,
+  UDFC_CODE_CONNECT_PIPE_ERR = -3,
+  UDFC_CODE_LOAD_UDF_FAILURE = -4,
+  UDFC_CODE_INVALID_STATE = -5
+};
+
+typedef void *UdfcFuncHandle;
+
+/**
+ * setup udf
+ * @param udf, in
+ * @param handle, out
+ * @return error code
+ */
+int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
+
+typedef struct SUdfColumnMeta {
+  int16_t type;
+  int32_t bytes;
+  uint8_t precision;
+  uint8_t scale;
+} SUdfColumnMeta;
+
+typedef struct SUdfColumnData {
+  int32_t numOfRows;
+  int32_t rowsAlloc;
+  union {
+    struct {
+      int32_t nullBitmapLen;
+      char   *nullBitmap;
+      int32_t dataLen;
+      char   *data;
+    } fixLenCol;
+
+    struct {
+      int32_t varOffsetsLen;
+      int32_t   *varOffsets;
+      int32_t payloadLen;
+      char   *payload;
+      int32_t payloadAllocLen;
+    } varLenCol;
+  };
+} SUdfColumnData;
+
+
+typedef struct SUdfColumn {
+  SUdfColumnMeta colMeta;
+  SUdfColumnData colData;
+} SUdfColumn;
+
+typedef struct SUdfDataBlock {
+  int32_t numOfRows;
+  int32_t numOfCols;
+  SUdfColumn **udfCols;
+} SUdfDataBlock;
+
+typedef struct SUdfInterBuf {
+  int32_t bufLen;
+  char* buf;
+  int8_t numOfResult; //zero or one
+} SUdfInterBuf;
+
+// output: interBuf
+int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
+// input: block, state
+// output: newState
+int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
+// input: interBuf
+// output: resultData
+int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
+// input: interbuf1, interbuf2
+// output: resultBuf
+int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
+// input: block
+// output: resultData
+int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+/**
+ * tearn down udf
+ * @param handle
+ * @return
+ */
+int32_t teardownUdf(UdfcFuncHandle handle);
+
+bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
+int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
+// end API to taosd and qworker
+//=============================================================================================================================
+// begin API to UDF writer.
+
+// dynamic lib init and destroy
+typedef int32_t (*TUdfSetupFunc)();
+typedef int32_t (*TUdfTeardownFunc)();
+
+//TODO: add API to check function arguments type, number etc.
+
+#define UDF_MEMORY_EXP_GROWTH 1.5
+
+static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) {
+  SUdfColumnMeta *meta = &pColumn->colMeta;
+  SUdfColumnData *data = &pColumn->colData;
+
+  if (newCapacity== 0 || newCapacity <= data->rowsAlloc) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int allocCapacity = TMAX(data->rowsAlloc, 8);
+  while (allocCapacity < newCapacity) {
+    allocCapacity *= UDF_MEMORY_EXP_GROWTH;
+  }
+
+  if (IS_VAR_DATA_TYPE(meta->type)) {
+    char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity);
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    data->varLenCol.varOffsets = (int32_t*)tmp;
+    data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity;
+    // for payload, add data in udfColDataAppend
+  } else {
+    char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity));
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    data->fixLenCol.nullBitmap = tmp;
+    data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity);
+    if (meta->type == TSDB_DATA_TYPE_NULL) {
+      return TSDB_CODE_SUCCESS;
+    }
+
+    tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes);
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    data->fixLenCol.data = tmp;
+    data->fixLenCol.dataLen = allocCapacity* meta->bytes;
+  }
+
+  data->rowsAlloc = allocCapacity;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) {
+  SUdfColumnMeta *meta = &pColumn->colMeta;
+  SUdfColumnData *data = &pColumn->colData;
+  udfColEnsureCapacity(pColumn, currentRow+1);
+  bool isVarCol = IS_VAR_DATA_TYPE(meta->type);
+  if (isNull) {
+    if (isVarCol) {
+      data->varLenCol.varOffsets[currentRow] = -1;
+    } else {
+      colDataSetNull_f(data->fixLenCol.nullBitmap, currentRow);
+    }
+  } else {
+    if (!isVarCol) {
+      colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow);
+      memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes);
+    } else {
+      int32_t dataLen = varDataTLen(pData);
+      if (meta->type == TSDB_DATA_TYPE_JSON) {
+        if (*pData == TSDB_DATA_TYPE_NULL) {
+          dataLen = 0;
+        } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+          dataLen = varDataTLen(pData + CHAR_BYTES);
+        } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+          dataLen = LONG_BYTES;
+        } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+          dataLen = CHAR_BYTES;
+        }
+        dataLen += CHAR_BYTES;
+      }
+
+      if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) {
+        uint32_t newSize = data->varLenCol.payloadAllocLen;
+        if (newSize <= 1) {
+          newSize = 8;
+        }
+
+        while (newSize < data->varLenCol.payloadLen + dataLen) {
+          newSize = newSize * UDF_MEMORY_EXP_GROWTH;
+        }
+
+        char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize);
+        if (buf == NULL) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+
+        data->varLenCol.payload = buf;
+        data->varLenCol.payloadAllocLen = newSize;
+      }
+
+      uint32_t len = data->varLenCol.payloadLen;
+      data->varLenCol.varOffsets[currentRow] = len;
+
+      memcpy(data->varLenCol.payload + len, pData, dataLen);
+      data->varLenCol.payloadLen += dataLen;
+    }
+  }
+  data->numOfRows = TMAX(currentRow + 1, data->numOfRows);
+  return 0;
+}
+
+typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
+typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
+
+typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
+typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
+typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
+
+
+// end API to UDF writer
+//=======================================================================================================================
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TUDF_H

include/libs/index/index.h

@@ -17,6 +17,7 @@
 #define _TD_INDEX_H_
 
 #include "os.h"
+#include "taoserror.h"
 #include "tarray.h"
 
 #ifdef __cplusplus
@@ -41,11 +42,22 @@ typedef enum {
   UPDATE_VALUE,  // update index column value
   ADD_INDEX,     // add index on specify column
   DROP_INDEX,    // drop existed index
-  DROP_SATBLE    // drop stable
+  DROP_SATBLE,   // drop stable
+  DEFAULT        // query
 } SIndexOperOnColumn;
 
-typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
+typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType;
-typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
+typedef enum {
+  QUERY_TERM = 0,
+  QUERY_PREFIX,
+  QUERY_SUFFIX,
+  QUERY_REGEX,
+  QUERY_LESS_THAN,
+  QUERY_LESS_EQUAL,
+  QUERY_GREATER_THAN,
+  QUERY_GREATER_EQUAL,
+  QUERY_RANGE
+} EIndexQueryType;
 
 /*
  * create multi query
@@ -166,8 +178,8 @@ void        indexOptsDestroy(SIndexOpts* opts);
  * @param:
  */
 
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName,
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType,
-                            int32_t nColName, const char* colVal, int32_t nColVal);
+                            const char* colName, int32_t nColName, const char* colVal, int32_t nColVal);
 void        indexTermDestroy(SIndexTerm* p);
 
 /*

include/libs/nodes/cmdnodes.h

@@ -23,286 +23,291 @@ extern "C" {
 #include "query.h"
 #include "querynodes.h"
 
-#define DESCRIBE_RESULT_COLS 4
+#define DESCRIBE_RESULT_COLS      4
 #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
-#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_TYPE_LEN  (20 + VARSTR_HEADER_SIZE)
-#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_NOTE_LEN  (8 + VARSTR_HEADER_SIZE)
 
 typedef struct SDatabaseOptions {
-  ENodeType type;
+  ENodeType   type;
-  SValueNode* pNumOfBlocks;
+  int32_t     buffer;
-  SValueNode* pCacheBlockSize;
+  int8_t      cachelast;
-  SValueNode* pCachelast;
+  int8_t      compressionLevel;
-  SValueNode* pCompressionLevel;
+  int32_t     daysPerFile;
   SValueNode* pDaysPerFile;
-  SValueNode* pFsyncPeriod;
+  int32_t     fsyncPeriod;
-  SValueNode* pMaxRowsPerBlock;
+  int32_t     maxRowsPerBlock;
-  SValueNode* pMinRowsPerBlock;
+  int32_t     minRowsPerBlock;
-  SNodeList* pKeep;
+  SNodeList*  pKeep;
-  SValueNode* pPrecision;
+  int32_t     keep[3];
-  SValueNode* pQuorum;
+  int32_t     pages;
-  SValueNode* pReplica;
+  int32_t     pagesize;
-  SValueNode* pTtl;
+  char        precisionStr[3];
-  SValueNode* pWalLevel;
+  int8_t      precision;
-  SValueNode* pNumOfVgroups;
+  int8_t      replica;
-  SValueNode* pSingleStable;
+  int8_t      strict;
-  SValueNode* pStreamMode;
+  int8_t      walLevel;
-  SValueNode* pStrict;
+  int32_t     numOfVgroups;
-  SNodeList* pRetentions;
+  int8_t      singleStable;
+  SNodeList*  pRetentions;
 } SDatabaseOptions;
 
 typedef struct SCreateDatabaseStmt {
-  ENodeType type;
+  ENodeType         type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char              dbName[TSDB_DB_NAME_LEN];
-  bool ignoreExists;
+  bool              ignoreExists;
   SDatabaseOptions* pOptions;
 } SCreateDatabaseStmt;
 
 typedef struct SUseDatabaseStmt {
   ENodeType type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
 } SUseDatabaseStmt;
 
 typedef struct SDropDatabaseStmt {
   ENodeType type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
 } SDropDatabaseStmt;
 
 typedef struct SAlterDatabaseStmt {
-  ENodeType type;
+  ENodeType         type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char              dbName[TSDB_DB_NAME_LEN];
   SDatabaseOptions* pOptions;
 } SAlterDatabaseStmt;
 
 typedef struct STableOptions {
-  ENodeType type;
+  ENodeType  type;
-  SNodeList* pKeep;
+  char       comment[TSDB_STB_COMMENT_LEN];
-  SValueNode* pTtl;
+  int32_t    delay;
-  SValueNode* pComments;
+  float      filesFactor;
+  SNodeList* pRollupFuncs;
+  int32_t    ttl;
   SNodeList* pSma;
-  SNodeList* pFuncs;
-  SValueNode* pFilesFactor;
-  SValueNode* pDelay;
 } STableOptions;
 
 typedef struct SColumnDefNode {
   ENodeType type;
-  char colName[TSDB_COL_NAME_LEN];
+  char      colName[TSDB_COL_NAME_LEN];
   SDataType dataType;
-  char comments[TSDB_STB_COMMENT_LEN];
+  char      comments[TSDB_STB_COMMENT_LEN];
-  bool sma;
+  bool      sma;
 } SColumnDefNode;
 
 typedef struct SCreateTableStmt {
-  ENodeType type;
+  ENodeType      type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char           dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char           tableName[TSDB_TABLE_NAME_LEN];
-  bool ignoreExists;
+  bool           ignoreExists;
-  SNodeList* pCols;
+  SNodeList*     pCols;
-  SNodeList* pTags;
+  SNodeList*     pTags;
   STableOptions* pOptions;
 } SCreateTableStmt;
 
 typedef struct SCreateSubTableClause {
-  ENodeType type;
+  ENodeType  type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char       dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char       tableName[TSDB_TABLE_NAME_LEN];
-  char useDbName[TSDB_DB_NAME_LEN];
+  char       useDbName[TSDB_DB_NAME_LEN];
-  char useTableName[TSDB_TABLE_NAME_LEN];
+  char       useTableName[TSDB_TABLE_NAME_LEN];
-  bool ignoreExists;
+  bool       ignoreExists;
   SNodeList* pSpecificTags;
   SNodeList* pValsOfTags;
 } SCreateSubTableClause;
 
 typedef struct SCreateMultiTableStmt {
-  ENodeType type;
+  ENodeType  type;
   SNodeList* pSubTables;
 } SCreateMultiTableStmt;
 
 typedef struct SDropTableClause {
   ENodeType type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
 } SDropTableClause;
 
 typedef struct SDropTableStmt {
-  ENodeType type;
+  ENodeType  type;
   SNodeList* pTables;
 } SDropTableStmt;
 
 typedef struct SDropSuperTableStmt {
   ENodeType type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
 } SDropSuperTableStmt;
 
 typedef struct SAlterTableStmt {
-  ENodeType type;
+  ENodeType      type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char           dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char           tableName[TSDB_TABLE_NAME_LEN];
-  int8_t alterType;
+  int8_t         alterType;
-  char colName[TSDB_COL_NAME_LEN];
+  char           colName[TSDB_COL_NAME_LEN];
-  char newColName[TSDB_COL_NAME_LEN];
+  char           newColName[TSDB_COL_NAME_LEN];
   STableOptions* pOptions;
-  SDataType dataType;
+  SDataType      dataType;
-  SValueNode* pVal;
+  SValueNode*    pVal;
 } SAlterTableStmt;
 
 typedef struct SCreateUserStmt {
   ENodeType type;
-  char useName[TSDB_USER_LEN];
+  char      useName[TSDB_USER_LEN];
-  char password[TSDB_USET_PASSWORD_LEN];
+  char      password[TSDB_USET_PASSWORD_LEN];
 } SCreateUserStmt;
 
 typedef struct SAlterUserStmt {
   ENodeType type;
-  char useName[TSDB_USER_LEN];
+  char      useName[TSDB_USER_LEN];
-  char password[TSDB_USET_PASSWORD_LEN];
+  char      password[TSDB_USET_PASSWORD_LEN];
-  int8_t alterType;
+  int8_t    alterType;
 } SAlterUserStmt;
 
 typedef struct SDropUserStmt {
   ENodeType type;
-  char useName[TSDB_USER_LEN];
+  char      useName[TSDB_USER_LEN];
 } SDropUserStmt;
 
 typedef struct SCreateDnodeStmt {
   ENodeType type;
-  char fqdn[TSDB_FQDN_LEN];
+  char      fqdn[TSDB_FQDN_LEN];
-  int32_t port;
+  int32_t   port;
 } SCreateDnodeStmt;
 
 typedef struct SDropDnodeStmt {
   ENodeType type;
-  int32_t dnodeId;
+  int32_t   dnodeId;
-  char fqdn[TSDB_FQDN_LEN];
+  char      fqdn[TSDB_FQDN_LEN];
-  int32_t port;
+  int32_t   port;
 } SDropDnodeStmt;
 
 typedef struct SAlterDnodeStmt {
   ENodeType type;
-  int32_t dnodeId;
+  int32_t   dnodeId;
-  char config[TSDB_DNODE_CONFIG_LEN];
+  char      config[TSDB_DNODE_CONFIG_LEN];
-  char value[TSDB_DNODE_VALUE_LEN];
+  char      value[TSDB_DNODE_VALUE_LEN];
 } SAlterDnodeStmt;
 
 typedef struct SShowStmt {
   ENodeType type;
-  SNode* pDbName;        // SValueNode
+  SNode*    pDbName;         // SValueNode
-  SNode* pTbNamePattern; // SValueNode
+  SNode*    pTbNamePattern;  // SValueNode
 } SShowStmt;
 
 typedef struct SShowCreatStmt {
   ENodeType type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
 } SShowCreatStmt;
 
-typedef enum EIndexType {
+typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType;
-  INDEX_TYPE_SMA = 1,
-  INDEX_TYPE_FULLTEXT
-} EIndexType;
 
 typedef struct SIndexOptions {
-  ENodeType type;
+  ENodeType  type;
   SNodeList* pFuncs;
-  SNode* pInterval;
+  SNode*     pInterval;
-  SNode* pOffset;
+  SNode*     pOffset;
-  SNode* pSliding;
+  SNode*     pSliding;
 } SIndexOptions;
 
 typedef struct SCreateIndexStmt {
-  ENodeType type;
+  ENodeType      type;
-  EIndexType indexType;
+  EIndexType     indexType;
-  bool ignoreExists;
+  bool           ignoreExists;
-  char indexName[TSDB_INDEX_NAME_LEN];
+  char           indexName[TSDB_INDEX_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char           tableName[TSDB_TABLE_NAME_LEN];
-  SNodeList* pCols;
+  SNodeList*     pCols;
   SIndexOptions* pOptions;
 } SCreateIndexStmt;
 
 typedef struct SDropIndexStmt {
   ENodeType type;
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
-  char indexName[TSDB_INDEX_NAME_LEN];
+  char      indexName[TSDB_INDEX_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
 } SDropIndexStmt;
 
 typedef struct SCreateComponentNodeStmt {
   ENodeType type;
-  int32_t dnodeId;
+  int32_t   dnodeId;
 } SCreateComponentNodeStmt;
 
 typedef struct SDropComponentNodeStmt {
   ENodeType type;
-  int32_t dnodeId;
+  int32_t   dnodeId;
 } SDropComponentNodeStmt;
 
-typedef struct SCreateTopicStmt {
+typedef struct STopicOptions {
   ENodeType type;
-  char topicName[TSDB_TABLE_NAME_LEN];
+  bool      withTable;
-  char subscribeDbName[TSDB_DB_NAME_LEN];
+  bool      withSchema;
-  bool ignoreExists;
+  bool      withTag;
-  SNode* pQuery;
+} STopicOptions;
+
+typedef struct SCreateTopicStmt {
+  ENodeType      type;
+  char           topicName[TSDB_TABLE_NAME_LEN];
+  char           subscribeDbName[TSDB_DB_NAME_LEN];
+  bool           ignoreExists;
+  SNode*         pQuery;
+  STopicOptions* pOptions;
 } SCreateTopicStmt;
 
 typedef struct SDropTopicStmt {
   ENodeType type;
-  char topicName[TSDB_TABLE_NAME_LEN];
+  char      topicName[TSDB_TABLE_NAME_LEN];
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
 } SDropTopicStmt;
 
 typedef struct SAlterLocalStmt {
   ENodeType type;
-  char config[TSDB_DNODE_CONFIG_LEN];
+  char      config[TSDB_DNODE_CONFIG_LEN];
-  char value[TSDB_DNODE_VALUE_LEN];
+  char      value[TSDB_DNODE_VALUE_LEN];
 } SAlterLocalStmt;
 
 typedef struct SDescribeStmt {
-  ENodeType type;
+  ENodeType   type;
-  char dbName[TSDB_DB_NAME_LEN];
+  char        dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char        tableName[TSDB_TABLE_NAME_LEN];
   STableMeta* pMeta;
 } SDescribeStmt;
 
 typedef struct SKillStmt {
   ENodeType type;
-  int32_t targetId;
+  int32_t   targetId;
 } SKillStmt;
 
 typedef struct SStreamOptions {
   ENodeType type;
-  int8_t triggerType;
+  int8_t    triggerType;
-  SNode* pWatermark;
+  SNode*    pWatermark;
 } SStreamOptions;
 
 typedef struct SCreateStreamStmt {
-  ENodeType type;
+  ENodeType       type;
-  char streamName[TSDB_TABLE_NAME_LEN];
+  char            streamName[TSDB_TABLE_NAME_LEN];
-  char targetDbName[TSDB_DB_NAME_LEN];
+  char            targetDbName[TSDB_DB_NAME_LEN];
-  char targetTabName[TSDB_TABLE_NAME_LEN];
+  char            targetTabName[TSDB_TABLE_NAME_LEN];
-  bool ignoreExists;
+  bool            ignoreExists;
   SStreamOptions* pOptions;
-  SNode* pQuery;
+  SNode*          pQuery;
 } SCreateStreamStmt;
 
 typedef struct SDropStreamStmt {
   ENodeType type;
-  char streamName[TSDB_TABLE_NAME_LEN];
+  char      streamName[TSDB_TABLE_NAME_LEN];
-  bool ignoreNotExists;
+  bool      ignoreNotExists;
 } SDropStreamStmt;
 
 typedef struct SCreateFunctionStmt {
   ENodeType type;
-  bool ignoreExists;
+  bool      ignoreExists;
-  char funcName[TSDB_FUNC_NAME_LEN];
+  char      funcName[TSDB_FUNC_NAME_LEN];
-  bool isAgg;
+  bool      isAgg;
-  char libraryPath[PATH_MAX];
+  char      libraryPath[PATH_MAX];
   SDataType outputDt;
-  int32_t bufSize;
+  int32_t   bufSize;
 } SCreateFunctionStmt;
 
 #ifdef __cplusplus

include/libs/nodes/nodes.h

@@ -22,19 +22,20 @@ extern "C" {
 
 #include "tdef.h"
 
-#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type)
+#define nodeType(nodeptr)          (((const SNode*)(nodeptr))->type)
 #define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type))
 
 #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0)
 
-#define FOREACH(node, list)	\
+#define FOREACH(node, list)                                       \
-  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
+  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
+       (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
 
 #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
 
 #define INSERT_LIST(target, src) nodesListInsertList((target), cell, src)
 
-#define WHERE_EACH(node, list) \
+#define WHERE_EACH(node, list)                               \
   SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
   while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false))
 
@@ -43,16 +44,26 @@ extern "C" {
 // only be use in WHERE_EACH
 #define ERASE_NODE(list) cell = nodesListErase((list), cell)
 
-#define FORBOTH(node1, list1, node2, list2) \
+#define FORBOTH(node1, list1, node2, list2)                                               \
-  for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \
+  for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL),                      \
-    (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
+                  *cell2 = (NULL != (list2) ? (list2)->pHead : NULL);                     \
-    cell1 = cell1->pNext, cell2 = cell2->pNext)
+       (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)),            \
+                  (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), \
+                  (node1 != NULL && node2 != NULL);                                       \
+       cell1 = cell1->pNext, cell2 = cell2->pNext)
 
 #define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode)
 #define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode)
 
-#define FOREACH_FOR_REWRITE(node, list)	\
+#define FOREACH_FOR_REWRITE(node, list)                           \
-  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
+  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
+       (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
+
+#define DESTORY_LIST(list)  \
+  do {                      \
+    nodesDestroyList(list); \
+    list = NULL;            \
+  } while (0)
 
 typedef enum ENodeType {
   // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
@@ -84,6 +95,7 @@ typedef enum ENodeType {
   QUERY_NODE_INDEX_OPTIONS,
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
+  QUERY_NODE_TOPIC_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
@@ -173,6 +185,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
   QUERY_NODE_LOGIC_PLAN_EXCHANGE,
   QUERY_NODE_LOGIC_PLAN_WINDOW,
+  QUERY_NODE_LOGIC_PLAN_FILL,
   QUERY_NODE_LOGIC_PLAN_SORT,
   QUERY_NODE_LOGIC_PLAN_PARTITION,
   QUERY_NODE_LOGIC_SUBPLAN,
@@ -190,6 +203,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
   QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_FILL,
   QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_PARTITION,
@@ -210,11 +224,11 @@ typedef struct SNode {
 typedef struct SListCell {
   struct SListCell* pPrev;
   struct SListCell* pNext;
-  SNode* pNode;
+  SNode*            pNode;
 } SListCell;
 
 typedef struct SNodeList {
-  int32_t length;
+  int32_t    length;
   SListCell* pHead;
   SListCell* pTail;
 } SNodeList;
@@ -222,29 +236,24 @@ typedef struct SNodeList {
 #define SNodeptr void*
 
 SNodeptr nodesMakeNode(ENodeType type);
-void nodesDestroyNode(SNodeptr pNode);
+void     nodesDestroyNode(SNodeptr pNode);
 
 SNodeList* nodesMakeList();
-int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
+int32_t    nodesListAppend(SNodeList* pList, SNodeptr pNode);
-int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
+int32_t    nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
-int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
+int32_t    nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
-int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode);
+int32_t    nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode);
-int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
+int32_t    nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
-int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
+int32_t    nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
-int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode);
+int32_t    nodesListPushFront(SNodeList* pList, SNodeptr pNode);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
-void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
+void       nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
-SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
+SNodeptr   nodesListGetNode(SNodeList* pList, int32_t index);
-void nodesDestroyList(SNodeList* pList);
+void       nodesDestroyList(SNodeList* pList);
 // Only clear the linked list structure, without releasing the elements inside
 void nodesClearList(SNodeList* pList);
 
-typedef enum EDealRes {
+typedef enum EDealRes { DEAL_RES_CONTINUE = 1, DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, DEAL_RES_END } EDealRes;
-  DEAL_RES_CONTINUE = 1,
-  DEAL_RES_IGNORE_CHILD,
-  DEAL_RES_ERROR,
-  DEAL_RES_END
-} EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
 void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext);
@@ -260,18 +269,18 @@ void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void*
 
 bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
 
-SNodeptr nodesCloneNode(const SNodeptr pNode);
+SNodeptr   nodesCloneNode(const SNodeptr pNode);
 SNodeList* nodesCloneList(const SNodeList* pList);
 
 const char* nodesNodeName(ENodeType type);
-int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen);
+int32_t     nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen);
-int32_t nodesStringToNode(const char* pStr, SNode** pNode);
+int32_t     nodesStringToNode(const char* pStr, SNode** pNode);
 
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
-int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
+int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len);
-char *nodesGetNameFromColumnNode(SNode *pNode);
+char*   nodesGetNameFromColumnNode(SNode* pNode);
 int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);
 
 #ifdef __cplusplus

include/libs/nodes/plannodes.h

@@ -20,50 +20,46 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 #include "tname.h"
 
 typedef struct SLogicNode {
-  ENodeType type;
+  ENodeType          type;
-  SNodeList* pTargets; // SColumnNode
+  SNodeList*         pTargets;  // SColumnNode
-  SNode* pConditions;
+  SNode*             pConditions;
-  SNodeList* pChildren;
+  SNodeList*         pChildren;
   struct SLogicNode* pParent;
-  int32_t optimizedFlag;
+  int32_t            optimizedFlag;
 } SLogicNode;
 
-typedef enum EScanType {
+typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
-  SCAN_TYPE_TAG,
-  SCAN_TYPE_TABLE,
-  SCAN_TYPE_SYSTEM_TABLE,
-  SCAN_TYPE_STREAM
-} EScanType;
 
 typedef struct SScanLogicNode {
-  SLogicNode node;
+  SLogicNode         node;
-  SNodeList* pScanCols;
+  SNodeList*         pScanCols;
+  SNodeList*         pScanPseudoCols;
   struct STableMeta* pMeta;
-  SVgroupsInfo* pVgroupList;
+  SVgroupsInfo*      pVgroupList;
-  EScanType scanType;
+  EScanType          scanType;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t            scanSeq[2];  // first is scan count, and second is reverse scan count
-  STimeWindow scanRange;
+  STimeWindow        scanRange;
-  SName tableName;
+  SName              tableName;
-  bool showRewrite;
+  bool               showRewrite;
-  double ratio;
+  double             ratio;
-  SNodeList* pDynamicScanFuncs;
+  SNodeList*         pDynamicScanFuncs;
-  int32_t dataRequired;
+  int32_t            dataRequired;
-  int64_t interval;
+  int64_t            interval;
-  int64_t offset;
+  int64_t            offset;
-  int64_t sliding;
+  int64_t            sliding;
-  int8_t intervalUnit;
+  int8_t             intervalUnit;
-  int8_t slidingUnit;
+  int8_t             slidingUnit;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
   SLogicNode node;
-  EJoinType joinType;
+  EJoinType  joinType;
-  SNode* pOnConditions;
+  SNode*     pOnConditions;
 } SJoinLogicNode;
 
 typedef struct SAggLogicNode {
@@ -75,49 +71,52 @@ typedef struct SAggLogicNode {
 typedef struct SProjectLogicNode {
   SLogicNode node;
   SNodeList* pProjections;
-  char stmtName[TSDB_TABLE_NAME_LEN];
+  char       stmtName[TSDB_TABLE_NAME_LEN];
-  int64_t limit;
+  int64_t    limit;
-  int64_t offset;
+  int64_t    offset;
-  int64_t slimit;
+  int64_t    slimit;
-  int64_t soffset;
+  int64_t    soffset;
 } SProjectLogicNode;
 
 typedef struct SVnodeModifLogicNode {
-  SLogicNode node;
+  SLogicNode     node;
-  int32_t msgType;
+  int32_t        msgType;
-  SArray* pDataBlocks;
+  SArray*        pDataBlocks;
   SVgDataBlocks* pVgDataBlocks;
 } SVnodeModifLogicNode;
 
 typedef struct SExchangeLogicNode {
   SLogicNode node;
-  int32_t srcGroupId;
+  int32_t    srcGroupId;
-  uint8_t precision;
+  uint8_t    precision;
 } SExchangeLogicNode;
 
-typedef enum EWindowType {
+typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
-  WINDOW_TYPE_INTERVAL = 1,
-  WINDOW_TYPE_SESSION,
-  WINDOW_TYPE_STATE
-} EWindowType;
 
 typedef struct SWindowLogicNode {
-  SLogicNode node;
+  SLogicNode  node;
   EWindowType winType;
-  SNodeList* pFuncs;
+  SNodeList*  pFuncs;
-  int64_t interval;
+  int64_t     interval;
-  int64_t offset;
+  int64_t     offset;
-  int64_t sliding;
+  int64_t     sliding;
-  int8_t  intervalUnit;
+  int8_t      intervalUnit;
-  int8_t  slidingUnit;
+  int8_t      slidingUnit;
-  SFillNode* pFill;
+  int64_t     sessionGap;
-  int64_t sessionGap;
+  SNode*      pTspk;
-  SNode* pTspk;
+  SNode*      pStateExpr;
-  SNode* pStateExpr;
+  int8_t      triggerType;
-  int8_t triggerType;
+  int64_t     watermark;
-  int64_t watermark;
 } SWindowLogicNode;
 
+typedef struct SFillLogicNode {
+  SLogicNode  node;
+  EFillMode   mode;
+  SNode*      pWStartTs;
+  SNode*      pValues;  // SNodeListNode
+  STimeWindow timeRange;
+} SFillLogicNode;
+
 typedef struct SSortLogicNode {
   SLogicNode node;
   SNodeList* pSortKeys;
@@ -137,62 +136,60 @@ typedef enum ESubplanType {
 
 typedef struct SSubplanId {
   uint64_t queryId;
-  int32_t groupId;
+  int32_t  groupId;
-  int32_t subplanId;
+  int32_t  subplanId;
 } SSubplanId;
 
 typedef struct SLogicSubplan {
-  ENodeType type;
+  ENodeType     type;
-  SSubplanId id;
+  SSubplanId    id;
-  SNodeList* pChildren;
+  SNodeList*    pChildren;
-  SNodeList* pParents;
+  SNodeList*    pParents;
-  SLogicNode* pNode;
+  SLogicNode*   pNode;
-  ESubplanType subplanType;
+  ESubplanType  subplanType;
   SVgroupsInfo* pVgroupList;
-  int32_t level;
+  int32_t       level;
-  int32_t splitFlag;
+  int32_t       splitFlag;
 } SLogicSubplan;
 
 typedef struct SQueryLogicPlan {
-  ENodeType type;
+  ENodeType  type;
   SNodeList* pTopSubplans;
 } SQueryLogicPlan;
 
 typedef struct SSlotDescNode {
   ENodeType type;
-  int16_t slotId;
+  int16_t   slotId;
   SDataType dataType;
-  bool reserve;
+  bool      reserve;
-  bool output;
+  bool      output;
-  bool tag;
+  bool      tag;
 } SSlotDescNode;
 
 typedef struct SDataBlockDescNode {
-  ENodeType type;
+  ENodeType  type;
-  int16_t dataBlockId;
+  int16_t    dataBlockId;
   SNodeList* pSlots;
-  int32_t totalRowSize;
+  int32_t    totalRowSize;
-  int32_t outputRowSize;
+  int32_t    outputRowSize;
-  uint8_t precision;
+  uint8_t    precision;
 } SDataBlockDescNode;
 
 typedef struct SPhysiNode {
-  ENodeType type;
+  ENodeType           type;
   SDataBlockDescNode* pOutputDataBlockDesc;
-  SNode* pConditions;
+  SNode*              pConditions;
-  SNodeList* pChildren;
+  SNodeList*          pChildren;
-  struct SPhysiNode* pParent;
+  struct SPhysiNode*  pParent;
 } SPhysiNode;
 
 typedef struct SScanPhysiNode {
   SPhysiNode node;
   SNodeList* pScanCols;
-  uint64_t uid;           // unique id of the table
+  SNodeList* pScanPseudoCols;
-  int8_t tableType;
+  uint64_t   uid;  // unique id of the table
-  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
+  int8_t     tableType;
-  int32_t count;         // repeat count
+  SName      tableName;
-  int32_t reverse;       // reverse scan count
-  SName tableName;
 } SScanPhysiNode;
 
 typedef SScanPhysiNode STagScanPhysiNode;
@@ -200,23 +197,23 @@ typedef SScanPhysiNode SStreamScanPhysiNode;
 
 typedef struct SSystemTableScanPhysiNode {
   SScanPhysiNode scan;
-  SEpSet mgmtEpSet;
+  SEpSet         mgmtEpSet;
-  bool showRewrite;
+  bool           showRewrite;
-  int32_t accountId;
+  int32_t        accountId;
 } SSystemTableScanPhysiNode;
 
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t        scanSeq[2];  // first is scan count, and second is reverse scan count
-  STimeWindow scanRange;
+  STimeWindow    scanRange;
-  double ratio;
+  double         ratio;
-  int32_t dataRequired;
+  int32_t        dataRequired;
-  SNodeList* pDynamicScanFuncs;
+  SNodeList*     pDynamicScanFuncs;
-  int64_t interval;
+  int64_t        interval;
-  int64_t offset;
+  int64_t        offset;
-  int64_t sliding;
+  int64_t        sliding;
-  int8_t intervalUnit;
+  int8_t         intervalUnit;
-  int8_t slidingUnit;
+  int8_t         slidingUnit;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
@@ -224,89 +221,99 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
-  int64_t limit;
+  int64_t    limit;
-  int64_t offset;
+  int64_t    offset;
-  int64_t slimit;
+  int64_t    slimit;
-  int64_t soffset;
+  int64_t    soffset;
 } SProjectPhysiNode;
 
 typedef struct SJoinPhysiNode {
   SPhysiNode node;
-  EJoinType joinType;
+  EJoinType  joinType;
-  SNode* pOnConditions; // in or out tuple ?
+  SNode*     pOnConditions;
   SNodeList* pTargets;
 } SJoinPhysiNode;
 
+typedef SJoinPhysiNode SSortMergeJoinPhysiNode;
+
 typedef struct SAggPhysiNode {
   SPhysiNode node;
-  SNodeList* pExprs;   // these are expression list of group_by_clause and parameter expression of aggregate function
+  SNodeList* pExprs;  // these are expression list of group_by_clause and parameter expression of aggregate function
   SNodeList* pGroupKeys;
   SNodeList* pAggFuncs;
 } SAggPhysiNode;
 
 typedef struct SDownstreamSourceNode {
-  ENodeType type;
+  ENodeType      type;
   SQueryNodeAddr addr;
-  uint64_t taskId;
+  uint64_t       taskId;
-  uint64_t schedId;
+  uint64_t       schedId;
 } SDownstreamSourceNode;
 
 typedef struct SExchangePhysiNode {
   SPhysiNode node;
-  int32_t    srcGroupId;  // group id of datasource suplans
+  int32_t    srcGroupId;     // group id of datasource suplans
   SNodeList* pSrcEndPoints;  // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
 } SExchangePhysiNode;
 
 typedef struct SWinodwPhysiNode {
   SPhysiNode node;
-  SNodeList* pExprs;   // these are expression list of parameter expression of function
+  SNodeList* pExprs;  // these are expression list of parameter expression of function
   SNodeList* pFuncs;
-  SNode* pTspk; // timestamp primary key
+  SNode*     pTspk;  // timestamp primary key
-  int8_t triggerType;
+  int8_t     triggerType;
-  int64_t watermark;
+  int64_t    watermark;
 } SWinodwPhysiNode;
 
 typedef struct SIntervalPhysiNode {
   SWinodwPhysiNode window;
-  int64_t    interval;
+  int64_t          interval;
-  int64_t    offset;
+  int64_t          offset;
-  int64_t    sliding;
+  int64_t          sliding;
-  int8_t     intervalUnit;
+  int8_t           intervalUnit;
-  int8_t     slidingUnit;
+  int8_t           slidingUnit;
-  SFillNode* pFill;
 } SIntervalPhysiNode;
 
+typedef struct SFillPhysiNode {
+  SPhysiNode  node;
+  EFillMode   mode;
+  SNode*      pWStartTs;  // SColumnNode
+  SNode*      pValues;    // SNodeListNode
+  SNodeList*  pTargets;
+  STimeWindow timeRange;
+} SFillPhysiNode;
+
 typedef struct SMultiTableIntervalPhysiNode {
   SIntervalPhysiNode interval;
-  SNodeList* pPartitionKeys;
+  SNodeList*         pPartitionKeys;
 } SMultiTableIntervalPhysiNode;
 
 typedef struct SSessionWinodwPhysiNode {
   SWinodwPhysiNode window;
-  int64_t gap;
+  int64_t          gap;
 } SSessionWinodwPhysiNode;
 
 typedef struct SStateWinodwPhysiNode {
   SWinodwPhysiNode window;
-  SNode* pStateKey;
+  SNode*           pStateKey;
 } SStateWinodwPhysiNode;
 
 typedef struct SSortPhysiNode {
   SPhysiNode node;
-  SNodeList* pExprs;   // these are expression list of order_by_clause and parameter expression of aggregate function
+  SNodeList* pExprs;     // these are expression list of order_by_clause and parameter expression of aggregate function
-  SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
+  SNodeList* pSortKeys;  // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
   SNodeList* pTargets;
 } SSortPhysiNode;
 
 typedef struct SPartitionPhysiNode {
   SPhysiNode node;
-  SNodeList* pExprs;   // these are expression list of partition_by_clause
+  SNodeList* pExprs;  // these are expression list of partition_by_clause
   SNodeList* pPartitionKeys;
   SNodeList* pTargets;
 } SPartitionPhysiNode;
 
 typedef struct SDataSinkNode {
-  ENodeType type;
+  ENodeType           type;
   SDataBlockDescNode* pInputDataBlockDesc;
 } SDataSinkNode;
 
@@ -316,45 +323,41 @@ typedef struct SDataDispatcherNode {
 
 typedef struct SDataInserterNode {
   SDataSinkNode sink;
-  int32_t   numOfTables;
+  int32_t       numOfTables;
-  uint32_t  size;
+  uint32_t      size;
-  char     *pData;
+  char*         pData;
 } SDataInserterNode;
 
 typedef struct SSubplan {
-  ENodeType type;
+  ENodeType      type;
-  SSubplanId id;           // unique id of the subplan
+  SSubplanId     id;  // unique id of the subplan
-  ESubplanType subplanType;
+  ESubplanType   subplanType;
-  int32_t msgType;      // message type for subplan, used to denote the send message type to vnode.
+  int32_t        msgType;  // message type for subplan, used to denote the send message type to vnode.
-  int32_t level;        // the execution level of current subplan, starting from 0 in a top-down manner.
+  int32_t        level;    // the execution level of current subplan, starting from 0 in a top-down manner.
-  char dbFName[TSDB_DB_FNAME_LEN];
+  char           dbFName[TSDB_DB_FNAME_LEN];
-  SQueryNodeAddr execNode;    // for the scan/modify subplan, the optional execution node
+  SQueryNodeAddr execNode;      // for the scan/modify subplan, the optional execution node
-  SQueryNodeStat execNodeStat; // only for scan subplan
+  SQueryNodeStat execNodeStat;  // only for scan subplan
-  SNodeList* pChildren;    // the datasource subplan,from which to fetch the result
+  SNodeList*     pChildren;     // the datasource subplan,from which to fetch the result
-  SNodeList* pParents;     // the data destination subplan, get data from current subplan
+  SNodeList*     pParents;      // the data destination subplan, get data from current subplan
-  SPhysiNode* pNode;        // physical plan of current subplan
+  SPhysiNode*    pNode;         // physical plan of current subplan
-  SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
+  SDataSinkNode* pDataSink;     // data of the subplan flow into the datasink
 } SSubplan;
 
-typedef enum EExplainMode {
+typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
-  EXPLAIN_MODE_DISABLE = 1,
-  EXPLAIN_MODE_STATIC,
-  EXPLAIN_MODE_ANALYZE
-} EExplainMode;
 
 typedef struct SExplainInfo {
   EExplainMode mode;
-  bool verbose;
+  bool         verbose;
-  double ratio;
+  double       ratio;
 } SExplainInfo;
 
 typedef struct SQueryPlan {
-  ENodeType type;
+  ENodeType    type;
-  uint64_t queryId;
+  uint64_t     queryId;
-  int32_t numOfSubplans;
+  int32_t      numOfSubplans;
-  SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
+  SNodeList*   pSubplans;  // Element is SNodeListNode. The execution level of subplan, starting from 0.
   SExplainInfo explainInfo;
-  SNodeList* pPlaceholderValues;
+  SArray*      pPlaceholderValues;
 } SQueryPlan;
 
 void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

include/libs/nodes/querynodes.h

@@ -25,14 +25,16 @@ extern "C" {
 #include "tvariant.h"
 
 #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
-#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define TABLE_META_SIZE(pMeta) \
-#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
+  (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define VGROUPS_INFO_SIZE(pInfo) \
+  (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
 
 typedef struct SRawExprNode {
   ENodeType nodeType;
-  char* p;
+  char*     p;
-  uint32_t n;
+  uint32_t  n;
-  SNode* pNode;
+  SNode*    pNode;
 } SRawExprNode;
 
 typedef struct SDataType {
@@ -45,170 +47,156 @@ typedef struct SDataType {
 typedef struct SExprNode {
   ENodeType type;
   SDataType resType;
-  char aliasName[TSDB_COL_NAME_LEN];
+  char      aliasName[TSDB_COL_NAME_LEN];
-  SNodeList* pAssociationList;
+  SArray*   pAssociation;
 } SExprNode;
 
-typedef enum EColumnType {
+typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType;
-  COLUMN_TYPE_COLUMN = 1,
-  COLUMN_TYPE_TAG
-} EColumnType;
 
 typedef struct SColumnNode {
-  SExprNode node; // QUERY_NODE_COLUMN
+  SExprNode   node;  // QUERY_NODE_COLUMN
-  uint64_t tableId;
+  uint64_t    tableId;
-  int8_t tableType;
+  int8_t      tableType;
-  col_id_t colId;
+  col_id_t    colId;
-  EColumnType colType; // column or tag
+  EColumnType colType;  // column or tag
-  char dbName[TSDB_DB_NAME_LEN];
+  char        dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char        tableName[TSDB_TABLE_NAME_LEN];
-  char tableAlias[TSDB_TABLE_NAME_LEN];
+  char        tableAlias[TSDB_TABLE_NAME_LEN];
-  char colName[TSDB_COL_NAME_LEN];
+  char        colName[TSDB_COL_NAME_LEN];
-  SNode* pProjectRef;
+  SNode*      pProjectRef;
-  int16_t dataBlockId;
+  int16_t     dataBlockId;
-  int16_t slotId;
+  int16_t     slotId;
 } SColumnNode;
 
 typedef struct STargetNode {
   ENodeType type;
-  int16_t dataBlockId;
+  int16_t   dataBlockId;
-  int16_t slotId;
+  int16_t   slotId;
-  SNode* pExpr;
+  SNode*    pExpr;
 } STargetNode;
 
 typedef struct SValueNode {
-  SExprNode node; // QUERY_NODE_VALUE
+  SExprNode node;  // QUERY_NODE_VALUE
-  char* literal;
+  char*     literal;
-  bool isDuration;
+  bool      isDuration;
-  bool translate;
+  bool      translate;
-  bool genByCalc;
+  int16_t   placeholderNo;
-  int16_t placeholderNo;
   union {
-    bool b;
+    bool     b;
-    int64_t i;
+    int64_t  i;
     uint64_t u;
-    double d;
+    double   d;
-    char* p;
+    char*    p;
   } datum;
-  char unit;
+  int64_t typeData;
+  char    unit;
 } SValueNode;
 
 typedef struct SOperatorNode {
-  SExprNode node; // QUERY_NODE_OPERATOR
+  SExprNode     node;  // QUERY_NODE_OPERATOR
   EOperatorType opType;
-  SNode* pLeft;
+  SNode*        pLeft;
-  SNode* pRight;
+  SNode*        pRight;
 } SOperatorNode;
 
-
 typedef struct SLogicConditionNode {
-  SExprNode node; // QUERY_NODE_LOGIC_CONDITION
+  SExprNode           node;  // QUERY_NODE_LOGIC_CONDITION
   ELogicConditionType condType;
-  SNodeList* pParameterList;
+  SNodeList*          pParameterList;
 } SLogicConditionNode;
 
 typedef struct SNodeListNode {
-  ENodeType type; // QUERY_NODE_NODE_LIST
+  ENodeType  type;  // QUERY_NODE_NODE_LIST
-  SDataType dataType;
+  SDataType  dataType;
   SNodeList* pNodeList;
 } SNodeListNode;
 
 typedef struct SFunctionNode {
-  SExprNode node; // QUERY_NODE_FUNCTION
+  SExprNode  node;  // QUERY_NODE_FUNCTION
-  char functionName[TSDB_FUNC_NAME_LEN];
+  char       functionName[TSDB_FUNC_NAME_LEN];
-  int32_t funcId;
+  int32_t    funcId;
-  int32_t funcType;
+  int32_t    funcType;
   SNodeList* pParameterList;
+  int32_t    udfBufSize;
 } SFunctionNode;
 
 typedef struct STableNode {
   SExprNode node;
-  char dbName[TSDB_DB_NAME_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
-  char tableName[TSDB_TABLE_NAME_LEN];
+  char      tableName[TSDB_TABLE_NAME_LEN];
-  char tableAlias[TSDB_TABLE_NAME_LEN];
+  char      tableAlias[TSDB_TABLE_NAME_LEN];
-  uint8_t precision;
+  uint8_t   precision;
 } STableNode;
 
 struct STableMeta;
 
 typedef struct SRealTableNode {
-  STableNode table; // QUERY_NODE_REAL_TABLE
+  STableNode         table;  // QUERY_NODE_REAL_TABLE
   struct STableMeta* pMeta;
-  SVgroupsInfo* pVgroupList;
+  SVgroupsInfo*      pVgroupList;
-  char useDbName[TSDB_DB_NAME_LEN];
+  char               qualDbName[TSDB_DB_NAME_LEN];  // SHOW qualDbName.TABLES
-  double ratio;
+  double             ratio;
 } SRealTableNode;
 
 typedef struct STempTableNode {
-  STableNode table; // QUERY_NODE_TEMP_TABLE
+  STableNode table;  // QUERY_NODE_TEMP_TABLE
-  SNode* pSubquery;
+  SNode*     pSubquery;
 } STempTableNode;
 
-typedef enum EJoinType {
+typedef enum EJoinType { JOIN_TYPE_INNER = 1 } EJoinType;
-  JOIN_TYPE_INNER = 1
-} EJoinType;
 
 typedef struct SJoinTableNode {
-  STableNode table; // QUERY_NODE_JOIN_TABLE
+  STableNode table;  // QUERY_NODE_JOIN_TABLE
-  EJoinType joinType;
+  EJoinType  joinType;
-  SNode* pLeft;
+  SNode*     pLeft;
-  SNode* pRight;
+  SNode*     pRight;
-  SNode* pOnCond;
+  SNode*     pOnCond;
 } SJoinTableNode;
 
-typedef enum EGroupingSetType {
+typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
-  GP_TYPE_NORMAL = 1
-} EGroupingSetType;
 
 typedef struct SGroupingSetNode {
-  ENodeType type; // QUERY_NODE_GROUPING_SET
+  ENodeType        type;  // QUERY_NODE_GROUPING_SET
   EGroupingSetType groupingSetType;
-  SNodeList* pParameterList; 
+  SNodeList*       pParameterList;
 } SGroupingSetNode;
 
-typedef enum EOrder {
+typedef enum EOrder { ORDER_ASC = 1, ORDER_DESC } EOrder;
-  ORDER_ASC = 1,
-  ORDER_DESC
-} EOrder;
 
-typedef enum ENullOrder {
+typedef enum ENullOrder { NULL_ORDER_DEFAULT = 1, NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder;
-  NULL_ORDER_DEFAULT = 1,
-  NULL_ORDER_FIRST,
-  NULL_ORDER_LAST
-} ENullOrder;
 
 typedef struct SOrderByExprNode {
-  ENodeType type; // QUERY_NODE_ORDER_BY_EXPR
+  ENodeType  type;  // QUERY_NODE_ORDER_BY_EXPR
-  SNode* pExpr;
+  SNode*     pExpr;
-  EOrder order;
+  EOrder     order;
   ENullOrder nullOrder;
 } SOrderByExprNode;
 
 typedef struct SLimitNode {
-  ENodeType type; // QUERY_NODE_LIMIT
+  ENodeType type;  // QUERY_NODE_LIMIT
-  int64_t limit;
+  int64_t   limit;
-  int64_t offset;
+  int64_t   offset;
 } SLimitNode;
 
 typedef struct SStateWindowNode {
-  ENodeType type; // QUERY_NODE_STATE_WINDOW
+  ENodeType type;  // QUERY_NODE_STATE_WINDOW
-  SNode* pCol; // timestamp primary key
+  SNode*    pCol;  // timestamp primary key
-  SNode* pExpr;
+  SNode*    pExpr;
 } SStateWindowNode;
 
 typedef struct SSessionWindowNode {
-  ENodeType type; // QUERY_NODE_SESSION_WINDOW
+  ENodeType    type;  // QUERY_NODE_SESSION_WINDOW
-  SColumnNode* pCol; // timestamp primary key
+  SColumnNode* pCol;  // timestamp primary key
-  SValueNode* pGap; // gap between two session window(in microseconds)
+  SValueNode*  pGap;  // gap between two session window(in microseconds)
 } SSessionWindowNode;
 
 typedef struct SIntervalWindowNode {
-  ENodeType type; // QUERY_NODE_INTERVAL_WINDOW
+  ENodeType type;       // QUERY_NODE_INTERVAL_WINDOW
-  SNode* pCol; // timestamp primary key
+  SNode*    pCol;       // timestamp primary key
-  SNode* pInterval; // SValueNode
+  SNode*    pInterval;  // SValueNode
-  SNode* pOffset;   // SValueNode
+  SNode*    pOffset;    // SValueNode
-  SNode* pSliding;  // SValueNode
+  SNode*    pSliding;   // SValueNode
-  SNode* pFill;
+  SNode*    pFill;
 } SIntervalWindowNode;
 
 typedef enum EFillMode {
@@ -221,42 +209,43 @@ typedef enum EFillMode {
 } EFillMode;
 
 typedef struct SFillNode {
-  ENodeType type; // QUERY_NODE_FILL
+  ENodeType   type;  // QUERY_NODE_FILL
-  EFillMode mode;
+  EFillMode   mode;
-  SNode* pValues; // SNodeListNode
+  SNode*      pValues;    // SNodeListNode
+  SNode*      pWStartTs;  // _wstartts pseudo column
+  STimeWindow timeRange;
 } SFillNode;
 
 typedef struct SSelectStmt {
-  ENodeType type; // QUERY_NODE_SELECT_STMT
+  ENodeType   type;  // QUERY_NODE_SELECT_STMT
-  bool isDistinct;
+  bool        isDistinct;
-  SNodeList* pProjectionList;
+  SNodeList*  pProjectionList;
-  SNode* pFromTable;
+  SNode*      pFromTable;
-  SNode* pWhere;
+  SNode*      pWhere;
-  SNodeList* pPartitionByList;
+  SNodeList*  pPartitionByList;
-  SNode* pWindow;
+  SNode*      pWindow;
-  SNodeList* pGroupByList; // SGroupingSetNode
+  SNodeList*  pGroupByList;  // SGroupingSetNode
-  SNode* pHaving;
+  SNode*      pHaving;
-  SNodeList* pOrderByList; // SOrderByExprNode
+  SNodeList*  pOrderByList;  // SOrderByExprNode
-  SNode* pLimit;
+  SLimitNode* pLimit;
-  SNode* pSlimit;
+  SLimitNode* pSlimit;
-  char stmtName[TSDB_TABLE_NAME_LEN];
+  char        stmtName[TSDB_TABLE_NAME_LEN];
-  uint8_t precision;
+  uint8_t     precision;
-  bool isEmptyResult;
+  bool        isEmptyResult;
+  bool        hasAggFuncs;
+  bool        isTimeOrderQuery;
 } SSelectStmt;
 
-typedef enum ESetOperatorType {
+typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
-  SET_OP_TYPE_UNION_ALL = 1,
-  SET_OP_TYPE_UNION
-} ESetOperatorType;
 
 typedef struct SSetOperator {
-  ENodeType type; // QUERY_NODE_SET_OPERATOR
+  ENodeType        type;  // QUERY_NODE_SET_OPERATOR
   ESetOperatorType opType;
-  SNodeList* pProjectionList;
+  SNodeList*       pProjectionList;
-  SNode* pLeft;
+  SNode*           pLeft;
-  SNode* pRight;
+  SNode*           pRight;
-  SNodeList* pOrderByList; // SOrderByExprNode
+  SNodeList*       pOrderByList;  // SOrderByExprNode
-  SNode* pLimit;
+  SNode*           pLimit;
 } SSetOperator;
 
 typedef enum ESqlClause {
@@ -271,7 +260,6 @@ typedef enum ESqlClause {
   SQL_CLAUSE_ORDER_BY
 } ESqlClause;
 
-
 typedef enum {
   PAYLOAD_TYPE_KV = 0,
   PAYLOAD_TYPE_RAW = 1,
@@ -281,38 +269,42 @@ typedef struct SVgDataBlocks {
   SVgroupInfo vg;
   int32_t     numOfTables;  // number of tables in current submit block
   uint32_t    size;
-  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
+  char*       pData;  // SMsgDesc + SSubmitReq + SSubmitBlk + ...
 } SVgDataBlocks;
 
 typedef struct SVnodeModifOpStmt {
-  ENodeType    nodeType;
+  ENodeType   nodeType;
-  ENodeType    sqlNodeType;
+  ENodeType   sqlNodeType;
-  SArray*      pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  SArray*     pDataBlocks;  // data block for each vgroup, SArray<SVgDataBlocks*>.
-  uint8_t      payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint8_t     payloadType;  // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t     insertType;          // insert data from [file|sql statement| bound statement]
+  uint32_t    insertType;   // insert data from [file|sql statement| bound statement]
-  const char*  sql;                 // current sql statement position
+  const char* sql;          // current sql statement position
 } SVnodeModifOpStmt;
 
 typedef struct SExplainOptions {
   ENodeType type;
-  bool verbose;
+  bool      verbose;
-  double ratio;
+  double    ratio;
 } SExplainOptions;
 
 typedef struct SExplainStmt {
-  ENodeType type;
+  ENodeType        type;
-  bool analyze;
+  bool             analyze;
   SExplainOptions* pOptions;
-  SNode* pQuery;
+  SNode*           pQuery;
 } SExplainStmt;
 
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
+typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
+                            SNodeList** pCols);
 
 typedef bool (*FFuncClassifier)(int32_t funcId);
-int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);
+int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs);
+
+int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes);
 
 bool nodesIsExprNode(const SNode* pNode);
 
@@ -324,10 +316,11 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
-void* nodesGetValueFromNode(SValueNode *pNode);
+void*   nodesGetValueFromNode(SValueNode* pNode);
-char* nodesGetStrValueFromNode(SValueNode *pNode);
+int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value);
-char *getFillModeString(EFillMode mode);
+char*   nodesGetStrValueFromNode(SValueNode* pNode);
-void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
+char*   getFillModeString(EFillMode mode);
+void    valueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
 
 #ifdef __cplusplus
 }

include/libs/parser/parser.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 
 typedef struct SStmtCallback {
   TAOS_STMT* pStmt;
@@ -34,24 +34,26 @@ typedef struct SStmtCallback {
 typedef struct SParseContext {
   uint64_t         requestId;
   int32_t          acctId;
-  const char      *db;
+  const char*      db;
   bool             topicQuery;
-  void            *pTransporter;
+  void*            pTransporter;
   SEpSet           mgmtEpSet;
-  const char      *pSql;           // sql string
+  const char*      pSql;    // sql string
-  size_t           sqlLen;         // length of the sql string
+  size_t           sqlLen;  // length of the sql string
-  char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
+  char*            pMsg;    // extended error message if exists to help identifying the problem in sql statement.
-  int32_t          msgLen;         // max length of the msg
+  int32_t          msgLen;  // max length of the msg
-  struct SCatalog *pCatalog;
+  struct SCatalog* pCatalog;
-  SStmtCallback   *pStmtCb;
+  SStmtCallback*   pStmtCb;
+  const char*      pUser;
+  bool             isSuperUser;
 } SParseContext;
 
 typedef struct SCmdMsgInfo {
   int16_t msgType;
-  SEpSet epSet;
+  SEpSet  epSet;
-  void* pMsg;
+  void*   pMsg;
   int32_t msgLen;
-  void* pExtension;  // todo remove it soon
+  void*   pExtension;  // todo remove it soon
 } SCmdMsgInfo;
 
 typedef enum EQueryExecMode {
@@ -63,20 +65,21 @@ typedef enum EQueryExecMode {
 
 typedef struct SQuery {
   EQueryExecMode execMode;
-  bool haveResultSet;
+  bool           haveResultSet;
-  SNode* pRoot;
+  SNode*         pRoot;
-  int32_t numOfResCols;
+  int32_t        numOfResCols;
-  SSchema* pResSchema;
+  SSchema*       pResSchema;
-  int8_t   precision;
+  int8_t         precision;
-  SCmdMsgInfo* pCmdMsg;
+  SCmdMsgInfo*   pCmdMsg;
-  int32_t msgType;
+  int32_t        msgType;
-  SArray* pDbList;
+  SArray*        pDbList;
-  SArray* pTableList;
+  SArray*        pTableList;
-  bool showRewrite;
+  bool           showRewrite;
+  int32_t        placeholderNum;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
-bool isInsertSql(const char* pStr, size_t length);
+bool    isInsertSql(const char* pStr, size_t length);
 
 void qDestroyQuery(SQuery* pQueryNode);
 
@@ -88,13 +91,17 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
 void    qFreeStmtDataBlock(void* pDataBlock);
 int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
 void    qDestroyStmtDataBlock(void* pBlock);
-int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+
-int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
+int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
-int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
-int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
+                                int32_t rowNum);
-int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind,
+                           char* msgBuf, int32_t msgBufLen);
 void    destroyBoundColumnInfo(void* pBoundInfo);
-int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
+                     int32_t msgBufLen);
 
 void*   smlInitHandle(SQuery *pQuery);
 void    smlDestroyHandle(void *pHandle);

include/libs/planner/planner.h

@@ -34,7 +34,11 @@ typedef struct SPlanContext {
   bool showRewrite;
   int8_t triggerType;
   int64_t watermark;
-  bool isStmtQuery;
+  int32_t placeholderNum;
+  void* pTransporter;
+  struct SCatalog* pCatalog;
+  char* pMsg;
+  int32_t msgLen;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.
@@ -46,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams);
+int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
 
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

include/libs/qcom/query.h

@@ -51,6 +51,10 @@ typedef struct STableComInfo {
 
 typedef struct SIndexMeta {
 
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
+
 } SIndexMeta;
 
 
@@ -129,12 +133,6 @@ typedef struct SMsgSendInfo {
   SDataBuf             msgInfo;
 } SMsgSendInfo;
 
-typedef struct SQueryNodeAddr {
-  int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epSet;
-} SQueryNodeAddr;
-
-
 typedef struct SQueryNodeStat {
   int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;

include/libs/stream/tstream.h

@@ -35,19 +35,6 @@ enum {
   STREAM_CREATED_BY__SMA,
 };
 
-#if 0
-// pipe  -> fetch/pipe queue
-// merge -> merge      queue
-// write -> write      queue
-enum {
-  TASK_DISPATCH_MSG__SND_PIPE = 1,
-  TASK_DISPATCH_MSG__SND_MERGE,
-  TASK_DISPATCH_MSG__VND_PIPE,
-  TASK_DISPATCH_MSG__VND_MERGE,
-  TASK_DISPATCH_MSG__VND_WRITE,
-};
-#endif
-
 typedef struct {
   int32_t nodeId;  // 0 for snode
   SEpSet  epSet;
@@ -83,8 +70,10 @@ typedef struct {
 } STaskDispatcherShuffle;
 
 typedef struct {
-  int8_t reserved;
+  int8_t          reserved;
+  SSchemaWrapper* pSchemaWrapper;
   // not applicable to encoder and decoder
+  STSchema* pTSchema;
   SHashObj* pHash;  // groupId to tbuid
 } STaskSinkTb;
 
@@ -100,10 +89,6 @@ typedef struct {
   int8_t reserved;
 } STaskSinkFetch;
 
-typedef struct {
-  int8_t reserved;
-} STaskSinkShow;
-
 enum {
   TASK_SOURCE__SCAN = 1,
   TASK_SOURCE__PIPE,
@@ -128,7 +113,6 @@ enum {
   TASK_SINK__TABLE,
   TASK_SINK__SMA,
   TASK_SINK__FETCH,
-  TASK_SINK__SHOW,
 };
 
 typedef struct {
@@ -155,7 +139,6 @@ typedef struct {
     STaskSinkTb    tbSink;
     STaskSinkSma   smaSink;
     STaskSinkFetch fetchSink;
-    STaskSinkShow  showSink;
   };
 
   // dispatch

include/libs/stream/tstreamUpdate.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TSTREAMUPDATE_H_
+#define _TSTREAMUPDATE_H_
+
+#include "taosdef.h"
+#include "tarray.h"
+#include "tmsg.h"
+#include "tscalablebf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SUpdateInfo {
+  SArray *pTsBuckets;
+  uint64_t numBuckets;
+  SArray *pTsSBFs;
+  uint64_t numSBFs;
+  int64_t interval;
+  int64_t watermark;
+  TSKEY minTS;
+} SUpdateInfo;
+
+SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
+SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
+bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
+void updateInfoDestroy(SUpdateInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef _TSTREAMUPDATE_H_ */

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
+//#include <tdatablock.h>
-#include "taosdef.h"
+#include "cJSON.h"
-#include "trpc.h"
+#include "tdef.h"
-#include "wal.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
+  uint16_t nodePort;
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
+
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
+
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
+
 typedef struct SSyncFSM {
   void* data;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-                     ESyncState state);
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
-
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
-
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
-
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN 0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
+struct SWal;
-typedef struct SStateMgr {
+typedef struct SWal SWal;
-  void* data;
 
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+struct SEpSet;
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+typedef struct SEpSet SEpSet;
-
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,27 +140,39 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
-struct SSyncNode;
+int32_t     syncInit();
-typedef struct SSyncNode SSyncNode;
+void        syncCleanUp();
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
+void        syncStop(int64_t rid);
+int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
+ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
+void        syncGetEpSet(int64_t rid, SEpSet* pEpSet);
+int32_t     syncGetVgId(int64_t rid);
 
-int32_t syncInit();
+typedef enum {
-void    syncCleanUp();
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
 
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-void       syncStop(int64_t rid);
-int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-ESyncState syncGetMyRole(int64_t rid);
 
-// propose with sequence number, to implement linearizable semantics
+bool syncEnvIsStart();
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
-
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
 extern int32_t sDebugFlag;
 
-const char *syncStr(ESyncState state);
+//-----------------------------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+struct SSyncBuffer;
+typedef struct SSyncBuffer SSyncBuffer;
+//-----------------------------------------
+
+const char* syncStr(ESyncState state);
 
 #ifdef __cplusplus
 }

include/libs/sync/syncTools.h

@@ -0,0 +1,414 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -43,6 +43,7 @@ typedef struct SRpcMsg {
   int32_t code;
   void *  handle;         // rpc handle returned to app
   void *  ahandle;        // app handle set by client
+  int64_t refId;          // refid, used by server
   int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int     persistHandle;  // persist handle or not
 
@@ -58,9 +59,13 @@ typedef struct {
   void *   pNode;
 } SNodeMsg;
 
-typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
+typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
 typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
-typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
+///
+// // SRpcMsg code
+// REDIERE,
+// NOT READY, EpSet
+typedef bool (*RpcRfp)(int32_t code);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port

include/libs/wal/wal.h

@@ -72,8 +72,6 @@ extern "C" {
 #define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
 #define WAL_MAGIC        0xFAFBFCFDULL
 
-#define WAL_CUR_FAILED 1
-
 #pragma pack(push, 1)
 typedef enum {
   TAOS_WAL_NOLOG = 0,
@@ -92,7 +90,7 @@ typedef struct SWalReadHead {
   int8_t  headVer;
   int8_t  reserved;
   int16_t msgType;
-  int32_t len;
+  int32_t bodyLen;
   int64_t ingestTs;  // not implemented
   int64_t version;
 
@@ -192,7 +190,13 @@ int32_t walEndSnapshot(SWal *);
 SWalReadHandle *walOpenReadHandle(SWal *);
 void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
-int32_t         walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
+
+// only for tq usage
+// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
+void    walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
+int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
+int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
+int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
 
 // deprecated
 #if 0

include/os/os.h

@@ -22,7 +22,6 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
-#include <semaphore.h>
 
 #include <regex.h>
 
@@ -51,6 +50,16 @@ extern "C" {
 #include <sys/prctl.h>
 #include <argp.h>
 #endif
+#else
+
+#include <time.h>
+#ifndef TD_USE_WINSOCK
+#include <winsock2.h>
+#else
+#include <winsock.h>
+#endif
+
+#define __typeof(a) auto
 
 #endif
 

include/os/osAtomic.h

@@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, int32_t val);
+void* atomic_add_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, int32_t val);
+void* atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
+void* atomic_sub_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
+void* atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);

include/os/osDef.h

@@ -62,7 +62,7 @@ extern "C" {
   #define strncasecmp _strnicmp
   #define wcsncasecmp _wcsnicmp
   #define strtok_r strtok_s
-  #define snprintf _snprintf
+  // #define snprintf _snprintf
   #define in_addr_t unsigned long
 //  #define socklen_t int
 
@@ -191,7 +191,7 @@ extern "C" {
   #define threadlocal __declspec( thread )
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define PRIzu "ld"  
 #else
   #define PRIzu "zu"  

include/os/osDir.h

@@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr;
 
 
 void    taosRemoveDir(const char *dirname);
-bool    taosDirExist(char *dirname);
+bool    taosDirExist(const char *dirname);
 int32_t taosMkDir(const char *dirname);
 int32_t taosMulMkDir(const char *dirname);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
-int32_t taosRealPath(char *dirname, int32_t maxlen);
+int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
 char*   taosDirName(char *dirname);
 char*   taosDirEntryBaseName(char *dirname);
@@ -50,7 +50,7 @@ TdDirPtr      taosOpenDir(const char *dirname);
 TdDirEntryPtr taosReadDir(TdDirPtr pDir);
 bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
 char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
-int32_t       taosCloseDir(TdDirPtr pDir);
+int32_t       taosCloseDir(TdDirPtr *ppDir);
 
 #ifdef __cplusplus
 }

include/os/osFile.h

@@ -93,7 +93,6 @@ void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix,
 
 int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
 
-void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
 bool taosValidFile(TdFilePtr pFile);
 
 int32_t taosGetErrorFile(TdFilePtr pFile);

include/os/osLz4.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   int32_t BUILDIN_CLZL(uint64_t val);
   int32_t BUILDIN_CLZ(uint32_t val);
   int32_t BUILDIN_CTZL(uint64_t val);

include/os/osMath.h

@@ -23,26 +23,21 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#define TSWAP(a, b)             \
+  do {                          \
+    __typeof(a) __tmp = (a); \
+    (a) = (b);               \
+    (b) = __tmp;                \
+  } while (0)
+
+#ifdef WINDOWS
 
-  #define TSWAP(a, b, c) \
-    do {                \
-      c __tmp = (c)(a); \
-      (a) = (c)(b);     \
-      (b) = __tmp;      \
-    } while (0)
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
+  #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 
-  #define TSWAP(a, b, c)       \
-    do {                       \
-      __typeof(a) __tmp = (a); \
-      (a) = (b);               \
-      (b) = __tmp;             \
-    } while (0)
-
   #define TMAX(a, b)             \
     ({                           \
       __typeof(a) __a = (a);     \
@@ -50,12 +45,12 @@ extern "C" {
       (__a > __b) ? __a : __b;   \
     })
 
-  #define TMIN(a, b)             \
+#define TMIN(a, b)             \
-    ({                           \
+  ({                           \
-      __typeof(a) __a = (a);     \
+    __typeof(a) __a = (a);     \
-      __typeof(b) __b = (b);     \
+    __typeof(b) __b = (b);     \
-      (__a < __b) ? __a : __b;   \
+    (__a < __b) ? __a : __b;   \
-    })
+  })
 
 #define TRANGE(a, b, c) \
   ({                    \

include/os/osSemaphore.h

@@ -22,21 +22,28 @@ extern "C" {
 
 #include <semaphore.h>
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
-  typedef struct tsem_s *tsem_t;
+
-  int tsem_init(tsem_t *sem, int pshared, unsigned int value);
+typedef struct tsem_s *tsem_t;
-  int tsem_wait(tsem_t *sem);
+
-  int tsem_post(tsem_t *sem);
+int tsem_init(tsem_t *sem, int pshared, unsigned int value);
-  int tsem_destroy(tsem_t *sem);
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+int tsem_post(tsem_t *sem);
+int tsem_destroy(tsem_t *sem);
+
 #else
-  #define tsem_t sem_t
+
-  #define tsem_init sem_init
+#define tsem_t       sem_t
-  int tsem_wait(tsem_t* sem);
+#define tsem_init    sem_init
-  #define tsem_post sem_post
+int tsem_wait(tsem_t *sem);
-  #define tsem_destroy sem_destroy
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+#define tsem_post    sem_post
+#define tsem_destroy sem_destroy
+
 #endif
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
 //  #define TdThreadRwlock TdThreadMutex
 //  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
 //  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
@@ -44,20 +51,20 @@ extern "C" {
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define TdThreadSpinlock TdThreadMutex
+#define TdThreadSpinlock               TdThreadMutex
-  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
+#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
-  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
+#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
-  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
+#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
 int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(TdThread* thread);
+void    taosResetPthread(TdThread *thread);
 bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
-int32_t taosGetAppName(char* name, int32_t* len);
+int32_t taosGetAppName(char *name, int32_t *len);
 
 #ifdef __cplusplus
 }

include/os/osSocket.h

@@ -19,32 +19,53 @@
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-    #define socket SOCKET_FUNC_TAOS_FORBID
+#define socket       SOCKET_FUNC_TAOS_FORBID
-    #define bind   BIND_FUNC_TAOS_FORBID
+#define bind         BIND_FUNC_TAOS_FORBID
-    #define listen LISTEN_FUNC_TAOS_FORBID
+#define listen       LISTEN_FUNC_TAOS_FORBID
-    #define accept ACCEPT_FUNC_TAOS_FORBID
+#define accept       ACCEPT_FUNC_TAOS_FORBID
-    #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
-    #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
+#define epoll_ctl    EPOLL_CTL_FUNC_TAOS_FORBID
-    #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
+#define epoll_wait   EPOLL_WAIT_FUNC_TAOS_FORBID
-    #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
+#define inet_addr    INET_ADDR_FUNC_TAOS_FORBID
-    #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
+#define inet_ntoa    INET_NTOA_FUNC_TAOS_FORBID
 #endif
 
 #if defined(WINDOWS)
-  #include "winsock2.h"
+#if BYTE_ORDER == LITTLE_ENDIAN
-  #include <WS2tcpip.h>
+#include <stdlib.h>
-  #include <winbase.h>
+#define htobe16(x) _byteswap_ushort(x)
-  #include <Winsock2.h>
+#define htole16(x) (x)
-#else
+#define be16toh(x) _byteswap_ushort(x)
-    #include <netinet/in.h>
+#define le16toh(x) (x)
-    #include <sys/socket.h>
 
-    #if defined(_TD_DARWIN_64)
+#define htobe32(x) _byteswap_ulong(x)
-        #include <osEok.h>
+#define htole32(x) (x)
-    #else
+#define be32toh(x) _byteswap_ulong(x)
-        #include <netinet/in.h>
+#define le32toh(x) (x)
-        #include <sys/epoll.h>
+
-    #endif
+#define htobe64(x) _byteswap_uint64(x)
+#define htole64(x) (x)
+#define be64toh(x) _byteswap_uint64(x)
+#define le64toh(x) (x)
+#else
+#error byte order not supported
+#endif
+
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
+
+#else
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+#if defined(_TD_DARWIN_64)
+#include <osEok.h>
+#else
+#include <netinet/in.h>
+#include <sys/epoll.h>
+#endif
 #endif
 
 #ifdef __cplusplus
@@ -52,60 +73,54 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  #define htobe64 htonll
+typedef int socklen_t;
+#define TAOS_EPOLL_WAIT_TIME 100
+typedef SOCKET eventfd_t;
+#define eventfd(a, b)      -1
+#define EpollClose(pollFd) epoll_close(pollFd)
+#ifndef EPOLLWAKEUP
+#define EPOLLWAKEUP (1u << 29)
 #endif
-
-#if defined(WINDOWS)
-  #define TAOS_EPOLL_WAIT_TIME 100
-  typedef SOCKET eventfd_t;
-  #define eventfd(a, b) -1
-  typedef SOCKET EpollFd;
-  #define EpollClose(pollFd) epoll_close(pollFd)
-  #ifndef EPOLLWAKEUP
-    #define EPOLLWAKEUP (1u << 29)
-  #endif
 #elif defined(_TD_DARWIN_64)
-  #define TAOS_EPOLL_WAIT_TIME 500
+#define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
+typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
+typedef SOCKET  EpollFd;
-  #define EpollClose(pollFd) epoll_close(pollFd)
+#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
-  #define TAOS_EPOLL_WAIT_TIME 500
+#define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
+typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
+typedef SOCKET  EpollFd;
-  #define EpollClose(pollFd) taosCloseSocket(pollFd)
+#define EpollClose(pollFd)   taosCloseSocket(pollFd)
 #endif
 
 #if defined(_TD_DARWIN_64)
 //  #define htobe64 htonll
 
-#	include <libkern/OSByteOrder.h>
+#include <libkern/OSByteOrder.h>
 
-#	define htobe16(x) OSSwapHostToBigInt16(x)
+#define htobe16(x) OSSwapHostToBigInt16(x)
-#	define htole16(x) OSSwapHostToLittleInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
-#	define be16toh(x) OSSwapBigToHostInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
-#	define le16toh(x) OSSwapLittleToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
 
-#	define htobe32(x) OSSwapHostToBigInt32(x)
+#define htobe32(x) OSSwapHostToBigInt32(x)
-#	define htole32(x) OSSwapHostToLittleInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
-#	define be32toh(x) OSSwapBigToHostInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
-#	define le32toh(x) OSSwapLittleToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
 
-#	define htobe64(x) OSSwapHostToBigInt64(x)
+#define htobe64(x) OSSwapHostToBigInt64(x)
-#	define htole64(x) OSSwapHostToLittleInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
-#	define be64toh(x) OSSwapBigToHostInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
-#	define le64toh(x) OSSwapLittleToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
 
-#	define __BYTE_ORDER    BYTE_ORDER
+#define __BYTE_ORDER    BYTE_ORDER
-#	define __BIG_ENDIAN    BIG_ENDIAN
+#define __BIG_ENDIAN    BIG_ENDIAN
-#	define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
-#	define __PDP_ENDIAN    PDP_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
+typedef int32_t  SocketFd;
-
+typedef SocketFd EpollFd;
-typedef int32_t SocketFd;
-typedef SocketFd  EpollFd;
 
 typedef struct TdSocket {
 #if SOCKET_WITH_LOCK
@@ -113,16 +128,17 @@ typedef struct TdSocket {
 #endif
   int      refId;
   SocketFd fd;
-} *TdSocketPtr, TdSocket;
+} * TdSocketPtr, TdSocket;
 
 typedef struct TdSocketServer *TdSocketServerPtr;
-typedef struct TdSocket *TdSocketPtr;
+typedef struct TdSocket *      TdSocketPtr;
-typedef struct TdEpoll *TdEpollPtr;
+typedef struct TdEpoll *       TdEpollPtr;
 
-int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr,
+                           int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -139,29 +155,32 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+void    taosWinSocketInit();
 
-TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
-TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+
+TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
+int32_t           taosKeepTcpAlive(TdSocketPtr pSocket);
-TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+TdSocketPtr       taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
 
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
 
-void     taosBlockSIGPIPE();
+void        taosBlockSIGPIPE();
-uint32_t taosGetIpv4FromFqdn(const char *);
+uint32_t    taosGetIpv4FromFqdn(const char *);
-int32_t  taosGetFqdn(char *);
+int32_t     taosGetFqdn(char *);
-void     tinet_ntoa(char *ipstr, uint32_t ip);
+void        tinet_ntoa(char *ipstr, uint32_t ip);
-uint32_t ip2uint(const char *const ip_addr);
+uint32_t    ip2uint(const char *const ip_addr);
-void     taosIgnSIGPIPE();
+void        taosIgnSIGPIPE();
-void     taosSetMaskSIGPIPE();
+void        taosSetMaskSIGPIPE();
-uint32_t taosInetAddr(const char *ipAddr);
+uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
 TdEpollPtr taosCreateEpoll(int32_t size);
-int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
+int32_t    taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
-int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
+int32_t    taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
-int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
+int32_t    taosCloseEpoll(TdEpollPtr *ppEpoll);
 
 #ifdef __cplusplus
 }

include/os/osString.h

@@ -39,7 +39,7 @@ typedef int32_t TdUcs4;
     #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define tstrdup(str) _strdup(str)
 #else
   #define tstrdup(str) strdup(str)
@@ -59,6 +59,8 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
+int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);
 int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);

include/os/osSystem.h

@@ -31,19 +31,19 @@ extern "C" {
 
 typedef struct TdCmd *TdCmdPtr;
 
-TdCmdPtr taosOpenCmd(const char *cmd);
+TdCmdPtr taosOpenCmd(const char* cmd);
-int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf);
+int64_t  taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf);
-int32_t taosEOFCmd(TdCmdPtr pCmd);
+int32_t  taosEOFCmd(TdCmdPtr pCmd);
-int64_t taosCloseCmd(TdCmdPtr *ppCmd);
+int64_t  taosCloseCmd(TdCmdPtr* ppCmd);
 
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
-void setTerminalMode();
+void    taosSetTerminalMode();
-int32_t getOldTerminalMode();
+int32_t taosGetOldTerminalMode();
-void resetTerminalMode();
+void    taosResetTerminalMode();
 
 #ifdef __cplusplus
 }

include/os/osThread.h

@@ -22,9 +22,12 @@
 extern "C" {
 #endif
 
+#ifndef WINDOWS
 #ifndef __USE_XOPEN2K
+#define TD_USE_SPINLOCK_AS_MUTEX
 typedef pthread_mutex_t pthread_spinlock_t;
 #endif
+#endif
 
 typedef pthread_t TdThread;
 typedef pthread_spinlock_t TdThreadSpinlock;
@@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey;
 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop pthread_cleanup_pop
 
+
+#ifdef WINDOWS
+#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
+#else
+#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#endif
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
 void taosThreadTestCancel(void);
-int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
+void taosThreadClear(TdThread *thread);
-int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -20,8 +20,6 @@
 extern "C" {
 #endif
 
-#include <time.h>
-
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -34,7 +32,7 @@ extern "C" {
     #define mktime MKTIME_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define CLOCK_REALTIME 	0
 
@@ -58,6 +56,8 @@ extern "C" {
 
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
+int32_t taosClockGetTime(int clock_id, struct timespec *pTS);
+
 //@return timestamp in second
 int32_t taosGetTimestampSec();
 
@@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() {
 //@return timestamp in nanosecond
 static FORCE_INLINE int64_t taosGetTimestampNs() {
   struct timespec systemTime = {0};
-  clock_gettime(CLOCK_REALTIME, &systemTime);
+  taosClockGetTime(CLOCK_REALTIME, &systemTime);
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 

include/util/taoserror.h

@@ -62,6 +62,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_APP_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0014)
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0015)
 #define TSDB_CODE_RPC_INVALID_VERSION           TAOS_DEF_ERROR_CODE(0, 0x0016)
+#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017)
 
 //common & util
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0100)
@@ -86,6 +87,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0113)
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x0114)
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x0115)
+#define TSDB_CODE_DUP_KEY                       TAOS_DEF_ERROR_CODE(0, 0x0116)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0140)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0141)
@@ -139,16 +141,9 @@ int32_t* taosGetErrno();
 // mnode-common
 #define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
 #define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0301)
-#define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0302)
+#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0302)
-#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0303)
+#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0303)
-#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED   TAOS_DEF_ERROR_CODE(0, 0x0304)
+#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0304)
-#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0305)
-#define TSDB_CODE_MND_INVALID_OPTIONS           TAOS_DEF_ERROR_CODE(0, 0x0306)
-#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0307)
-#define TSDB_CODE_MND_INVALID_MSG_VERSION       TAOS_DEF_ERROR_CODE(0, 0x0308)
-#define TSDB_CODE_MND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0309)
-#define TSDB_CODE_MND_INVALID_MSG_TYPE          TAOS_DEF_ERROR_CODE(0, 0x030A)
-#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS      TAOS_DEF_ERROR_CODE(0, 0x030B)
 
 // mnode-show
 #define TSDB_CODE_MND_INVALID_SHOWOBJ           TAOS_DEF_ERROR_CODE(0, 0x0310)
@@ -250,9 +245,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
 #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
+#define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_INFOS_TBL         TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
@@ -268,21 +264,21 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TRANS_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03D0)
 #define TSDB_CODE_MND_TRANS_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03D1)
 #define TSDB_CODE_MND_TRANS_INVALID_STAGE       TAOS_DEF_ERROR_CODE(0, 0x03D2)
-#define TSDB_CODE_MND_TRANS_CANT_PARALLEL       TAOS_DEF_ERROR_CODE(0, 0x03D4)
+#define TSDB_CODE_MND_TRANS_CONFLICT            TAOS_DEF_ERROR_CODE(0, 0x03D3)
+#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR        TAOS_DEF_ERROR_CODE(0, 0x03D4)
 
 // mnode-mq
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E0)
 #define TSDB_CODE_MND_TOPIC_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03E1)
 #define TSDB_CODE_MND_TOO_MANY_TOPICS           TAOS_DEF_ERROR_CODE(0, 0x03E2)
 #define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x03E3)
-#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E4)
+#define TSDB_CODE_MND_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x03E4)
-#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E5)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E5)
-#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB    TAOS_DEF_ERROR_CODE(0, 0x03E6)
+#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E6)
-#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
+#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E7)
-#define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
+#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
-#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
-#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
-#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -324,6 +320,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_TB_NOT_EXIST              TAOS_DEF_ERROR_CODE(0, 0x0515)
 #define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
 #define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
+#define TSDB_CODE_VND_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0518)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
@@ -411,6 +408,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+// sync integration
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)
@@ -569,7 +570,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x2602)
 #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN          TAOS_DEF_ERROR_CODE(0, 0x2603)
 #define TSDB_CODE_PAR_WRONG_VALUE_TYPE          TAOS_DEF_ERROR_CODE(0, 0x2604)
-#define TSDB_CODE_PAR_INVALID_FUNTION           TAOS_DEF_ERROR_CODE(0, 0x2605)
 #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION  TAOS_DEF_ERROR_CODE(0, 0x2608)
 #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT    TAOS_DEF_ERROR_CODE(0, 0x2609)
 #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION   TAOS_DEF_ERROR_CODE(0, 0x260A)
@@ -590,7 +590,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_RANGE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x2619)
 #define TSDB_CODE_PAR_INVALID_STR_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261A)
 #define TSDB_CODE_PAR_INVALID_ENUM_OPTION       TAOS_DEF_ERROR_CODE(0, 0x261B)
-#define TSDB_CODE_PAR_INVALID_TTL_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261C)
 #define TSDB_CODE_PAR_INVALID_KEEP_NUM          TAOS_DEF_ERROR_CODE(0, 0x261D)
 #define TSDB_CODE_PAR_INVALID_KEEP_ORDER        TAOS_DEF_ERROR_CODE(0, 0x261E)
 #define TSDB_CODE_PAR_INVALID_KEEP_VALUE        TAOS_DEF_ERROR_CODE(0, 0x261F)
@@ -615,15 +614,25 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
 #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2633)
 #define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL      TAOS_DEF_ERROR_CODE(0, 0x2634)
+#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL   TAOS_DEF_ERROR_CODE(0, 0x2635)
+#define TSDB_CODE_PAR_INVALID_DAYS_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2636)
+#define TSDB_CODE_PAR_OFFSET_LESS_ZERO          TAOS_DEF_ERROR_CODE(0, 0x2637)
+#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY  TAOS_DEF_ERROR_CODE(0, 0x2638)
+#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x2639)
+#define TSDB_CODE_PAR_INVALID_DROP_STABLE       TAOS_DEF_ERROR_CODE(0, 0x263A)
+#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE   TAOS_DEF_ERROR_CODE(0, 0x263B)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
+#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL        TAOS_DEF_ERROR_CODE(0, 0x2701)
+#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN   TAOS_DEF_ERROR_CODE(0, 0x2702)
 
 //function
 #define TSDB_CODE_FUNC_FUNTION_ERROR            TAOS_DEF_ERROR_CODE(0, 0x2800)
 #define TSDB_CODE_FUNC_FUNTION_PARA_NUM         TAOS_DEF_ERROR_CODE(0, 0x2801)
 #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE        TAOS_DEF_ERROR_CODE(0, 0x2802)
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
+#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2604)
 
 #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
 #define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)

include/util/tarray.h

@@ -205,7 +205,6 @@ SArray* taosArrayDup(const SArray* pSrc);
  */
 SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
 
-
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -219,6 +218,13 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
+/**
+ * clear the array (remove all element)
+ * @param pArray
+ * @param fp
+ */
+void taosArrayClearP(SArray* pArray, FDelete fp);
+
 void* taosArrayDestroy(SArray* pArray);
 void  taosArrayDestroyP(SArray* pArray, FDelete fp);
 void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
@@ -272,6 +278,8 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
 int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
 void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
 
+char* taosShowStrArray(const SArray* pArray);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tbloomfilter.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_BLOOMFILTER_H_
+#define _TD_UTIL_BLOOMFILTER_H_
+
+#include "os.h"
+#include "thash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBloomFilter {
+  uint32_t hashFunctions;
+  uint64_t expectedEntries;
+  uint64_t numUnits;
+  uint64_t numBits;
+  uint64_t size;
+  _hash_fn_t hashFn1;
+  _hash_fn_t hashFn2;
+  void *buffer;
+  double errorRate;
+} SBloomFilter;
+
+SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
+int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
+int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
+                              uint32_t len);
+void tBloomFilterDestroy(SBloomFilter *pBF);
+void tBloomFilterDump(const SBloomFilter *pBF);
+bool tBloomFilterIsFull(const SBloomFilter *pBF);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_BLOOMFILTER_H_*/

include/util/tcache.h

@@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
 /**
  * initialize the cache object
  * @param keyType              key type
- * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
+ * @param refreshTimeInMs      refresh operation interval time, the maximum survival time when one element is expired
  *                             and not referenced by other objects
  * @param extendLifespan       auto extend lifespan, if accessed
  * @param fn                   free resource callback function
  * @return
  */
-SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
+SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName);
 
 /**
@@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
  * @param pCacheObj
  * @return
  */
-size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
+size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj);
 
 /**
  *  move all data node into trash, clear node in trash can if it is not referenced by any clients
@@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
  */
 void taosStopCacheRefreshWorker();
 
-SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
+SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj);
-bool taosCacheIterNext(SCacheIter* pIter);
+bool        taosCacheIterNext(SCacheIter *pIter);
-void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
+void       *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen);
-void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
+void       *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen);
-void taosCacheDestroyIter(SCacheIter* pIter);
+void        taosCacheDestroyIter(SCacheIter *pIter);
 
 #ifdef __cplusplus
 }

include/util/tconfig.h

@@ -30,6 +30,7 @@ typedef enum {
   CFG_STYPE_CFG_FILE,
   CFG_STYPE_ENV_FILE,
   CFG_STYPE_ENV_VAR,
+  CFG_STYPE_ENV_CMD,
   CFG_STYPE_APOLLO_URL,
   CFG_STYPE_ARG_LIST,
   CFG_STYPE_TAOS_OPTIONS
@@ -82,7 +83,7 @@ typedef struct SConfig {
 } SConfig;
 
 SConfig *cfgInit();
-int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
+int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
 int32_t  cfgLoadFromArray(SConfig *pCfg, SArray *pArgs);  // SConfigPair
 void     cfgCleanup(SConfig *pCfg);
 
@@ -105,6 +106,8 @@ const char *cfgDtypeStr(ECfgDataType type);
 
 void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
 
+int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DEFAULT_PASS "taosdata"
 #endif
 
-#define SHELL_MAX_PASSWORD_LEN 20
-
 #define TSDB_TRUE  1
 #define TSDB_FALSE 0
 #define TSDB_OK    0
@@ -103,42 +101,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MICRO_DIGITS 16
 #define TSDB_TIME_PRECISION_NANO_DIGITS  19
 
-#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
-#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
-#define TSDB_INS_TABLE_DNODES                 "dnodes"
-#define TSDB_INS_TABLE_MNODES                 "mnodes"
-#define TSDB_INS_TABLE_MODULES                "modules"
-#define TSDB_INS_TABLE_QNODES                 "qnodes"
-#define TSDB_INS_TABLE_BNODES                 "bnodes"
-#define TSDB_INS_TABLE_SNODES                 "snodes"
-#define TSDB_INS_TABLE_CLUSTER                "cluster"
-#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
-#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
-#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
-#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
-#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
-#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
-#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
-#define TSDB_INS_TABLE_USER_USERS             "user_users"
-#define TSDB_INS_TABLE_LICENCES               "grants"
-#define TSDB_INS_TABLE_VGROUPS                "vgroups"
-#define TSDB_INS_TABLE_TOPICS                 "topics"
-#define TSDB_INS_TABLE_CONSUMERS              "consumers"
-#define TSDB_INS_TABLE_SUBSCRIBES             "subscribes"
-#define TSDB_INS_TABLE_TRANS                  "trans"
-#define TSDB_INS_TABLE_SMAS                   "smas"
-#define TSDB_INS_TABLE_CONFIGS                "configs"
-#define TSDB_INS_TABLE_CONNS                  "connections"
-#define TSDB_INS_TABLE_QUERIES                "queries"
-#define TSDB_INS_TABLE_VNODES                 "vnodes"
-
-#define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
-#define TSDB_PERFS_TABLE_CONNECTIONS "connections"
-#define TSDB_PERFS_TABLE_QUERIES     "queries"
-#define TSDB_PERFS_TABLE_TOPICS      "topics"
-#define TSDB_PERFS_TABLE_CONSUMERS   "consumers"
-#define TSDB_PERFS_TABLE_SUBSCRIBES  "subscribes"
-
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
 
@@ -202,22 +164,14 @@ typedef enum EOperatorType {
   OP_TYPE_JSON_CONTAINS
 } EOperatorType;
 
+#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
+
 typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_AND = 1,
   LOGIC_COND_TYPE_OR,
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
-#define FUNCTION_CEIL  4500
-#define FUNCTION_FLOOR 4501
-#define FUNCTION_ABS   4502
-#define FUNCTION_ROUND 4503
-
-#define FUNCTION_LENGTH 4800
-#define FUNCTION_CONCAT 4801
-#define FUNCTION_LTRIM  4802
-#define FUNCTION_RTRIM  4803
-
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24
@@ -277,7 +231,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
-#define TSDB_MAX_JSON_TAG_LEN   16384
+#define TSDB_MAX_JSON_TAG_LEN 16384
 
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
@@ -291,8 +245,9 @@ typedef enum ELogicConditionType {
 #define TSDB_IPv4ADDR_LEN       16
 #define TSDB_FILENAME_LEN       128
 #define TSDB_SHOW_SQL_LEN       512
-#define TSDB_SHOW_SUBQUERY_LEN  1000
 #define TSDB_SLOW_QUERY_SQL_LEN 512
+#define TSDB_SHOW_SUBQUERY_LEN  1000
+#define TSDB_SHOW_LIST_LEN      1000
 
 #define TSDB_TRANS_STAGE_LEN 12
 #define TSDB_TRANS_TYPE_LEN  16
@@ -326,72 +281,73 @@ typedef enum ELogicConditionType {
 
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta
 
-#define TSDB_MIN_VNODES_PER_DB        1
+#define TSDB_MIN_VNODES_PER_DB          1
-#define TSDB_MAX_VNODES_PER_DB        4096
+#define TSDB_MAX_VNODES_PER_DB          4096
-#define TSDB_DEFAULT_VN_PER_DB        2
+#define TSDB_DEFAULT_VN_PER_DB          2
-#define TSDB_MIN_CACHE_BLOCK_SIZE     1
+#define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
-#define TSDB_MAX_CACHE_BLOCK_SIZE     128  // 128MB for each vnode
+#define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB
-#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
+#define TSDB_DEFAULT_BUFFER_PER_VNODE   96
-#define TSDB_MIN_TOTAL_BLOCKS         3
+#define TSDB_MIN_PAGES_PER_VNODE        64
-#define TSDB_MAX_TOTAL_BLOCKS         10000
+#define TSDB_MAX_PAGES_PER_VNODE        16384
-#define TSDB_DEFAULT_TOTAL_BLOCKS     6
+#define TSDB_DEFAULT_PAGES_PER_VNODE    256
-#define TSDB_MIN_DAYS_PER_FILE        60  // unit minute
+#define TSDB_MIN_PAGESIZE_PER_VNODE     1  // unit KB
-#define TSDB_MAX_DAYS_PER_FILE        (3650 * 1440)
+#define TSDB_MAX_PAGESIZE_PER_VNODE     16384
-#define TSDB_DEFAULT_DAYS_PER_FILE    (10 * 1440)
+#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
-#define TSDB_MIN_KEEP                 (1 * 1440)       // data in db to be reserved. unit minute
+#define TSDB_MIN_DAYS_PER_FILE          60  // unit minute
-#define TSDB_MAX_KEEP                 (365000 * 1440)  // data in db to be reserved.
+#define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
-#define TSDB_DEFAULT_KEEP             (3650 * 1440)    // ten years
+#define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
-#define TSDB_MIN_MINROWS_FBLOCK       10
+#define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
-#define TSDB_MAX_MINROWS_FBLOCK       1000
+#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
-#define TSDB_DEFAULT_MINROWS_FBLOCK   100
+#define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
-#define TSDB_MIN_MAXROWS_FBLOCK       200
+#define TSDB_MIN_KEEP                   (1 * 1440)       // data in db to be reserved. unit minute
-#define TSDB_MAX_MAXROWS_FBLOCK       10000
+#define TSDB_MAX_KEEP                   (365000 * 1440)  // data in db to be reserved.
-#define TSDB_DEFAULT_MAXROWS_FBLOCK   4096
+#define TSDB_DEFAULT_KEEP               (3650 * 1440)    // ten years
-#define TSDB_MIN_COMMIT_TIME          30
+#define TSDB_MIN_MINROWS_FBLOCK         10
-#define TSDB_MAX_COMMIT_TIME          40960
+#define TSDB_MAX_MINROWS_FBLOCK         1000
-#define TSDB_DEFAULT_COMMIT_TIME      3600
+#define TSDB_DEFAULT_MINROWS_FBLOCK     100
-#define TSDB_MIN_FSYNC_PERIOD         0
+#define TSDB_MIN_MAXROWS_FBLOCK         200
-#define TSDB_MAX_FSYNC_PERIOD         180000  // millisecond
+#define TSDB_MAX_MAXROWS_FBLOCK         10000
-#define TSDB_DEFAULT_FSYNC_PERIOD     3000    // three second
+#define TSDB_DEFAULT_MAXROWS_FBLOCK     4096
-#define TSDB_MIN_DB_TTL               1
+#define TSDB_MIN_FSYNC_PERIOD           0
-#define TSDB_DEFAULT_DB_TTL           1
+#define TSDB_MAX_FSYNC_PERIOD           180000  // millisecond
-#define TSDB_MIN_WAL_LEVEL            1
+#define TSDB_DEFAULT_FSYNC_PERIOD       3000    // three second
-#define TSDB_MAX_WAL_LEVEL            2
+#define TSDB_MIN_WAL_LEVEL              1
-#define TSDB_DEFAULT_WAL_LEVEL        1
+#define TSDB_MAX_WAL_LEVEL              2
-#define TSDB_MIN_PRECISION            TSDB_TIME_PRECISION_MILLI
+#define TSDB_DEFAULT_WAL_LEVEL          1
-#define TSDB_MAX_PRECISION            TSDB_TIME_PRECISION_NANO
+#define TSDB_MIN_PRECISION              TSDB_TIME_PRECISION_MILLI
-#define TSDB_DEFAULT_PRECISION        TSDB_TIME_PRECISION_MILLI
+#define TSDB_MAX_PRECISION              TSDB_TIME_PRECISION_NANO
-#define TSDB_MIN_COMP_LEVEL           0
+#define TSDB_DEFAULT_PRECISION          TSDB_TIME_PRECISION_MILLI
-#define TSDB_MAX_COMP_LEVEL           2
+#define TSDB_MIN_COMP_LEVEL             0
-#define TSDB_DEFAULT_COMP_LEVEL       2
+#define TSDB_MAX_COMP_LEVEL             2
-#define TSDB_MIN_DB_REPLICA           1
+#define TSDB_DEFAULT_COMP_LEVEL         2
-#define TSDB_MAX_DB_REPLICA           3
+#define TSDB_MIN_DB_REPLICA             1
-#define TSDB_DEFAULT_DB_REPLICA       1
+#define TSDB_MAX_DB_REPLICA             3
-#define TSDB_DB_STRICT_OFF            0
+#define TSDB_DEFAULT_DB_REPLICA         1
-#define TSDB_DB_STRICT_ON             1
+#define TSDB_DB_STRICT_OFF              0
-#define TSDB_DEFAULT_DB_STRICT        0
+#define TSDB_DB_STRICT_ON               1
-#define TSDB_MIN_DB_UPDATE            0
+#define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MAX_DB_UPDATE            2
+#define TSDB_MIN_DB_CACHE_LAST_ROW      0
-#define TSDB_DEFAULT_DB_UPDATE        0
+#define TSDB_MAX_DB_CACHE_LAST_ROW      3
-#define TSDB_MIN_DB_CACHE_LAST_ROW    0
+#define TSDB_DEFAULT_CACHE_LAST_ROW     0
-#define TSDB_MAX_DB_CACHE_LAST_ROW    3
+#define TSDB_DB_STREAM_MODE_OFF         0
-#define TSDB_DEFAULT_CACHE_LAST_ROW   0
+#define TSDB_DB_STREAM_MODE_ON          1
-#define TSDB_DB_STREAM_MODE_OFF       0
+#define TSDB_DEFAULT_DB_STREAM_MODE     0
-#define TSDB_DB_STREAM_MODE_ON        1
+#define TSDB_DB_SINGLE_STABLE_ON        0
-#define TSDB_DEFAULT_DB_STREAM_MODE   0
+#define TSDB_DB_SINGLE_STABLE_OFF       1
-#define TSDB_DB_SINGLE_STABLE_ON     0
+#define TSDB_DEFAULT_DB_SINGLE_STABLE   0
-#define TSDB_DB_SINGLE_STABLE_OFF     1
-#define TSDB_DEFAULT_DB_SINGLE_STABLE 0
 
-#define TSDB_MIN_DB_FILE_FACTOR     0
+#define TSDB_MIN_ROLLUP_FILE_FACTOR     0
-#define TSDB_MAX_DB_FILE_FACTOR     1
+#define TSDB_MAX_ROLLUP_FILE_FACTOR     1
-#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1
+#define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1
-#define TSDB_MIN_DB_DELAY           1
+#define TSDB_MIN_ROLLUP_DELAY           1
-#define TSDB_MAX_DB_DELAY           10
+#define TSDB_MAX_ROLLUP_DELAY           10
-#define TSDB_DEFAULT_DB_DELAY       2
+#define TSDB_DEFAULT_ROLLUP_DELAY       2
-#define TSDB_MIN_EXPLAIN_RATIO      0
+#define TSDB_MIN_TABLE_TTL              0
-#define TSDB_MAX_EXPLAIN_RATIO      1
+#define TSDB_DEFAULT_TABLE_TTL          0
-#define TSDB_DEFAULT_EXPLAIN_RATIO  0.001
+
+#define TSDB_MIN_EXPLAIN_RATIO     0
+#define TSDB_MAX_EXPLAIN_RATIO     1
+#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
 
 #define TSDB_MAX_JOIN_TABLE_NUM 10
 #define TSDB_MAX_UNION_CLAUSE   5
@@ -418,21 +374,9 @@ typedef enum ELogicConditionType {
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag
  */
-#define TSDB_QUERY_TYPE_SUBQUERY        0x02u
-#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u  // two-stage subquery for super table
-
-#define TSDB_QUERY_TYPE_TABLE_QUERY      0x08u  // query ordinary table; below only apply to client side
-#define TSDB_QUERY_TYPE_STABLE_QUERY     0x10u  // query on super table
-#define TSDB_QUERY_TYPE_JOIN_QUERY       0x20u  // join query
-#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u  // select *,columns... query
-#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE   0x80u  // join sub query at the second stage
-
-#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
 #define TSDB_QUERY_TYPE_INSERT           0x100u  // insert type
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
 #define TSDB_QUERY_TYPE_FILE_INSERT      0x400u   // insert data from file
 #define TSDB_QUERY_TYPE_STMT_INSERT      0x800u   // stmt insert type
-#define TSDB_QUERY_TYPE_NEST_SUBQUERY    0x1000u  // nested sub query
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)   (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)   ((x) |= (_type))

include/util/tencode.h

@@ -17,7 +17,8 @@
 #define _TD_UTIL_ENCODE_H_
 
 #include "tcoding.h"
-#include "tfreelist.h"
+#include "tlist.h"
+// #include "tfreelist.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -62,10 +63,14 @@ struct SCoderNode {
   CODER_NODE_FIELDS
 };
 
+typedef struct SCoderMem {
+  struct SCoderMem* next;
+} SCoderMem;
+
 typedef struct {
   td_coder_t  type;
   td_endian_t endian;
-  SFreeList   fl;
+  SCoderMem*  mList;
   CODER_NODE_FIELDS
   TD_SLIST(SCoderNode) stack;
 } SCoder;
@@ -74,7 +79,42 @@ typedef struct {
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
-#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER)          TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl))
+static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) {
+  void*      ptr = NULL;
+  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size);
+  if (pMem) {
+    pMem->next = pCoder->mList;
+    pCoder->mList = pMem;
+    ptr = (void*)&pMem[1];
+  }
+  return ptr;
+}
+
+#define tEncodeSize(E, S, SIZE, RET)                           \
+  do {                                                         \
+    SCoder coder = {0};                                        \
+    RET = 0;                                                   \
+    tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
+    if ((E)(&coder, S) == 0) {                                 \
+      SIZE = coder.pos;                                        \
+    } else {                                                   \
+      RET = -1;                                                \
+    }                                                          \
+    tCoderClear(&coder);                                       \
+  } while (0)
+// #define tEncodeSize(E, S, SIZE)                                \
+//   ({                                                           \
+//     SCoder coder = {0};                                        \
+//     int    ret = 0;                                            \
+//     tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
+//     if ((E)(&coder, S) == 0) {                                 \
+//       SIZE = coder.pos;                                        \
+//     } else {                                                   \
+//       ret = -1;                                                \
+//     }                                                          \
+//     tCoderClear(&coder);                                       \
+//     ret;                                                       \
+//   })
 
 void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
 void tCoderClear(SCoder* pCoder);
@@ -236,7 +276,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) {
   union {
     uint32_t ui;
     float    f;
-  } v = {.f = val};
+  } v;
+  v.f = val;
 
   return tEncodeU32(pEncoder, v.ui);
 }
@@ -245,7 +286,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) {
   union {
     uint64_t ui;
     double   d;
-  } v = {.d = val};
+  } v;
+  v.d = val;
 
   return tEncodeU64(pEncoder, v.ui);
 }
@@ -375,7 +417,9 @@ static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, ui
   if (tDecodeU64v(pDecoder, len) < 0) return -1;
 
   if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
-  *val = (void*)TD_CODER_CURRENT(pDecoder);
+  if (val) {
+    *val = (void*)TD_CODER_CURRENT(pDecoder);
+  }
 
   TD_CODER_MOVE_POS(pDecoder, *len);
   return 0;

include/util/tenv.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -13,17 +14,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TNETTEST_H
+#ifndef _TD_ENV_H_
-#define TDENGINE_TNETTEST_H
+#define _TD_ENV_H_
+
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
+int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen);
+int32_t taosEnvToCfg(const char *envStr, char *cfgStr);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_TNETTEST_H
+#endif /*_TD_ENV_H_*/

include/util/tjson.h

@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#ifdef WINDOWS
+#define tjsonGetNumberValue(pJson, pName, val) -1
+#else
 #define tjsonGetNumberValue(pJson, pName, val) \
   ({ \
     uint64_t _tmp = 0; \
@@ -29,6 +32,7 @@ extern "C" {
     val = _tmp; \
     _code; \
   })
+#endif
 
 typedef void SJson;
 
@@ -45,6 +49,8 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
 int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
 
 SJson*  tjsonGetObjectItem(const SJson* pJson, const char* pName);
+int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
+int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
 int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
 int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
 int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
@@ -77,6 +83,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
 bool   tjsonValidateJson(const char* pJson);
+const char* tjsonGetError();
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
+extern int32_t metaDebugFlag;
 extern int32_t fnDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
@@ -79,6 +80,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #endif
     ;
 
+// clang-format off
 #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@@ -88,6 +90,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
+// clang-format on
 
 #ifdef __cplusplus
 }

include/util/tprocess.h

@@ -53,8 +53,8 @@ int32_t   taosProcRun(SProcObj *pProc);
 void      taosProcStop(SProcObj *pProc);
 
 int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                            void *handle, EProcFuncType ftype);
+                            void *handle, int64_t handleRef, EProcFuncType ftype);
-void    taosProcRemoveHandle(SProcObj *pProc, void *handle);
+int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle);
 void    taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
 void    taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
                              EProcFuncType ftype);

include/util/tscalablebf.h

@@ -13,36 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TDB_ENV_H_
+#ifndef _TD_UTIL_SCALABLEBF_H_
-#define _TDB_ENV_H_
+#define _TD_UTIL_SCALABLEBF_H_
+
+#include "tbloomfilter.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct STEnv {
+typedef struct SScalableBf {
-  char    *rootDir;
+  SArray *bfArray; // array of bloom filters
-  char    *jfname;
+  uint32_t growth;
-  int      jfd;
+  uint64_t numBits;
-  SPCache *pCache;
+} SScalableBf;
-  SPager  *pgrList;
-  int      nPager;
-  int      nPgrHash;
-  SPager **pgrHash;
-} TENV;
 
-int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
+SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
-int tdbEnvClose(TENV *pEnv);
+int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
-int tdbBegin(TENV *pEnv, TXN *pTxn);
+int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
-int tdbCommit(TENV *pEnv, TXN *pTxn);
+                             uint32_t len);
-int tdbRollback(TENV *pEnv, TXN *pTxn);
+void tScalableBfDestroy(SScalableBf *pSBf);
-
+void tScalableBfDump(const SScalableBf *pSBf);
-void    tdbEnvAddPager(TENV *pEnv, SPager *pPager);
-void    tdbEnvRemovePager(TENV *pEnv, SPager *pPager);
-SPager *tdbEnvGetPager(TENV *pEnv, const char *fname);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TDB_ENV_H_*/
+#endif /*_TD_UTIL_SCALABLEBF_H_*/

include/util/tskiplist.h

@@ -52,72 +52,21 @@ typedef struct SSkipListNode {
 #define SL_NODE_GET_FORWARD_POINTER(n, l)  (n)->forwards[(l)]
 #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
 
-/*
- * @version 0.3
- * @date   2017/11/12
- * the simple version of skip list.
- *
- * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate
- * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance.
- * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of
- * this simple version in a multi-thread environment, to achieve higher performance of read/write operations.
- *
- * Note: Duplicated primary key situation.
- * In case of duplicated primary key, two ways can be employed to handle this situation:
- * 1. add as normal insertion without special process.
- * 2. add an overflow pointer at each list node, all nodes with the same key will be added in the overflow pointer.
- *    In this case, the total steps of each search will be reduced significantly.
- *    Currently, we implement the skip list in a line with the first means, maybe refactor it soon.
- *
- *    Memory consumption: the memory alignment causes many memory wasted. So, employ a memory
- *    pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs.
- *
- */
-
-// state struct, record following information:
-// number of links in each level.
-// avg search steps, for latest 1000 queries
-// avg search rsp time, for latest 1000 queries
-// total memory size
-typedef struct tSkipListState {
-  // in bytes, sizeof(SSkipList)+sizeof(SSkipListNode)*SSkipList->nSize
-  uint64_t nTotalMemSize;
-  uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL];
-  uint64_t queryCount;  // total query count
-
-  /*
-   * only record latest 1000 queries
-   * when the value==1000, = 0,
-   * nTotalStepsForQueries = 0,
-   * nTotalElapsedTimeForQueries = 0
-   */
-  uint64_t nRecQueries;
-  uint16_t nTotalStepsForQueries;
-  uint64_t nTotalElapsedTimeForQueries;
-
-  uint16_t nInsertObjs;
-  uint16_t nTotalStepsForInsert;
-  uint64_t nTotalElapsedTimeForInsert;
-} tSkipListState;
-
 typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus;
 
 typedef struct SSkipList {
-  uint32_t          seed;
+  uint32_t           seed;
-  __compar_fn_t     comparFn;
+  __compar_fn_t      comparFn;
-  __sl_key_fn_t     keyFn;
+  __sl_key_fn_t      keyFn;
-  TdThreadRwlock *lock;
+  TdThreadRwlock    *lock;
-  uint16_t          len;
+  uint16_t           len;
-  uint8_t           maxLevel;
+  uint8_t            maxLevel;
-  uint8_t           flags;
+  uint8_t            flags;
-  uint8_t           type;  // static info above
+  uint8_t            type;  // static info above
-  uint8_t           level;
+  uint8_t            level;
-  uint32_t          size;
+  uint32_t           size;
-  SSkipListNode    *pHead;  // point to the first element
+  SSkipListNode     *pHead;  // point to the first element
-  SSkipListNode    *pTail;  // point to the last element
+  SSkipListNode     *pTail;  // point to the last element
-#if SKIP_LIST_RECORD_PERFORMANCE
-  tSkipListState state;  // skiplist state
-#endif
   tGenericSavedFunc *insertHandleFn;
 } SSkipList;
 

include/util/tskiplist2.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_UTIL_SKIPLIST2_H_
+#define _TD_UTIL_SKIPLIST2_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SL_MAX_LEVEL 15
+
+typedef struct SSkipList2 SSkipList2;
+typedef struct SSLCursor  SSLCursor;
+typedef struct SSLCfg     SSLCfg;
+typedef struct SSLNode    SSLNode;
+
+typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
+
+// SSkipList2
+int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
+int32_t slClose(SSkipList2 *pSl);
+int32_t slClear(SSkipList2 *pSl);
+
+// SSLCursor
+int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
+int32_t slcClose(SSLCursor *pSlc);
+int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey);
+int32_t slcMoveToNext(SSLCursor *pSlc);
+int32_t slcMoveToPrev(SSLCursor *pSlc);
+int32_t slcMoveToFirst(SSLCursor *pSlc);
+int32_t slcMoveToLast(SSLCursor *pSlc);
+int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData);
+int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData);
+int32_t slcDrop(SSLCursor *pSlc);
+
+// struct
+struct SSLCfg {
+  int8_t    maxLevel;
+  int32_t   nKey;
+  int32_t   nData;
+  tslCmprFn cmprFn;
+  void     *pPool;
+  void *(*xMalloc)(void *, int32_t size);
+  void (*xFree)(void *, void *);
+};
+
+struct SSLCursor {
+  SSkipList2 *pSl;
+  SSLNode   **forwards[SL_MAX_LEVEL];
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SKIPLIST2_H_*/

include/util/ttimer.h

@@ -41,10 +41,6 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *han
 
 void taosTmrCleanUp(void *handle);
 
-int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);
-
-void taosUninitTimer();
-
 #ifdef __cplusplus
 }
 #endif

source/client/CMakeLists.txt

@@ -1,5 +1,9 @@
 aux_source_directory(src CLIENT_SRC)
-add_library(taos SHARED ${CLIENT_SRC})
+if(TD_WINDOWS)
+    add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in)
+else()
+    add_library(taos SHARED ${CLIENT_SRC})
+endif ()
 target_include_directories(
     taos
     PUBLIC "${TD_SOURCE_DIR}/include/client"
@@ -10,6 +14,13 @@ target_link_libraries(
     INTERFACE api
     PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
 )
+if(TD_WINDOWS)
+    set_target_properties(taos 
+        PROPERTIES 
+        LINK_FLAGS 
+        /DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/taos.def
+    )
+endif ()
 
 set_target_properties(
     taos

source/client/inc/clientInt.h

@@ -231,6 +231,10 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver
   msg->resIter++;
   if (msg->resIter < msg->rsp.blockNum) {
     SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    if (msg->rsp.withSchema) {
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
+      setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
+    }
     setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
     return &msg->resInfo;
   }
@@ -250,8 +254,6 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int taos_init();
-
 void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);
@@ -306,6 +308,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 void hbMgrInitMqHbRspHandle();
 
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
+int32_t      getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
+int32_t      scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
 
 #ifdef __cplusplus
 }

source/client/inc/clientStmt.h

@@ -34,8 +34,7 @@ typedef enum {
   STMT_PREPARE,
   STMT_SETTBNAME,
   STMT_SETTAGS,
-  STMT_FETCH_TAG_FIELDS,
+  STMT_FETCH_FIELDS,
-  STMT_FETCH_COL_FIELDS,
   STMT_BIND,
   STMT_BIND_COL,
   STMT_ADD_BATCH,
@@ -47,6 +46,13 @@ typedef struct SStmtTableCache {
   void*             boundTags;
 } SStmtTableCache;
 
+typedef struct SStmtQueryResInfo {
+  TAOS_FIELD*    fields;
+  TAOS_FIELD*    userFields;
+  uint32_t       numOfCols;
+  int32_t        precision;
+} SStmtQueryResInfo;
+
 typedef struct SStmtBindInfo {
   bool         needParse;
   uint64_t     tbUid;
@@ -67,14 +73,17 @@ typedef struct SStmtExecInfo {
 } SStmtExecInfo;
 
 typedef struct SStmtSQLInfo {
-  STMT_TYPE    type;
+  STMT_TYPE         type;
-  STMT_STATUS  status;
+  STMT_STATUS       status;
-  bool         autoCreate;
+  bool              autoCreate;
-  uint64_t     runTimes;
+  uint64_t          runTimes;
-  SHashObj*    pTableCache;   //SHash<SStmtTableCache>
+  SHashObj*         pTableCache;   //SHash<SStmtTableCache>
-  SQuery*      pQuery;
+  SQuery*           pQuery;
-  char*        sqlStr;
+  char*             sqlStr;
-  int32_t      sqlLen;
+  int32_t           sqlLen;
+  SArray*           nodeList;
+  SQueryPlan*       pQueryPlan;
+  SStmtQueryResInfo queryRes;
 } SStmtSQLInfo;
 
 typedef struct STscStmt {
@@ -87,6 +96,8 @@ typedef struct STscStmt {
   SStmtBindInfo bInfo;
 } STscStmt;
 
+#define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
+#define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
 
 #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
 #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
@@ -97,14 +108,15 @@ int stmtClose(TAOS_STMT *stmt);
 int stmtExec(TAOS_STMT *stmt);
 const char *stmtErrstr(TAOS_STMT *stmt);
 int stmtAffectedRows(TAOS_STMT *stmt);
+int stmtAffectedRowsOnce(TAOS_STMT *stmt);
 int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
 int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
-int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags);
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags);
 int stmtIsInsert(TAOS_STMT *stmt, int *insert);
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums);
 int stmtAddBatch(TAOS_STMT *stmt);
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt);
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx);
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx);
 
 
 #ifdef __cplusplus

source/client/src/clientEnv.c

@@ -254,12 +254,12 @@ void taos_init_imp(void) {
 
   deltaToUtcInitOnce();
 
-  if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) {
+  if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
     tscInitRes = -1;
     return;
   }
 
-  if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
     tscInitRes = -1;
     return;
   }

source/client/src/clientHb.c

@@ -14,9 +14,9 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "scheduler.h"
 #include "trpc.h"
 
 static SClientHbMgr clientHbMgr = {0};
@@ -110,7 +110,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
 static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
   SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == info) {
-    tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType);
+    tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid,
+            pRsp->connKey.connType);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -236,24 +237,24 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code)
 }
 
 int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
-  int64_t now = taosGetTimestampUs();
+  int64_t    now = taosGetTimestampUs();
   SQueryDesc desc = {0};
-  int32_t code = 0;
+  int32_t    code = 0;
 
-  void   *pIter = taosHashIterate(pObj->pRequests, NULL);
+  void *pIter = taosHashIterate(pObj->pRequests, NULL);
   while (pIter != NULL) {
-    int64_t *rid = pIter;
+    int64_t     *rid = pIter;
     SRequestObj *pRequest = acquireRequest(*rid);
     if (NULL == pRequest) {
       continue;
     }
 
     tstrncpy(desc.sql, pRequest->sqlstr, sizeof(desc.sql));
-    desc.stime    = pRequest->metric.start;
+    desc.stime = pRequest->metric.start;
-    desc.queryId  = pRequest->requestId;
+    desc.queryId = pRequest->requestId;
     desc.useconds = now - pRequest->metric.start;
-    desc.reqRid   = pRequest->self;
+    desc.reqRid = pRequest->self;
-    desc.pid      = hbBasic->pid;
+    desc.pid = hbBasic->pid;
     taosGetFqdn(desc.fqdn);
     desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0;
 
@@ -471,10 +472,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
     pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
   }
 
-//  if (code) {
+  //  if (code) {
-//    taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq);
+  //    taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq);
-//    taosMemoryFreeClear(pBatchReq);
+  //    taosMemoryFreeClear(pBatchReq);
-//  }
+  //  }
 
   return pBatchReq;
 }
@@ -631,7 +632,7 @@ void appHbMgrCleanup(void) {
   for (int i = 0; i < sz; i++) {
     SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
 
-    void   *pIter = taosHashIterate(pTarget->activeInfo, NULL);
+    void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
     while (pIter != NULL) {
       SClientHbReq *pOneReq = pIter;
       hbFreeReq(pOneReq);
@@ -641,7 +642,6 @@ void appHbMgrCleanup(void) {
     taosHashCleanup(pTarget->activeInfo);
     pTarget->activeInfo = NULL;
 
-
     pIter = taosHashIterate(pTarget->connInfo, NULL);
     while (pIter != NULL) {
       SHbConnInfo *info = pIter;
@@ -668,13 +668,13 @@ int hbMgrInit() {
   hbMgrInitHandle();
 
   // init backgroud thread
-  //hbCreateThread();
+  /*hbCreateThread();*/
 
   return 0;
 }
 
 void hbMgrCleanUp() {
-  //hbStopThread();
+  // hbStopThread();
 
   // destroy all appHbMgr
   int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);

source/client/src/clientImpl.c

@@ -95,19 +95,20 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
     if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
       return NULL;
     }
-
-    if (port) {
-      epSet.epSet.eps[0].port = port;
-    }
   } else {
     if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
       return NULL;
     }
   }
 
-  char*          key = getClusterKey(user, secretEncrypt, ip, port);
+  if (port) {
-  SAppInstInfo** pInst = NULL;
+    epSet.epSet.eps[0].port = port;
+    epSet.epSet.eps[1].port = port;
+  }
 
+  char* key = getClusterKey(user, secretEncrypt, ip, port);
+
+  SAppInstInfo** pInst = NULL;
   taosThreadMutexLock(&appInfo.mutex);
 
   pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
@@ -187,8 +188,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
       setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
     }
 
-    TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
+    TSWAP(pRequest->dbList, (*pQuery)->pDbList);
-    TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
+    TSWAP(pRequest->tableList, (*pQuery)->pTableList);
   }
 
   return code;
@@ -230,10 +231,14 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
                       .acctId = pRequest->pTscObj->acctId,
                       .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
                       .pAstRoot = pQuery->pRoot,
-                      .showRewrite = pQuery->showRewrite};
+                      .showRewrite = pQuery->showRewrite,
-  int32_t      code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
+                      .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
-  if (code != 0) {
+                      .pMsg = pRequest->msgBuf,
-    return code;
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .placeholderNum = pQuery->placeholderNum};
+  int32_t      code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
   }
   return code;
 }
@@ -242,6 +247,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   ASSERT(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
+  if (pResInfo->fields != NULL) {
+    taosMemoryFree(pResInfo->fields);
+  }
+  if (pResInfo->userFields != NULL) {
+    taosMemoryFree(pResInfo->userFields);
+  }
   pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
   pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
 
@@ -277,7 +288,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
 
   SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
   int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr,
-                                  pRequest->metric.start, &res);
+                                       pRequest->metric.start, &res);
   if (code != TSDB_CODE_SUCCESS) {
     if (pRequest->body.queryJob != 0) {
       schedulerFreeJob(pRequest->body.queryJob);
@@ -301,9 +312,12 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   return pRequest->code;
 }
 
-SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
+int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) {
-  SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
+  *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
+  return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList);
+}
 
+SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     switch (pQuery->execMode) {
       case QUERY_EXEC_MODE_LOCAL:
@@ -312,12 +326,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
       case QUERY_EXEC_MODE_RPC:
         code = execDdlQuery(pRequest, pQuery);
         break;
-      case QUERY_EXEC_MODE_SCHEDULE:
+      case QUERY_EXEC_MODE_SCHEDULE: {
+        SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
         code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
         if (TSDB_CODE_SUCCESS == code) {
           code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
         }
+        taosArrayDestroy(pNodeList);
         break;
+      }
       case QUERY_EXEC_MODE_EMPTY_RESULT:
         pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
         break;
@@ -326,7 +343,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
     }
   }
 
-  taosArrayDestroy(pNodeList);
   if (!keepQuery) {
     qDestroyQuery(pQuery);
   }
@@ -343,8 +359,15 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   SQuery*      pQuery = NULL;
 
   int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
-  if (TSDB_CODE_SUCCESS == code) {
+  if (code != TSDB_CODE_SUCCESS) {
-    code = parseSql(pRequest, false, &pQuery, NULL);
+    terrno = code;
+    return NULL;
+  }
+
+  code = parseSql(pRequest, false, &pQuery, NULL);
+  if (code != TSDB_CODE_SUCCESS) {
+    pRequest->code = code;
+    return pRequest;
   }
 
   return launchQueryImpl(pRequest, pQuery, code, false);
@@ -395,7 +418,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
     pRequest = launchQuery(pTscObj, sql, sqlLen);
-    if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
+    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
     }
 
@@ -436,7 +459,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
       return -1;
     }
 
-    taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return terrno;
+    }
+
     mgmtEpSet->numOfEps++;
   }
 
@@ -830,10 +858,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     return code;
   }
 
-  int32_t* colLength = (int32_t*)pResultInfo->pData;
+  char* p = (char*)pResultInfo->pData;
-  char*    pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
+
+  int32_t dataLen = *(int32_t*)p;
+  p += sizeof(int32_t);
+
+  uint64_t groupId = *(uint64_t*)p;
+  p += sizeof(uint64_t);
+
+  int32_t* colLength = (int32_t*)p;
+  p += sizeof(int32_t) * numOfCols;
+
+  char* pStart = p;
   for (int32_t i = 0; i < numOfCols; ++i) {
     colLength[i] = htonl(colLength[i]);
+    ASSERT(colLength[i] < dataLen);
 
     if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       pResultInfo->pCol[i].offset = (int32_t*)pStart;

source/client/src/clientMain.c

@@ -110,17 +110,24 @@ int taos_errno(TAOS_RES *tres) {
     return terrno;
   }
 
+  if (TD_RES_TMQ(tres)) {
+    return 0;
+  }
+
   return ((SRequestObj *)tres)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
-  SRequestObj *pRequest = (SRequestObj *)res;
+  if (res == NULL) {
-
-  if (pRequest == NULL) {
     return (const char *)tstrerror(terrno);
   }
 
-  if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
+  if (TD_RES_TMQ(res)) {
+    return "success";
+  }
+
+  SRequestObj *pRequest = (SRequestObj *)res;
+  if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
     return (const char *)tstrerror(pRequest->code);
@@ -603,7 +610,7 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
   return stmtPrepare(stmt, sql, length);
 }
 
-int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) {
+int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags) {
   if (stmt == NULL || name == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -632,11 +639,9 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
   return stmtSetTbName(stmt, name);
 }
 
-int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
+int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); }
-  return taos_stmt_set_tbname(stmt, name);
-}
 
-int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -652,7 +657,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
   return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -665,10 +670,18 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
     return terrno;
   }
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert && bind->num > 1) {
+    tscError("only one row data allowed for query");
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
   return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) {
+int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -681,6 +694,14 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int c
     return terrno;
   }
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert && bind->num > 1) {
+    tscError("only one row data allowed for query");
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
   return stmtBindBatch(stmt, bind, colIdx);
 }
 
@@ -734,9 +755,7 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
   return stmtUseResult(stmt);
 }
 
-char *taos_stmt_errstr(TAOS_STMT *stmt) {
+char *taos_stmt_errstr(TAOS_STMT *stmt) { return (char *)stmtErrstr(stmt); }
-  return (char *)stmtErrstr(stmt);
-}
 
 int taos_stmt_affected_rows(TAOS_STMT *stmt) {
   if (stmt == NULL) {
@@ -748,6 +767,16 @@ int taos_stmt_affected_rows(TAOS_STMT *stmt) {
   return stmtAffectedRows(stmt);
 }
 
+int taos_stmt_affected_rows_once(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return 0;
+  }
+
+  return stmtAffectedRowsOnce(stmt);
+}
+
 int taos_stmt_close(TAOS_STMT *stmt) {
   if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);

source/client/src/clientMsgHandler.c

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
+#include "catalog.h"
-#include "tdef.h"
-#include "tname.h"
 #include "clientInt.h"
 #include "clientLog.h"
-#include "catalog.h"
+#include "os.h"
 #include "query.h"
+#include "tdef.h"
+#include "tname.h"
 
 int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
 
@@ -50,7 +50,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
 
   SConnectRsp connectRsp = {0};
   tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp);
-  assert(connectRsp.epSet.numOfEps > 0);
+  /*assert(connectRsp.epSet.numOfEps > 0);*/
+  if (connectRsp.epSet.numOfEps == 0) {
+    taosMemoryFree(pMsg->pData);
+    setErrno(pRequest, TSDB_CODE_MND_APP_ERROR);
+    tsem_post(&pRequest->body.rspSem);
+    return code;
+  }
 
   if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
     updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
@@ -82,18 +88,20 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   return 0;
 }
 
-SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
+SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) {
   SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
 
   pMsgSendInfo->requestObjRefId = pRequest->self;
-  pMsgSendInfo->requestId       = pRequest->requestId;
+  pMsgSendInfo->requestId = pRequest->requestId;
-  pMsgSendInfo->param           = pRequest;
+  pMsgSendInfo->param = pRequest;
-  pMsgSendInfo->msgType         = pRequest->type;
+  pMsgSendInfo->msgType = pRequest->type;
 
   assert(pRequest != NULL);
   pMsgSendInfo->msgInfo = pRequest->body.requestMsg;
 
-  pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)];
+  pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)
+                         ? genericRspCallback
+                         : handleRequestRspFp[TMSG_INDEX(pRequest->type)];
   return pMsgSendInfo;
 }
 
@@ -114,7 +122,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
     SUseDbRsp usedbRsp = {0};
     tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
-    struct SCatalog *pCatalog = NULL;
+    struct SCatalog* pCatalog = NULL;
 
     if (usedbRsp.vgVersion >= 0) {
       int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
@@ -151,13 +159,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     taosMemoryFreeClear(output.dbVgroup);
 
     tscError("failed to build use db output since %s", terrstr());
-  } else {
+  } else if (output.dbVgroup) {
     struct SCatalog* pCatalog = NULL;
 
     int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (code1 != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
               tstrerror(code1));
+      taosMemoryFreeClear(output.dbVgroup);
     } else {
       catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
     }

source/client/src/clientStmt.c

@@ -5,14 +5,52 @@
 #include "tdef.h"
 
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
+  int32_t code = 0;
+  
   switch (newStatus) {
+    case STMT_PREPARE:
+      break;
     case STMT_SETTBNAME:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_SETTAGS:
+      if (STMT_STATUS_NE(SETTBNAME)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_FETCH_FIELDS:
+      if (STMT_STATUS_EQ(INIT)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_BIND:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_BIND_COL:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_ADD_BATCH:
+      if (STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL) && STMT_STATUS_NE(FETCH_FIELDS)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_EXECUTE:
+      if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
       break;
     default:
+      code = TSDB_CODE_TSC_APP_ERROR;
       break;
   }
 
-  //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  STMT_ERR_RET(code);
 
   pStmt->sql.status = newStatus;
 
@@ -35,6 +73,49 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t stmtBackupQueryFields(STscStmt* pStmt) {
+  SStmtQueryResInfo *pRes = &pStmt->sql.queryRes;
+  pRes->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols;
+  pRes->precision = pStmt->exec.pRequest->body.resInfo.precision;
+  
+  int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD);
+  pRes->fields = taosMemoryMalloc(size);
+  pRes->userFields = taosMemoryMalloc(size);
+  if (NULL == pRes->fields || NULL == pRes->userFields) {
+    STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size);
+  memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtRestoreQueryFields(STscStmt* pStmt) {
+  SStmtQueryResInfo *pRes = &pStmt->sql.queryRes;
+  int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD);
+  
+  pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols;
+  pStmt->exec.pRequest->body.resInfo.precision = pRes->precision;
+
+  if (NULL == pStmt->exec.pRequest->body.resInfo.fields) {
+    pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size);
+    if (NULL == pStmt->exec.pRequest->body.resInfo.fields) {
+      STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
+    }
+    memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size);
+  }
+
+  if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) {
+    pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size);
+    if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) {
+      STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
+    }
+    memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -69,15 +150,10 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
     return TSDB_CODE_SUCCESS;
   }
 
-  uint64_t uid;
+  uint64_t uid = pStmt->bInfo.tbUid; 
-  if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) {
+  uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid;
-    uid = pStmt->bInfo.tbSuid;
-  } else {
-    ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType);
-    uid = pStmt->bInfo.tbUid;
-  }
 
-  if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) {
+  if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -91,7 +167,7 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
     .boundTags = pStmt->bInfo.boundTags,
   };
 
-  if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) {
+  if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
@@ -149,9 +225,11 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
+int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
-  taos_free_result(pStmt->exec.pRequest);
+  if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) {
-  pStmt->exec.pRequest = NULL;
+    taos_free_result(pStmt->exec.pRequest);
+    pStmt->exec.pRequest = NULL;
+  }
 
   void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
   while (pIter) {
@@ -184,8 +262,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
 }
 
 int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
+  taosMemoryFree(pStmt->sql.queryRes.fields);
+  taosMemoryFree(pStmt->sql.queryRes.userFields);
   taosMemoryFree(pStmt->sql.sqlStr);
   qDestroyQuery(pStmt->sql.pQuery);
+  qDestroyQueryPlan(pStmt->sql.pQueryPlan);
+  taosArrayDestroy(pStmt->sql.nodeList);
   
   void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
   while (pIter) {
@@ -201,7 +283,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
 
   memset(&pStmt->sql, 0, sizeof(pStmt->sql));
 
-  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false));
+  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true));
   STMT_ERR_RET(stmtCleanBindInfo(pStmt));
 
   return TSDB_CODE_SUCCESS;
@@ -221,37 +303,42 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
   STableMeta *pTableMeta = NULL;
   SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
   STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+  uint64_t uid = pTableMeta->uid;
+  uint64_t suid = pTableMeta->suid;
+  int8_t tableType = pTableMeta->tableType;
+  taosMemoryFree(pTableMeta);
   
-  if (pTableMeta->uid == pStmt->bInfo.tbUid) {
+  if (uid == pStmt->bInfo.tbUid) {
     pStmt->bInfo.needParse = false;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) {
+  if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) {
-    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
     if (NULL == pCache) {
-      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid);
+      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid);
+      
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
     }
     
     pStmt->bInfo.needParse = false;
     
-    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbUid = uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbSuid = suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
   if (pCache) {
     pStmt->bInfo.needParse = false;
 
-    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbUid = uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbSuid = suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     STableDataBlocks* pNewBlock = NULL;
@@ -333,6 +420,13 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert) {
+    tscError("set tb name not available for none insert statement");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
   if (NULL == pStmt->exec.pRequest) {
     STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
   }
@@ -349,7 +443,7 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
@@ -370,15 +464,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
 }
 
 
-int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
-  STscStmt* pStmt = (STscStmt*)stmt;
-
-  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS));
-
-  if (pStmt->bInfo.needParse) {
-    STMT_ERR_RET(stmtParseSql(pStmt));
-  }
-
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     tscError("invalid operation to get query tag fileds");
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -395,15 +481,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
-  STscStmt* pStmt = (STscStmt*)stmt;
-
-  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS));
-
-  if (pStmt->bInfo.needParse) {
-    STMT_ERR_RET(stmtParseSql(pStmt));
-  }
-
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     tscError("invalid operation to get query column fileds");
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -420,7 +498,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
   return TSDB_CODE_SUCCESS;  
 }
 
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
@@ -429,6 +507,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
     pStmt->bInfo.needParse = false;
   }
 
+  if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
+    taos_free_result(pStmt->exec.pRequest);
+    pStmt->exec.pRequest = NULL;
+  }
+  
   if (NULL == pStmt->exec.pRequest) {
     STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
   }
@@ -437,6 +520,19 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
     STMT_ERR_RET(stmtParseSql(pStmt));
   }
 
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    if (NULL == pStmt->sql.pQueryPlan) {
+      STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
+      pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
+      pStmt->exec.pRequest->body.pDag = NULL;
+      STMT_ERR_RET(stmtBackupQueryFields(pStmt));
+    } else {
+      STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
+    }
+    
+    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId));
+  }
+  
   STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
   if (NULL == pDataBlock) {
     tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
@@ -444,7 +540,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
   }
 
   if (colIdx < 0) {
-    qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
+    int32_t code = qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
+    if (code) {
+      tscError("qBindStmtColsValue failed, error:%s", tstrerror(code));
+      STMT_ERR_RET(code);
+    }
   } else {
     if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) {
       tscError("bind column index not in sequence");
@@ -480,9 +580,12 @@ int stmtExec(TAOS_STMT *stmt) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
 
-  STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
-
+    scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList);
-  launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
+  } else {
+    STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
+    launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
+  }
   
   STMT_ERR_JRET(pStmt->exec.pRequest->code);
 
@@ -491,7 +594,7 @@ int stmtExec(TAOS_STMT *stmt) {
 
 _return:
 
-  stmtCleanExecInfo(pStmt, (code ? false : true));
+  stmtCleanExecInfo(pStmt, (code ? false : true), false);
   
   ++pStmt->sql.runTimes;
   
@@ -503,6 +606,8 @@ int stmtClose(TAOS_STMT *stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_RET(stmtCleanSQLInfo(pStmt));
+
+  taosMemoryFree(stmt);
 }
 
 const char *stmtErrstr(TAOS_STMT *stmt) {
@@ -523,6 +628,10 @@ int stmtAffectedRows(TAOS_STMT *stmt) {
   return ((STscStmt*)stmt)->affectedRows;
 }
 
+int stmtAffectedRowsOnce(TAOS_STMT *stmt) {
+  return ((STscStmt*)stmt)->exec.affectedRows;
+}
+
 int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -536,13 +645,54 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
 }
 
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
-  STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
+
+  if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
+    pStmt->bInfo.needParse = false;
+  }
+
+  if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
+    taos_free_result(pStmt->exec.pRequest);
+    pStmt->exec.pRequest = NULL;
+  }
+  
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    if (NULL == pStmt->sql.pQueryPlan) {
+      STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
+      pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
+      pStmt->exec.pRequest->body.pDag = NULL;
+      STMT_ERR_RET(stmtBackupQueryFields(pStmt));
+    } else {
+      STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
+    }
+    
+    *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
+  } else {
+    STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
+  }
   
   return TSDB_CODE_SUCCESS;
 }
 
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt) {
-  return NULL;
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (STMT_TYPE_QUERY != pStmt->sql.type) {
+    tscError("useResult only for query statement");
+    return NULL;
+  }
+
+  return pStmt->exec.pRequest;
 }
 
 

source/client/src/taos.def

@@ -0,0 +1,81 @@
+taos_cleanup
+taos_options
+taos_set_config
+taos_init
+taos_connect
+taos_connect_l
+taos_connect_auth
+taos_close
+taos_data_type
+taos_stmt_init
+taos_stmt_prepare
+taos_stmt_set_tbname_tags
+taos_stmt_set_tbname
+taos_stmt_set_sub_tbname
+taos_stmt_is_insert
+taos_stmt_num_params
+taos_stmt_get_param
+taos_stmt_bind_param
+taos_stmt_bind_param_batch
+taos_stmt_bind_single_param_batch
+taos_stmt_add_batch
+taos_stmt_execute
+taos_stmt_use_result
+taos_stmt_close
+taos_stmt_errstr
+taos_stmt_affected_rows
+taos_stmt_affected_rows_once
+taos_query
+taos_query_l
+taos_fetch_row
+taos_result_precision
+taos_free_result
+taos_field_count
+taos_num_fields
+taos_affected_rows
+taos_fetch_fields
+taos_select_db
+taos_print_row
+taos_stop_query
+taos_is_null
+taos_is_update_query
+taos_fetch_block
+taos_fetch_block_s
+taos_fetch_raw_block
+taos_get_column_data_offset
+taos_validate_sql
+taos_reset_current_db
+taos_fetch_lengths
+taos_result_block
+taos_get_server_info
+taos_get_client_info
+taos_errstr
+taos_errno
+taos_query_a
+taos_fetch_rows_a
+taos_subscribe
+taos_consume
+taos_unsubscribe
+taos_load_table_info
+taos_schemaless_insert
+tmq_list_new
+tmq_list_append
+tmq_list_destroy
+tmq_list_get_size
+tmq_list_to_c_array
+tmq_consumer_new
+tmq_err2str
+tmq_subscribe
+tmq_unsubscribe
+tmq_subscription
+tmq_consumer_poll
+tmq_consumer_close
+tmq_commit
+tmq_conf_new
+tmq_conf_set
+tmq_conf_destroy
+tmq_conf_set_offset_commit_cb
+tmq_get_topic_name
+tmq_get_vgroup_id
+tmq_create_stream
+taos_check_server_status

source/client/src/taos.rc.in

@@ -0,0 +1,31 @@
+1 VERSIONINFO
+ FILEVERSION ${TD_VER_NUMBER}
+ PRODUCTVERSION ${TD_VER_NUMBER}
+ FILEFLAGSMASK 0x17L
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x0L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "FileDescription", "Native C Driver for TDengine"
+            VALUE "FileVersion", "${TD_VER_NUMBER}"
+            VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})"
+            VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data"
+            VALUE "OriginalFilename", ""
+            VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})"
+            VALUE "ProductVersion", "${TD_VER_NUMBER}"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END

source/client/test/clientTests.cpp

@@ -659,10 +659,15 @@ TEST(testCase, agg_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+  TAOS_RES* pRes = taos_query(pConn, "use db");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to use db, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    ASSERT_TRUE(false);
+  }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select now() from m1");
+  pRes = taos_query(pConn, "select tbname from st1");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/CMakeLists.txt

@@ -5,8 +5,8 @@ target_include_directories(
     PUBLIC "${TD_SOURCE_DIR}/include/common"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 IF(${TD_WINDOWS})
-    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread"
-    PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex"
 ENDIF ()
 )
 target_link_libraries(

source/common/src/systable.c

@@ -0,0 +1,340 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "systable.h"
+#include "tdef.h"
+#include "types.h"
+#include "taos.h"
+
+#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+#define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+#define SYSTABLE_SCH_COL_NAME_LEN   ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+
+static const SSysDbTableSchema dnodesSchema[] = {
+    {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema mnodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema modulesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema qnodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema snodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema bnodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema clusterSchema[] = {
+    {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema userDBSchema[] = {
+    {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "duration", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "pages", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},  // disable update
+};
+
+static const SSysDbTableSchema userFuncSchema[] = {
+    {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
+static const SSysDbTableSchema userIdxSchema[] = {
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema userStbsSchema[] = {
+    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema userStreamsSchema[] = {
+    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema userTblsSchema[] = {
+    {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "table_comment", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema userTblDistSchema[] = {
+    {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
+    {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
+static const SSysDbTableSchema userUsersSchema[] = {
+    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema grantsSchema[] = {
+    {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema vgroupsSchema[] = {
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
+static const SSysDbTableSchema smaSchema[] = {
+    {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema transSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema configSchema[] = {
+    {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysTableMeta infosMeta[] = {
+    {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
+    {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
+    {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
+    {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
+    {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)},
+    {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)},
+    {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)},
+    {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
+    {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
+    {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
+    {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
+    {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
+    {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
+    {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
+    {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
+    {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
+    {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
+    {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)},
+};
+
+static const SSysDbTableSchema connectionsSchema[] = {
+    {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT},
+    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema topicSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    // TODO config
+};
+
+static const SSysDbTableSchema consumerSchema[] = {
+    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+};
+
+static const SSysDbTableSchema subscriptionSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+};
+
+static const SSysDbTableSchema offsetSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+};
+
+static const SSysDbTableSchema querySchema[] = {
+    {.name = "query_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL},
+    {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+};
+
+static const SSysDbTableSchema streamSchema[] = {
+    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
+static const SSysTableMeta perfsMeta[] = {
+    {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)},
+    {TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema)},
+    {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
+    {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
+    {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
+    {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
+    {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)},
+    {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
+    {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
+};
+
+void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) {
+  *pInfosTableMeta = infosMeta;
+  *size = tListLen(infosMeta);
+}
+
+void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size) {
+  *pPerfsTableMeta = perfsMeta;
+  *size = tListLen(perfsMeta);
+}

source/common/src/tdatablock.c

@@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
 
   if (pEp->port == 0) {
     pEp->port = tsServerPort;
-    return -1;
   }
 
   return 0;
@@ -80,7 +79,11 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return pColumnInfoData->varmeta.length;
   } else {
-    return pColumnInfoData->info.bytes * numOfRows;
+    if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) {
+      return 0;
+    } else {
+      return pColumnInfoData->info.bytes * numOfRows;
+    }
   }
 }
 
@@ -114,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
     int32_t dataLen = varDataTLen(pData);
-    if(type == TSDB_DATA_TYPE_JSON) {
+    if (type == TSDB_DATA_TYPE_JSON) {
-      if(*pData == TSDB_DATA_TYPE_NULL) {
+      if (*pData == TSDB_DATA_TYPE_NULL) {
         dataLen = 0;
-      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
+      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
-        dataLen = varDataTLen(pData+CHAR_BYTES);
+        dataLen = varDataTLen(pData + CHAR_BYTES);
-      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+      } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
         dataLen = LONG_BYTES;
-      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
         dataLen = CHAR_BYTES;
       }
       dataLen += CHAR_BYTES;
     }
+
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
     if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
-      if (newSize == 0) {
+      if (newSize <= 1) {
         newSize = 8;
       }
 
@@ -164,13 +168,6 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
 
   uint32_t total = numOfRow1 + numOfRow2;
 
-  if (BitmapLen(numOfRow1) < BitmapLen(total)) {
-    char*    tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total));
-    uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1);
-    memset(tmp + BitmapLen(numOfRow1), 0, extend);
-    pColumnInfoData->nullbitmap = tmp;
-  }
-
   uint32_t remindBits = BitPos(numOfRow1);
   uint32_t shiftBits = 8 - remindBits;
 
@@ -190,25 +187,24 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
   int32_t i = 0;
 
   uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
-  int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1);
+  int32_t  overCount = BitmapLen(total) - BitmapLen(numOfRow1);
-  while (i < len) { // size limit of pSource->nullbitmap
+  while (i < len) {  // size limit of pSource->nullbitmap
     if (i >= 1) {
-      start[i - 1] |= (p[i] >> remindBits); //copy remind bits
+      start[i - 1] |= (p[i] >> remindBits);  // copy remind bits
     }
 
-    if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap
+    if (i >= overCount) {  // size limit of pColumnInfoData->nullbitmap
       return;
     }
 
-    start[i] |= (p[i] << shiftBits);  //copy shift bits
+    start[i] |= (p[i] << shiftBits);  // copy shift bits
     i += 1;
   }
 }
 
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
-                        uint32_t numOfRow2) {
+                        const SColumnInfoData* pSource, uint32_t numOfRow2) {
   ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
-
   if (numOfRow2 == 0) {
     return numOfRow1;
   }
@@ -217,16 +213,25 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     pColumnInfoData->hasNull = pSource->hasNull;
   }
 
+  uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     // Handle the bitmap
-    char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
+    if (finalNumOfRows > *capacity) {
-    if (p == NULL) {
+      char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
-      // TODO
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      *capacity = finalNumOfRows;
+      pColumnInfoData->varmeta.offset = (int32_t*)p;
     }
 
-    pColumnInfoData->varmeta.offset = (int32_t*)p;
     for (int32_t i = 0; i < numOfRow2; ++i) {
-      pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
+      if (pSource->varmeta.offset[i] == -1) {
+        pColumnInfoData->varmeta.offset[i + numOfRow1] = -1;
+      } else {
+        pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
+      }
     }
 
     // copy data
@@ -235,7 +240,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     if (pColumnInfoData->varmeta.allocLen < len + oldLen) {
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen);
       if (tmp == NULL) {
-        return TSDB_CODE_VND_OUT_OF_MEMORY;
+        return TSDB_CODE_OUT_OF_MEMORY;
       }
 
       pColumnInfoData->pData = tmp;
@@ -245,15 +250,27 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
     pColumnInfoData->varmeta.length = len + oldLen;
   } else {
-    doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2);
+    if (finalNumOfRows > *capacity) {
+      char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
+      if (tmp == NULL) {
+        return TSDB_CODE_VND_OUT_OF_MEMORY;
+      }
 
-    int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes;
+      pColumnInfoData->pData = tmp;
-    char*   tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize);
+
-    if (tmp == NULL) {
+      if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) {
-      return TSDB_CODE_VND_OUT_OF_MEMORY;
+        char*    btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows));
+        uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1);
+        memset(btmp + BitmapLen(numOfRow1), 0, extend);
+
+        pColumnInfoData->nullbitmap = btmp;
+      }
+
+      *capacity = finalNumOfRows;
     }
 
-    pColumnInfoData->pData = tmp;
+    doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2);
+
     int32_t offset = pColumnInfoData->info.bytes * numOfRow1;
     memcpy(pColumnInfoData->pData + offset, pSource->pData, pSource->info.bytes * numOfRow2);
   }
@@ -308,6 +325,8 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
     memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
   }
 
+  pColumnInfoData->hasNull = pSource->hasNull;
+  pColumnInfoData->info = pSource->info;
   return 0;
 }
 
@@ -340,29 +359,22 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
 // if pIndexMap = NULL, merger one column by on column
 int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) {
   assert(pSrc != NULL && pDest != NULL);
+  int32_t capacity = pDest->info.capacity;
 
-  int32_t numOfCols = pDest->info.numOfCols;
+  for (int32_t i = 0; i < pDest->info.numOfCols; ++i) {
-  for (int32_t i = 0; i < numOfCols; ++i) {
     int32_t mapIndex = i;
-    if(pIndexMap) {
+    if (pIndexMap) {
       mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
     }
+
     SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
     SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex);
 
-    uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows);
+    capacity = pDest->info.capacity;
-    uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows);
+    colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows);
-
-    int32_t newSize = oldLen + newLen;
-    char*   tmp = taosMemoryRealloc(pCol2->pData, newSize);
-    if (tmp != NULL) {
-      pCol2->pData = tmp;
-      colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows);
-    } else {
-      return TSDB_CODE_VND_OUT_OF_MEMORY;
-    }
   }
 
+  pDest->info.capacity = capacity;
   pDest->info.rows += pSrc->info.rows;
   return TSDB_CODE_SUCCESS;
 }
@@ -441,7 +453,6 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
   // all fit in
   *stopIndex = numOfRows - 1;
   return TSDB_CODE_SUCCESS;
-
 }
 
 SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) {
@@ -480,7 +491,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
     SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
 
     for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
-      bool  isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg);
+      bool isNull = false;
+      if (pBlock->pBlockAgg == NULL) {
+        isNull = colDataIsNull(pColData, pBlock->info.rows, j, NULL);
+      } else {
+        isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
+      }
       char* p = colDataGetData(pColData, j);
 
       colDataAppend(pDstCol, j - startIndex, p, isNull);
@@ -546,7 +562,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
 
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       size_t metaSize = pBlock->info.rows * sizeof(int32_t);
-      char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize);    // preview calloc is too small
+      char*  tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize);  // preview calloc is too small
       if (tmp == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
       }
@@ -645,12 +661,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
 
 /**
  * @refitem blockDataToBuf for the meta size
- *
  * @param pBlock
  * @return
  */
 size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
-  return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t);
+  // | total rows/total length | block group id | each column length |
+  return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
 }
 
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@@ -692,8 +708,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
     SColumnInfoData* pColInfoData = pOrder->pColData;  // TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex);
 
     if (pColInfoData->hasNull) {
-      bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg);
+      bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, NULL);
-      bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg);
+      bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, NULL);
       if (leftNull && rightNull) {
         continue;  // continue to next slot
       }
@@ -732,7 +748,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
     SColumnInfoData* pDst = &pDstCols[i];
     SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i);
 
-    if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg)) {
+    if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) {
       code = colDataAppend(pDst, numOfRows, NULL, true);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
@@ -928,8 +944,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
   copyBackToBlock(pDataBlock, pCols);
   int64_t p4 = taosGetTimestampUs();
 
-  uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
+  uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64
-         p3 - p2, p4 - p3, rows);
+         ", rows:%d\n",
+         p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows);
   destroyTupleIndex(index);
 
   return TSDB_CODE_SUCCESS;
@@ -1166,7 +1183,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
 }
 
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
-  if(pDataBlock == NULL){
+  if (pDataBlock == NULL) {
     return NULL;
   }
 
@@ -1177,7 +1194,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 
   pBlock->info.numOfCols = numOfCols;
   pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
-  pBlock->info.rowSize   = pDataBlock->info.rows;
+  pBlock->info.rowSize = pDataBlock->info.rows;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
@@ -1207,7 +1224,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 }
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock));
+  return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock));
 }
 
 void colDataDestroy(SColumnInfoData* pColData) {
@@ -1220,19 +1237,18 @@ void colDataDestroy(SColumnInfoData* pColData) {
   taosMemoryFree(pColData->pData);
 }
 
-
 static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
   int32_t len = BitmapLen(total);
 
   int32_t newLen = BitmapLen(total - n);
-  if (n%8 == 0) {
+  if (n % 8 == 0) {
-    memmove(nullBitmap, nullBitmap + n/8, newLen);
+    memmove(nullBitmap, nullBitmap + n / 8, newLen);
   } else {
     int32_t tail = n % 8;
     int32_t i = 0;
 
-    uint8_t* p = (uint8_t*) nullBitmap;
+    uint8_t* p = (uint8_t*)nullBitmap;
-    while(i < len) {
+    while (i < len) {
       uint8_t v = p[i];
 
       p[i] = 0;
@@ -1259,7 +1275,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
   }
 }
 
-int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
+int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
   if (n == 0) {
     return TSDB_CODE_SUCCESS;
   }
@@ -1267,7 +1283,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
   if (pBlock->info.rows <= n) {
     blockDataCleanup(pBlock);
   } else {
-    for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
       colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
     }
@@ -1453,3 +1469,193 @@ void blockDebugShowData(const SArray* dataBlocks) {
   }
 }
 
+/**
+ * @brief TODO: Assume that the final generated result it less than 3M
+ *
+ * @param pReq
+ * @param pDataBlocks
+ * @param vgId
+ * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in
+ * SDataBlock->info.uid
+ * @param suid  // TODO: check with Liao whether suid response is reasonable
+ * 
+ * TODO: colId should be set
+ */
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid,
+                                        tb_uid_t suid) {
+  int32_t sz = taosArrayGetSize(pDataBlocks);
+  int32_t bufSize = sizeof(SSubmitReq);
+  for (int32_t i = 0; i < sz; ++i) {
+    SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
+    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
+    bufSize += sizeof(SSubmitBlk);
+  }
+
+  ASSERT(bufSize < 3 * 1024 * 1024);
+
+  *pReq = taosMemoryCalloc(1, bufSize);
+  if(!(*pReq)) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+  void* pDataBuf = *pReq;
+
+  int32_t msgLen = sizeof(SSubmitReq);
+  int32_t     numOfBlks = 0;
+  SRowBuilder rb = {0};
+  tdSRowInit(&rb, 0); // TODO: use the latest version
+
+  for (int32_t i = 0; i < sz; ++i) {
+    SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
+    int32_t      colNum = pDataBlock->info.numOfCols;
+    int32_t      rows = pDataBlock->info.rows;
+    int32_t      rowSize = pDataBlock->info.rowSize;
+    int64_t      groupId = pDataBlock->info.groupId;
+    
+    if(rb.nCols != colNum) {
+      tdSRowSetTpInfo(&rb, colNum, pTSchema->flen);
+    }
+
+    SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen);
+    pSubmitBlk->suid = suid;
+    pSubmitBlk->uid = uid;
+    pSubmitBlk->numOfRows = rows;
+
+    ++numOfBlks;
+
+    msgLen += sizeof(SSubmitBlk);
+    int32_t dataLen = 0;
+    for (int32_t j = 0; j < rows; ++j) {     // iterate by row
+      tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf
+      printf("|");
+      bool  isStartKey = false;
+      for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
+        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+        switch (pColInfoData->info.type) {
+          case TSDB_DATA_TYPE_TIMESTAMP:
+            if (!isStartKey) {
+              isStartKey = true;
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0);
+            } else {
+              tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k);
+              break;
+            }
+            break;
+          case TSDB_DATA_TYPE_NCHAR: {
+            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k);
+            break;
+          }
+          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
+            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k);
+            break;
+          }
+          case TSDB_DATA_TYPE_VARBINARY:
+          case TSDB_DATA_TYPE_DECIMAL:
+          case TSDB_DATA_TYPE_BLOB:
+          case TSDB_DATA_TYPE_MEDIUMBLOB:
+            printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type);
+            TASSERT(0);
+            break;
+          default:
+            if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
+              tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k);
+            } else {
+              printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
+              TASSERT(0);
+            }
+            break;
+        }
+      }
+      dataLen += TD_ROW_LEN(rb.pBuf);
+    }
+    pSubmitBlk->dataLen = dataLen;
+    msgLen += pSubmitBlk->dataLen;
+  }
+
+  (*pReq)->length = msgLen;
+
+  (*pReq)->header.vgId = htonl(vgId);
+  (*pReq)->header.contLen = htonl(msgLen);
+  (*pReq)->length = (*pReq)->header.contLen;
+  (*pReq)->numOfBlocks = htonl(numOfBlks);
+  SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1);
+  while (numOfBlks--) {
+    int32_t dataLen = blk->dataLen;
+    blk->uid = htobe64(blk->uid);
+    blk->suid = htobe64(blk->suid);
+    blk->padding = htonl(blk->padding);
+    blk->sversion = htonl(blk->sversion);
+    blk->dataLen = htonl(blk->dataLen);
+    blk->schemaLen = htonl(blk->schemaLen);
+    blk->numOfRows = htons(blk->numOfRows);
+    blk = (SSubmitBlk*)(blk->data + dataLen);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) {
+  SSubmitReq* ret = NULL;
+
+  // cal size
+  int32_t cap = sizeof(SSubmitReq);
+  int32_t sz = taosArrayGetSize(pBlocks);
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+    int32_t      rows = pDataBlock->info.rows;
+    // TODO min
+    int32_t rowSize = pDataBlock->info.rowSize;
+    int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+    cap += sizeof(SSubmitBlk) + rows * maxLen;
+  }
+
+  // assign data
+  ret = taosMemoryCalloc(1, cap);
+  ret->version = htonl(1);
+  ret->length = htonl(cap - sizeof(SSubmitReq));
+  ret->numOfBlocks = htonl(sz);
+
+  void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq));
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+
+    SSubmitBlk* blkHead = submitBlk;
+    blkHead->numOfRows = htons(pDataBlock->info.rows);
+    blkHead->schemaLen = 0;
+    blkHead->sversion = htonl(pTSchema->version);
+    // TODO
+    blkHead->suid = 0;
+    blkHead->uid = htobe64(pDataBlock->info.uid);
+
+    int32_t rows = pDataBlock->info.rows;
+    int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+    /*blkHead->dataLen = htonl(rows * maxLen);*/
+    blkHead->dataLen = 0;
+
+    void*   blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk));
+    STSRow* rowData = blockData;
+
+    for (int32_t j = 0; j < pDataBlock->info.rows; j++) {
+      SRowBuilder rb = {0};
+      tdSRowInit(&rb, pTSchema->version);
+      tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);
+      tdSRowResetBuf(&rb, rowData);
+
+      for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
+        const STColumn*  pColumn = &pTSchema->columns[k];
+        SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k);
+        void*            data = colDataGetData(pColData, j);
+        tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k);
+      }
+      int32_t rowLen = TD_ROW_LEN(rowData);
+      rowData = POINTER_SHIFT(rowData, rowLen);
+      blkHead->dataLen += rowLen;
+    }
+    int32_t len = blkHead->dataLen;
+    blkHead->dataLen = htonl(len);
+    blkHead = POINTER_SHIFT(blkHead, len);
+  }
+
+  return ret;
+}

source/common/src/tdataformat.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "tdataformat.h"
 #include "tcoding.h"
+#include "tdatablock.h"
 #include "tlog.h"
 
 static void dataColSetNEleNull(SDataCol *pCol, int nEle);
@@ -87,7 +88,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) {
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     tlen += taosEncodeFixedI8(buf, colType(pCol));
-    tlen += taosEncodeFixedI8(buf, colSma(pCol));
+    tlen += taosEncodeFixedI8(buf, colFlags(pCol));
     tlen += taosEncodeFixedI16(buf, colColId(pCol));
     tlen += taosEncodeFixedI16(buf, colBytes(pCol));
   }
@@ -110,14 +111,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
-    int8_t      sma = 0;
+    int8_t      flags = 0;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);
-    buf = taosDecodeFixedI8(buf, &sma);
+    buf = taosDecodeFixedI8(buf, &flags);
     buf = taosDecodeFixedI16(buf, &colId);
     buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) {
+    if (tdAddColToSchema(&schemaBuilder, type, flags, colId, bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;
     }
@@ -128,6 +129,50 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
   return buf;
 }
 
+#if 0
+int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) {
+  if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1;
+  if (tEncodeI8(pEncoder, pCol->type) < 0) return -1;
+  if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1;
+  if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1;
+  if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1;
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) {
+  if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1;
+  return 0;
+}
+
+int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) {
+  if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1;
+  if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1;
+  if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1;
+  if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1;
+  if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1;
+
+  for (int32_t i = 0; i < schemaNCols(pSchema); i++) {
+    const STColumn *pCol = schemaColAt(pSchema, i);
+    if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1;
+  }
+  return 0;
+}
+
+int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) {
+  if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1;
+  if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1;
+  if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1;
+
+  return 0;
+}
+#endif
+
 int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   if (pBuilder == NULL) return -1;
 
@@ -153,7 +198,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->version = version;
 }
 
-int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) {
+int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) {
   if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
@@ -166,7 +211,7 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
   colSetType(pCol, type);
   colSetColId(pCol, colId);
-  colSetSma(pCol, sma);
+  colSetFlags(pCol, flags);
   if (pBuilder->nCols == 0) {
     colSetOffset(pCol, 0);
   } else {