Merge pull request #12642 from taosdata/3.0

3.0

.devcontainer/Dockerfile

@@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
 # [Optional] Uncomment this section to install additional packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
-RUN apt-get update && apt-get -y install tree vim tmux
+RUN apt-get update && apt-get -y install tree vim tmux python3-pip

.gitignore

@@ -40,6 +40,7 @@ taosdalipu/
 Target/
 *.failed
 *.sql
+sim
 sim/
 psim/
 pysim/
@@ -107,3 +108,4 @@ TAGS
 contrib/*
 !contrib/CMakeLists.txt
 !contrib/test
+sql

CMakeLists.txt

@@ -7,7 +7,7 @@ project(
 )
 
 if (NOT DEFINED TD_SOURCE_DIR)
-  set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} )
+  set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
 endif()
 
 set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")

Jenkinsfile

@@ -113,7 +113,7 @@ pipeline {
               '''
               sh'''
               cd ${WKC}/debug
-              ctest
+              ctest -VV
               '''
             }
           }

Jenkinsfile2

@@ -33,60 +33,183 @@ def abort_previous(){
     milestone(buildNumber)
 }
 def pre_test(){
-    sh'hostname'
     sh '''
+        hostname
         date
-    sudo rmtaos || echo "taosd has not installed"
     '''
     sh '''
-    killall -9 taosd ||echo "no taosd running"
+        cd ${WK}
-    killall -9 gdb || echo "no gdb running"
+        git reset --hard
-    killall -9 python3.8 || echo "no python program running"
+        git fetch || git fetch
         cd ${WKC}
+        git reset --hard
+        git fetch || git fetch
     '''
     script {
         if (env.CHANGE_TARGET == 'master') {
             sh '''
+                cd ${WK}
+                git checkout master
                 cd ${WKC}
                 git checkout master
             '''
-        }
+        } else if(env.CHANGE_TARGET == '2.0') {
-      else if(env.CHANGE_TARGET == '2.0'){
             sh '''
+                cd ${WK}
+                git checkout 2.0
                 cd ${WKC}
                 git checkout 2.0
             '''
-      }
+        } else if(env.CHANGE_TARGET == '3.0') {
-      else if(env.CHANGE_TARGET == '3.0'){
             sh '''
+                cd ${WK}
+                git checkout 3.0
                 cd ${WKC}
                 git checkout 3.0
-        [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../..
             '''
-      }
+        } else {
-      else{
             sh '''
+                cd ${WK}
+                git checkout develop
                 cd ${WKC}
                 git checkout develop
             '''
         }
     }
+    if (env.CHANGE_URL =~ /\/TDengine\//) {
         sh '''
             cd ${WKC}
             git pull >/dev/null
             git fetch origin +refs/pull/${CHANGE_ID}/merge
             git checkout -qf FETCH_HEAD
+            git log -5
+            cd ${WK}
+            git pull >/dev/null
+            git log -5
+        '''
+    } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
+        sh '''
+            cd ${WK}
+            git pull >/dev/null
+            git fetch origin +refs/pull/${CHANGE_ID}/merge
+            git checkout -qf FETCH_HEAD
+            git log -5
+            cd ${WKC}
+            git pull >/dev/null
+            git log -5
+        '''
+    } else {
+        sh '''
+            echo "unmatched reposiotry ${CHANGE_URL}"
+        '''
+    }
+    sh '''
+        cd ${WKC}
         git submodule update --init --recursive
     '''
     sh '''
-    cd ${WKC}
+        cd ${WKPY}
-    export TZ=Asia/Harbin
+        git reset --hard
-    date
+        git pull
-    rm -rf debug
+    '''
+    return 1
+}
+def pre_test_win(){
+    bat '''
+        hostname
+        date /t
+        time /t
+        taskkill /f /t /im python.exe
+        taskkill /f /t /im bash.exe
+        rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug
+        exit 0
+    '''
+    bat '''
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        git reset --hard
+        git fetch || git fetch
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        git reset --hard
+        git fetch || git fetch
+        git checkout -f
+    '''
+    script {
+        if (env.CHANGE_TARGET == 'master') {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout master
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git checkout master
+            '''
+        } else if(env.CHANGE_TARGET == '2.0') {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout 2.0
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git checkout 2.0
+            '''
+        } else if(env.CHANGE_TARGET == '3.0') {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout 3.0
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git checkout 3.0
+            '''
+        } else {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout develop
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git checkout develop
+            '''
+        }
+    }
+    script {
+        if (env.CHANGE_URL =~ /\/TDengine\//) {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git pull
+                git log -5
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git pull
+                git fetch origin +refs/pull/${CHANGE_ID}/merge
+                git checkout -qf FETCH_HEAD
+                git log -5
+            '''
+        } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git pull
+                git fetch origin +refs/pull/${CHANGE_ID}/merge
+                git checkout -qf FETCH_HEAD
+                git log -5
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git pull
+                git log -5
+            '''
+        } else {
+            sh '''
+                echo "unmatched reposiotry ${CHANGE_URL}"
+            '''
+        }
+    }
+    bat '''
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        git submodule update --init --recursive
+    '''
+}
+def pre_test_build_win() {
+    bat '''
+        echo "building ..."
+        time /t
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
         mkdir debug
         cd debug
-    cmake .. > /dev/null
+        call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64
-    make -j4> /dev/null
+        set CL=/MP8
+        cmake .. -G "NMake Makefiles JOM"
+        jom -j 4 || exit 8
+        time /t
     '''
     return 1
 }
@@ -95,35 +218,44 @@ pipeline {
     agent none
     options { skipDefaultCheckout() } 
     environment{
+        WKDIR = '/var/lib/jenkins/workspace'
         WK = '/var/lib/jenkins/workspace/TDinternal'
-      WKC= '/var/lib/jenkins/workspace/TDengine'
+        WKC = '/var/lib/jenkins/workspace/TDinternal/community'
+        WKPY = '/var/lib/jenkins/workspace/taos-connector-python'
     }
     stages {
-      stage('pre_build'){
+        stage('run test') {
+            parallel {
+                stage('linux test') {
                     agent{label " slave3_0 || slave15 || slave16 || slave17 "}
                     options { skipDefaultCheckout() } 
                     when {
                         changeRequest()
                     }
                     steps {
-            script{
+                        timeout(time: 40, unit: 'MINUTES'){
-              abort_previous()
-              abortPreviousBuilds()
-            }
-            timeout(time: 45, unit: 'MINUTES'){
                             pre_test()
+                            script {
                                 sh '''
-              cd ${WKC}/tests
+                                    cd ${WKC}/tests/parallel_test
-              ./test-all.sh b1fq
+                                    date
+                                    time ./container_build.sh -w ${WKDIR} -t 8 -e
+                                    rm -f /tmp/cases.task
+                                    ./collect_cases.sh -e
                                 '''
                                 sh '''
-              cd ${WKC}/debug
+                                    cd ${WKC}/tests/parallel_test
-              ctest
+                                    export DEFAULT_RETRY_TIME=2
+                                    date
+                                    timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME} -l ${WKDIR}/log -o 480
                                 '''
                             }
                         }
                     }
                 }
+            }
+        }
+    }
     post {  
         success {
             emailext (

cmake/addr2line_CMakeLists.txt.in

@@ -0,0 +1,12 @@
+
+# addr2line
+ExternalProject_Add(addr2line
+        GIT_REPOSITORY https://github.com/davea42/libdwarf-addr2line.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/addr2line"
+        BINARY_DIR "${TD_CONTRIB_DIR}/addr2line"
+        CONFIGURE_COMMAND "" 
+        BUILD_COMMAND ""  
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

cmake/cmake.define

@@ -1,5 +1,9 @@
 cmake_minimum_required(VERSION 3.16)
 
+set(CMAKE_VERBOSE_MAKEFILE OFF)
+
+SET(BUILD_SHARED_LIBS "OFF")
+
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
@@ -42,28 +46,44 @@ ENDIF ()
 
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+    SET(COMMON_FLAGS "/W3 /D_WIN32")
-    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
+    # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+    #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    # ENDIF ()
 
-    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
-    ENDIF ()
 
 ELSE ()
+    IF (${COVER} MATCHES "true")
+        MESSAGE(STATUS "Test coverage mode, add extra flags")
+        SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+        SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+    ENDIF ()
+
     IF (${SANITIZER} MATCHES "true")
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3")
     ENDIF ()
 
     MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
     IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
         ADD_DEFINITIONS("-D_TD_ARM_")
     ELSE ()
-        ADD_DEFINITIONS("-msse4.2 -mfma")
+        ADD_DEFINITIONS("-msse4.2")
+        IF("${FMA_SUPPORT}" MATCHES "true")
+            MESSAGE(STATUS "turn fma function support on")
+            ADD_DEFINITIONS("-mfma")
+        ELSE ()
+            MESSAGE(STATUS "turn fma function support off")
+        ENDIF()
     ENDIF ()
 
 ENDIF ()

cmake/cmake.install

@@ -1,5 +1,5 @@
 IF (TD_LINUX)
-  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/make_install.sh")
+  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh")
   INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
   INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})")
 ELSEIF (TD_WINDOWS)

cmake/cmake.options

@@ -25,9 +25,34 @@ IF(${TD_WINDOWS})
             ON
           )
 
+    MESSAGE("build msvcregex Win32")
+    option(
+            BUILD_MSVCREGEX
+            "If build msvcregex on Windows"
+            ON
+          )
+
+    MESSAGE("build wcwidth Win32")
+    option(
+            BUILD_WCWIDTH
+            "If build wcwidth on Windows"
+            ON
+          )
+
+    MESSAGE("build wingetopt Win32")
+    option(
+            BUILD_WINGETOPT 
+                "If build wingetopt on Windows"
+            ON
+        )
+
 ENDIF ()
 
-IF(${TD_LINUX} MATCHES TRUE)
+option(
+    BUILD_ADDR2LINE
+    "If build addr2line"
+    OFF
+    )
 
 option(
     BUILD_TEST 
@@ -35,8 +60,6 @@ option(
     ON
 )
 
-ENDIF ()
-
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 
@@ -55,6 +78,12 @@ option(
     OFF
 )
 
+option(
+    BUILD_WITH_BDB
+    "If build with BDB" 
+    OFF
+)
+
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 

cmake/libdwarf_CMakeLists.txt.in

@@ -0,0 +1,12 @@
+
+# libdwarf
+ExternalProject_Add(libdwarf
+        GIT_REPOSITORY https://github.com/davea42/libdwarf-code.git
+        GIT_TAG libdwarf-0.3.1
+        SOURCE_DIR "${TD_CONTRIB_DIR}/libdwarf"
+        BINARY_DIR "${TD_CONTRIB_DIR}/libdwarf"
+        CONFIGURE_COMMAND "" 
+        BUILD_COMMAND ""  
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

cmake/msvcregex_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# msvcregex
+ExternalProject_Add(msvcregex
+        GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/pthread_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(pthread
         GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
         GIT_TAG v3.0.3.1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/wcwidth_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# wcwidth
+ExternalProject_Add(wcwidth
+        GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/wingetopt_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# wingetopt
+ExternalProject_Add(wingetopt
+        GIT_REPOSITORY https://github.com/alex85k/wingetopt.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

contrib/CMakeLists.txt

@@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
     cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
-# gnu regex
+# iconv
-if(${BUILD_GNUREGEX})
+if(${BUILD_WITH_ICONV})
-    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# msvc regex
+if(${BUILD_MSVCREGEX})
+    cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wcwidth
+if(${BUILD_WCWIDTH})
+    cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
@@ -63,9 +78,9 @@ if(${BUILD_WITH_UV})
 endif(${BUILD_WITH_UV})
 
 # bdb
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})
@@ -83,6 +98,12 @@ if(${BUILD_WITH_NURAFT})
     cat("${TD_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_NURAFT})
 
+# addr2line
+if(${BUILD_ADDR2LINE})
+    cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_ADDR2LINE})
+
 # download dependencies
 configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -95,12 +116,31 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 # ================================================================================================
 # googletest
 if(${BUILD_TEST})
-    add_subdirectory(googletest)
+    add_subdirectory(googletest EXCLUDE_FROM_ALL)
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
+    )
+    if(${TD_WINDOWS})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
+        )
+    endif(${TD_WINDOWS})
+    if(${TD_LINUX})
+        target_include_directories(
+            gtest
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
         )
+    endif(${TD_LINUX})
+    if(${TD_DARWIN})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
+        )
+    endif(${TD_DARWIN})
+    
+    
 endif(${BUILD_TEST})
 
 # cJson
@@ -109,7 +149,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
 option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
 option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
-add_subdirectory(cJson)
+add_subdirectory(cJson EXCLUDE_FROM_ALL)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
@@ -118,7 +158,7 @@ target_include_directories(
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
 # lz4
-add_subdirectory(lz4/build/cmake)
+add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
@@ -126,7 +166,7 @@ target_include_directories(
 
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
-add_subdirectory(zlib)
+add_subdirectory(zlib EXCLUDE_FROM_ALL)
 target_include_directories(
     zlibstatic
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@@ -142,7 +182,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 # leveldb
 if(${BUILD_WITH_LEVELDB})
     option(LEVELDB_BUILD_TESTS "" OFF)
-    add_subdirectory(leveldb)
+    add_subdirectory(leveldb EXCLUDE_FROM_ALL)
     target_include_directories(
         leveldb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
@@ -158,7 +198,7 @@ if(${BUILD_WITH_ROCKSDB})
     option(WITH_TOOLS "" OFF)
     option(WITH_LIBURING "" OFF)
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    add_subdirectory(rocksdb)
+    add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
     target_include_directories(
         rocksdb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
@@ -169,7 +209,7 @@ endif(${BUILD_WITH_ROCKSDB})
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
-    add_subdirectory(lucene)
+    add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
       lucene++ 
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
@@ -179,9 +219,56 @@ endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
 if(${BUILD_WITH_NURAFT})
-    add_subdirectory(nuraft)
+    add_subdirectory(nuraft EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_NURAFT})
 
+# pthread
+if(${BUILD_PTHREAD})
+    set(CMAKE_BUILD_TYPE release)
+    add_definitions(-DPTW32_STATIC_LIB)
+    add_subdirectory(pthread)
+    set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
+    add_library(pthread STATIC IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+endif()
+
+# iconv
+if(${BUILD_WITH_ICONV})
+    add_library(iconv STATIC iconv/win_iconv.c)
+endif(${BUILD_WITH_ICONV})
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
+endif(${BUILD_WINGETOPT})
+
+# msvcregex
+if(${BUILD_MSVCREGEX})
+    add_library(msvcregex STATIC "")
+    target_sources(msvcregex
+        PRIVATE "msvcregex/regex.c"
+    )
+    target_include_directories(msvcregex
+        PRIVATE "msvcregex"
+    )
+    target_link_libraries(msvcregex 
+        INTERFACE Shell32
+    )
+    SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
+endif(${BUILD_MSVCREGEX})
+
+# msvcregex
+if(${BUILD_WCWIDTH})
+    add_library(wcwidth STATIC "")
+    target_sources(wcwidth
+        PRIVATE "wcwidth/wcwidth.c"
+    )
+    target_include_directories(wcwidth
+        PRIVATE "wcwidth"
+    )
+    SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
+endif(${BUILD_WCWIDTH})
+
 # CRAFT
 if(${BUILD_WITH_CRAFT})
     add_library(craft STATIC IMPORTED GLOBAL)
@@ -212,7 +299,7 @@ if(${BUILD_WITH_UV})
         MESSAGE("Windows need set no-sign-compare")
         add_compile_options(-Wno-sign-compare)
     endif ()
-    add_subdirectory(libuv)
+    add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})
 
 # BDB
@@ -238,16 +325,61 @@ if(${BUILD_WITH_SQLITE})
     target_link_libraries(sqlite 
         INTERFACE m 
         INTERFACE pthread 
+    )
+    if(NOT TD_WINDOWS)
+        target_link_libraries(sqlite 
             INTERFACE dl
         )
+    endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
-# pthread
+# addr2line
+if(${BUILD_ADDR2LINE})
+    check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H) 
+    check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H ) 
+    check_include_file( "inttypes.h"      HAVE_INTTYPES_H   ) 
+    check_include_file( "stddef.h"        HAVE_STDDEF_H   ) 
+    check_include_file( "stdlib.h"        HAVE_STDLIB_H   ) 
+    check_include_file( "string.h"        HAVE_STRING_H   ) 
+    check_include_file( "memory.h"        HAVE_MEMORY_H   ) 
+    check_include_file( "strings.h"       HAVE_STRINGS_H  ) 
+    check_include_file( "stdint.h"        HAVE_STDINT_H   )
+    check_include_file( "unistd.h"        HAVE_UNISTD_H   )
+    check_include_file( "sgidefs.h"       HAVE_SGIDEFS_H  )
+    check_include_file( "stdafx.h"        HAVE_STDAFX_H   )
+    check_include_file( "elf.h"           HAVE_ELF_H      ) 
+    check_include_file( "libelf.h"        HAVE_LIBELF_H   ) 
+    check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) 
+    check_include_file( "alloca.h"        HAVE_ALLOCA_H   )
+    check_include_file( "elfaccess.h"     HAVE_ELFACCESS_H)
+    check_include_file( "sys/elf_386.h"   HAVE_SYS_ELF_386_H  )
+    check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H)
+    check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
+    check_include_file( "sys/ia64/elf.h"  HAVE_SYS_IA64_ELF_H )
+    set(VERSION 0.3.1)
+    set(PACKAGE_VERSION "\"${VERSION}\"") 
+    configure_file(libdwarf/cmake/config.h.cmake config.h)
+    file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
+    add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
+    set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf")
+    if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H)
+        target_link_libraries(libdwarf PUBLIC libelf)
+    endif()
+    target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR})
+    file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT)
+    string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}")
+    add_library(addr2line STATIC "addr2line/addr2line.c")
+    target_link_libraries(addr2line PUBLIC libdwarf dl z)
+    target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
+endif(${BUILD_ADDR2LINE})
 
 
 # ================================================================================================
 # Build test
 # ================================================================================================
 if(${BUILD_DEPENDENCY_TESTS})
-    add_subdirectory(test)
+    add_subdirectory(test EXCLUDE_FROM_ALL)
 endif(${BUILD_DEPENDENCY_TESTS})

contrib/test/CMakeLists.txt

@@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
-#if(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
-    #add_subdirectory(bdb)
+    add_subdirectory(bdb)
-#endif(${BUILD_WITH_BDB})
+endif(${BUILD_WITH_BDB})
 
 if(${BUILD_WITH_SQLITE})
     add_subdirectory(sqlite)

contrib/test/craft/CMakeLists.txt

@@ -1,2 +1,9 @@
 add_executable(simulate_vnode "simulate_vnode.c")
 target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
+if(${BUILD_WINGETOPT})
+    target_link_libraries(simulate_vnode PUBLIC wingetopt)
+    target_include_directories(
+        simulate_vnode
+        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+    )
+endif()

contrib/test/tdev/src/main.c

@@ -6,43 +6,39 @@
 #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
 #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
 
-#define tPutA(buf, val)              \
+static inline void tPutA(void **buf, uint64_t val) {
-  ({                                 \
+  memcpy(buf, &val, sizeof(val));
-    memcpy(buf, &val, sizeof(val));  \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val)); \
+}
-  })
 
-#define tPutB(buf, val)                         \
+static inline void tPutB(void **buf, uint64_t val) {
-  ({                                            \
+  ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
-    ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
+  ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
-    ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
+  ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
-    ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
+  ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
-    ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
+  ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
-    ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
+  ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
-    ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
+  ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
-    ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;  \
+  ((uint8_t *)buf)[0] = (val)&0xff;
-    ((uint8_t *)buf)[0] = (val)&0xff;           \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val));            \
+}
-  })
 
-#define tPutC(buf, val)                \
+static inline void tPutC(void **buf, uint64_t val) {
-  ({                                   \
+  if (buf) {
-    if (buf) {                         \
+    ((uint64_t *)buf)[0] = (val);
-      ((uint64_t *)buf)[0] = (val);    \
+    POINTER_SHIFT(buf, sizeof(val));
-      POINTER_SHIFT(buf, sizeof(val)); \
+  }
-    }                                  \
+  *buf = NULL;
-    NULL;                                 \
+}
-  })
 
-#define tPutD(buf, val)                        \
+static inline void tPutD(void **buf, uint64_t val) {
-  ({                                           \
+  uint64_t tmp = val;
-    uint64_t tmp = val;                        \
+  for (size_t i = 0; i < sizeof(val); i++) {
-    for (size_t i = 0; i < sizeof(val); i++) { \
+    ((uint8_t *)buf)[i] = tmp & 0xff;
-      ((uint8_t *)buf)[i] = tmp & 0xff;        \
+    tmp >>= 8;
-      tmp >>= 8;                               \
+  }
-    }                                          \
+  *buf = POINTER_SHIFT(buf, sizeof(val));
-    POINTER_SHIFT(buf, sizeof(val));           \
+}
-  })
 
 static inline void tPutE(void **buf, uint64_t val) {
   if (buf) {
@@ -61,7 +57,7 @@ static void func(T t) {
   switch (t) {
     case A:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutA(pBuf, val);
+        tPutA(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -69,7 +65,7 @@ static void func(T t) {
       break;
     case B:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutB(pBuf, val);
+        tPutB(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -77,7 +73,7 @@ static void func(T t) {
       break;
     case C:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutC(pBuf, val);
+        tPutC(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -85,7 +81,7 @@ static void func(T t) {
       break;
     case D:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutD(pBuf, val);
+        tPutD(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }

example/CMakeLists.txt

@@ -30,6 +30,7 @@ target_link_libraries(demoapi
 )
 
 target_include_directories(tmq
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
@@ -39,6 +40,7 @@ target_include_directories(tstream
 
 target_include_directories(demoapi
     PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

example/src/demoapi.c

@@ -4,10 +4,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
+// #include <unistd.h>
 #include <inttypes.h>
+#ifndef WINDOWS
 #include <argp.h>
-
+#endif
+#include "osSleep.h"
 #include "taos.h"
 
 #define debugPrint(fmt, ...) \
@@ -32,6 +34,7 @@
 int64_t g_num_of_tb = 2;
 int64_t g_num_of_rec = 3;
 
+#ifndef WINDOWS
 static struct argp_option options[] = {
     {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
     {"records", 'n', "NUMBER", 0,
@@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 }
 
 static struct argp argp = {options, parse_opt, "", ""};
-
+#endif
 static void prepare_data(TAOS* taos) {
     TAOS_RES *res;
     res = taos_query(taos, "drop database if exists test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
 
     res = taos_query(taos, "create database test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
     taos_select_db(taos, "test");
 
     char command[1024] = {0};
@@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) {
     taos_free_result(res);
 
     for (int64_t i = 0; i < g_num_of_tb; i ++) {
+        // sprintf(command, "create table t%"PRId64" using meters "
+        //         "tags(%"PRId64", '%s', '%s', '%s');",
+        //         i, i, (i%2)?"beijing":"shanghai",
+        //         (i%2)?"朝阳区":"黄浦区",
+        //         (i%2)?"长安街":"中山路");
         sprintf(command, "create table t%"PRId64" using meters "
-                "tags(%"PRId64", '%s', '%s', '%s');",
-                i, i, (i%2)?"beijing":"shanghai",
-                (i%2)?"朝阳区":"黄浦区",
-                (i%2)?"长安街":"中山路");
-/*        sprintf(command, "create table t%"PRId64" using meters "
                 "tags(%"PRId64", '%s', '%s', '%s');",
                 i, i,
                 (i%2)?"beijing":"shanghai",
                 (i%2)?"chaoyang":"huangpu",
                 (i%2?"changan street":"jianguo rd"));
-                */
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             okPrint("t%" PRId64 " created\n", i);
@@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) {
                     "'%c%d', '%s%c%d', '%c%d')",
                     i, 1650000000000+j, (float)j, j,
                     'a'+(int)j%25, rand(),
-                    "涛思", 'z' - (int)j%25, rand(),
+                    // "涛思", 'z' - (int)j%25, rand(),
+                    "TAOS", 'z' - (int)j%25, rand(),
                     'b' - (int)j%25, rand()
                     );
             res = taos_query(taos,  command);
@@ -176,7 +179,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
     }
 
     if (block) {
-        warnPrint("%s", "call taos_fetch_block()\n");
+        warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
             int *lengths = taos_fetch_lengths(res);
@@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %"PRId64"\n",
                                             f, c,
-                                            *(int64_t*)(row[f]+c*sizeof(int64_t)));
+                                            *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
                                 }
                                 break;
 
@@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %d\n",
                                             f, c,
-                                            *(int32_t*)(row[f]+c*sizeof(int32_t)));
+                                            *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
                                 }
                                 break;
 
@@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %f\n",
                                             f, c,
-                                            *(float*)(row[f]+c*sizeof(float)));
+                                            *(float*)((char*)(row[f])+c*sizeof(float)));
                                 }
                                 break;
 
@@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     if (offsets) {
                         for (int c = 0; c < rows; c++) {
                             if (offsets[c] != -1) {
-                                int length = *(int16_t*)(row[f] + offsets[c]);
+                                int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
                                 char *buf = calloc(1, length + 1);
-                                strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
+                                strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
                                 printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
                                         c, f, offsets[c], length, buf);
                                 free(buf);
@@ -314,8 +317,9 @@ int main(int argc, char *argv[]) {
     const char* host = "127.0.0.1";
     const char* user = "root";
     const char* passwd = "taosdata";
-
+#ifndef WINDOWS
     argp_parse(&argp, argc, argv, 0, 0, NULL);
+#endif
     TAOS* taos = taos_connect(host, user, passwd, "", 0);
     if (taos == NULL) {
         printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));

example/src/tmq.c

@@ -22,6 +22,7 @@
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
   char buf[1024];
+  /*memset(buf, 0, 1024);*/
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
   while (1) {
@@ -31,6 +32,11 @@ static void msg_process(TAOS_RES* msg) {
     int32_t     numOfFields = taos_field_count(msg);
     taos_print_row(buf, row, fields, numOfFields);
     printf("%s\n", buf);
+
+    const char* tbName = tmq_get_table_name(msg);
+    if (tbName) {
+      printf("from tb: %s\n", tbName);
+    }
   }
 }
 
@@ -55,7 +61,7 @@ int32_t init_env() {
   taos_free_result(pRes);
 
   pRes =
-      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
+      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -100,9 +106,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*const char* sql = "select * from tu1";*/
+  pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
-  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
-  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -141,7 +146,7 @@ int32_t create_topic() {
 }
 
 void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
-  printf("commit %d\n", resp);
+  printf("commit %d tmq %p offsets %p param %p\n", resp, tmq, offsets, param);
 }
 
 tmq_t* build_consumer() {
@@ -160,15 +165,18 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "group.id", "tg2");
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  tmq_conf_set(conf, "td.connect.db", "abc1");
+  /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
+  tmq_conf_set(conf, "msg.with.table.name", "true");
-  tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
+  tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+  assert(tmq);
   return tmq;
 }
 
 tmq_list_t* build_topic_list() {
   tmq_list_t* topic_list = tmq_list_new();
   tmq_list_append(topic_list, "topic_ctb_column");
+  /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/
   return topic_list;
 }
 
@@ -183,12 +191,12 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
-    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
     if (tmqmessage) {
       cnt++;
       /*printf("get data\n");*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
       /*} else {*/
       /*break;*/
     }
@@ -214,13 +222,25 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     return;
   }
 
+  tmq_list_t* subList = NULL;
+  tmq_subscription(tmq, &subList);
+  char**  subTopics = tmq_list_to_c_array(subList);
+  int32_t sz = tmq_list_get_size(subList);
+  printf("subscribed topics: ");
+  for (int32_t i = 0; i < sz; i++) {
+    printf("%s, ", subTopics[i]);
+  }
+  printf("\n");
+  tmq_list_destroy(subList);
+
   while (running) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
       msg_process(tmqmessage);
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
 
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
+      tmq_commit(tmq, NULL, 1);
+      /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
 
@@ -248,7 +268,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
       batchCnt++;
       /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
     } else {
       break;
     }
@@ -265,10 +285,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 int main(int argc, char* argv[]) {
-  int code;
   if (argc > 1) {
     printf("env init\n");
-    code = init_env();
+    if (init_env() < 0) {
+      return -1;
+    }
     create_topic();
   }
   tmq_t*      tmq = build_consumer();

example/src/tstream.c

@@ -25,7 +25,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -78,11 +78,15 @@ int32_t create_stream() {
   taos_free_result(pRes);
 
   /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
-  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
-  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
+  pRes = taos_query(
+      pConn,
+      "create stream stream1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
+      "from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);

examples/c/CMakeLists.txt

@@ -1,7 +1,7 @@
 PROJECT(TDengine)
 
 IF (TD_LINUX)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo apitest.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
@@ -13,7 +13,7 @@ IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()
 IF (TD_DARWIN)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo demo.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)

examples/c/asyncdemo.c

@@ -45,7 +45,7 @@ typedef struct {
 
 void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
 void taos_select_call_back(void *param, TAOS_RES *tres, int code);
-void taos_error(TAOS *taos);
+void shellPrintError(TAOS *taos);
 
 static void queryDB(TAOS *taos, char *command) {
   int i;
@@ -102,7 +102,7 @@ int main(int argc, char *argv[])
 
   taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
   if (taos == NULL)
-    taos_error(taos);
+    shellPrintError(taos);
 
   printf("success to connect to server\n");
 
@@ -193,7 +193,7 @@ int main(int argc, char *argv[])
   return 0;
 }
 
-void taos_error(TAOS *con)
+void shellPrintError(TAOS *con)
 {
   fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
   taos_close(con);

include/client/taos.h

@@ -27,9 +27,6 @@ typedef void   TAOS;
 typedef void   TAOS_STMT;
 typedef void   TAOS_RES;
 typedef void **TAOS_ROW;
-#if 0
-typedef void   TAOS_STREAM;
-#endif
 typedef void   TAOS_SUB;
 
 // Data type definition
@@ -88,34 +85,14 @@ typedef struct taosField {
   int32_t bytes;
 } TAOS_FIELD;
 
+#ifdef WINDOWS
+#define DLL_EXPORT __declspec(dllexport)
+#else
 #define DLL_EXPORT
+#endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-typedef struct TAOS_BIND {
-  int        buffer_type;
-  void      *buffer;
-  uintptr_t  buffer_length;  // unused
-  uintptr_t *length;
-  int       *is_null;
-
-  int  is_unsigned;  // unused
-  int *error;        // unused
-  union {
-    int64_t        ts;
-    int8_t         b;
-    int8_t         v1;
-    int16_t        v2;
-    int32_t        v4;
-    int64_t        v8;
-    float          f4;
-    double         f8;
-    unsigned char *bin;
-    char          *nchar;
-  } u;
-  unsigned int allocated;
-} TAOS_BIND;
-
 typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
   void     *buffer;
@@ -144,6 +121,7 @@ typedef struct setConfRet {
 DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
+DLL_EXPORT int        taos_init(void);
 DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
@@ -154,14 +132,14 @@ const char *taos_data_type(int type);
 
 DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
 DLL_EXPORT int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
 DLL_EXPORT int        taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
 DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
 DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
 DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
 DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
 DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
 DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
 DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
@@ -170,6 +148,7 @@ DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@@ -215,12 +194,6 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-#if 0
-DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
-                                         int64_t stime, void *param, void (*callback)(void *));
-DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
-#endif
-
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
 
@@ -239,37 +212,34 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
 
 typedef struct tmq_conf_t tmq_conf_t;
 typedef struct tmq_list_t tmq_list_t;
-// typedef struct tmq_message_t tmq_message_t;
 
 typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param));
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
+DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
-#if 1
+DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
-DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
-#endif
-
-DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
-DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
+
+DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
-#if 0
-DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups);
-DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups);
-#endif
 DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
+DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
+DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 #endif
+
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {
@@ -283,34 +253,21 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
 DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
-DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
+DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
-
-#if 0
-// temporary used function for demo only
-void    tmqShowMsg(tmq_message_t *tmq_message);
-int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
-#endif
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
-DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
+DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res);
+
 #if 0
-DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
-DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
-DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
 #endif
-DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
-/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
-#if 0
-DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
-#endif
-
-DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
 
 /* ------------------------------ TMQ END -------------------------------- */
+
 #if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 #endif

include/common/systable.h

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#ifndef TDENGINE_SYSTABLE_H
+#define TDENGINE_SYSTABLE_H
+
+#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
+#define TSDB_INS_TABLE_DNODES                 "dnodes"
+#define TSDB_INS_TABLE_MNODES                 "mnodes"
+#define TSDB_INS_TABLE_MODULES                "modules"
+#define TSDB_INS_TABLE_QNODES                 "qnodes"
+#define TSDB_INS_TABLE_BNODES                 "bnodes"
+#define TSDB_INS_TABLE_SNODES                 "snodes"
+#define TSDB_INS_TABLE_CLUSTER                "cluster"
+#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
+#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
+#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
+#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
+#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
+#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
+#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
+#define TSDB_INS_TABLE_USER_USERS             "user_users"
+#define TSDB_INS_TABLE_LICENCES               "grants"
+#define TSDB_INS_TABLE_VGROUPS                "vgroups"
+#define TSDB_INS_TABLE_VNODES                 "vnodes"
+#define TSDB_INS_TABLE_CONFIGS                "configs"
+
+#define TSDB_PERFORMANCE_SCHEMA_DB     "performance_schema"
+#define TSDB_PERFS_TABLE_SMAS          "smas"
+#define TSDB_PERFS_TABLE_CONNECTIONS   "connections"
+#define TSDB_PERFS_TABLE_QUERIES       "queries"
+#define TSDB_PERFS_TABLE_TOPICS        "topics"
+#define TSDB_PERFS_TABLE_CONSUMERS     "consumers"
+#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
+#define TSDB_PERFS_TABLE_OFFSETS       "offsets"
+#define TSDB_PERFS_TABLE_TRANS         "trans"
+#define TSDB_PERFS_TABLE_STREAMS       "streams"
+
+typedef struct SSysDbTableSchema {
+  const char*   name;
+  const int32_t type;
+  const int32_t bytes;
+} SSysDbTableSchema;
+
+typedef struct SSysTableMeta {
+  const char*              name;
+  const SSysDbTableSchema* schema;
+  const int32_t            colNum;
+} SSysTableMeta;
+
+void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size);
+void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SYSTABLE_H

include/common/taosdef.h

@@ -28,6 +28,7 @@ typedef int64_t tb_uid_t;
 #define TSWINDOW_INITIALIZER       ((STimeWindow){INT64_MIN, INT64_MAX})
 #define TSWINDOW_DESC_INITIALIZER  ((STimeWindow){INT64_MAX, INT64_MIN})
 #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
+#define TSWINDOW_IS_EQUAL(t1, t2)  (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
 
 typedef enum {
   TSDB_SUPER_TABLE = 1,   // super table
@@ -36,7 +37,8 @@ typedef enum {
   TSDB_STREAM_TABLE = 4,  // table created from stream computing
   TSDB_TEMP_TABLE = 5,    // temp table created by nest query
   TSDB_SYSTEM_TABLE = 6,
-  TSDB_TABLE_MAX    = 7
+  TSDB_TSMA_TABLE = 7,  // time-range-wise sma
+  TSDB_TABLE_MAX = 8
 } ETableType;
 
 typedef enum {
@@ -71,7 +73,6 @@ typedef enum {
   TSDB_SMA_STAT_DROPPED = 2,   // sma dropped
 } ETsdbSmaStat;                // bit operation
 
-
 typedef enum {
   TSDB_SMA_TYPE_BLOCK = 0,       // Block-wise SMA
   TSDB_SMA_TYPE_TIME_RANGE = 1,  // Time-range-wise SMA
@@ -79,16 +80,18 @@ typedef enum {
 } ETsdbSmaType;
 
 typedef enum {
-  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
+  TSDB_RETENTION_L0 = 0,
-  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
+  TSDB_RETENTION_L1 = 1,
-} ETsdbBSmaType;
+  TSDB_RETENTION_L2 = 2,
-
+  TSDB_RETENTION_MAX = 3
-#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
+} ERetentionLevel;
 
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP 11
 
+#undef TD_DEBUG_PRINT_ROW
+
 #ifdef __cplusplus
 }
 #endif

include/common/tcommon.h

@@ -45,6 +45,13 @@ enum {
   STREAM_TRIGGER__BY_EVENT_TIME,
 };
 
+typedef enum EStreamType {
+  STREAM_NORMAL = 1,
+  STREAM_INVERT,
+  STREAM_REPROCESS,
+  STREAM_INVALID,
+} EStreamType;
+
 typedef struct {
   uint32_t  numOfTables;
   SArray*   pGroupList;
@@ -65,18 +72,17 @@ typedef struct SDataBlockInfo {
   STimeWindow window;
   int32_t     rows;
   int32_t     rowSize;
-  union {
+  int64_t     uid;      // the uid of table, from which current data block comes
-    int64_t uid;        // from which table of uid, comes from this data block
+  int64_t     blockId;  // block id, generated by physical planner
-    int64_t blockId;
-  };
   uint64_t    groupId;  // no need to serialize
   int16_t     numOfCols;
   int16_t     hasVarCol;
-  int16_t     capacity;
+  int32_t     capacity;
+  EStreamType type;
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
-  SColumnDataAgg* pBlockAgg;
+  SColumnDataAgg** pBlockAgg;
   SArray*          pDataBlock;  // SArray<SColumnInfoData>
   SDataBlockInfo   info;
 } SSDataBlock;
@@ -99,6 +105,15 @@ typedef struct SColumnInfoData {
   };
 } SColumnInfoData;
 
+typedef struct SQueryTableDataCond {
+  STimeWindow  twindow;
+  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} SQueryTableDataCond;
+
 void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
@@ -108,8 +123,6 @@ void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
 void    colDataDestroy(SColumnInfoData* pColData);
 
 static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
-  // WARNING: do not use info.numOfCols,
-  // sometimes info.numOfCols != array size
   int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
   for (int32_t i = 0; i < numOfOutput; ++i) {
     SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
@@ -122,59 +135,6 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
 
 static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
 
-static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
-  int32_t tlen = 0;
-  int32_t sz = 0;
-  // tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
-  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
-  tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return tlen;
-  tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema);
-  if (pRsp->pBlockData) {
-    sz = taosArrayGetSize(pRsp->pBlockData);
-  }
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-    tlen += tEncodeDataBlock(buf, pBlock);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
-  int32_t sz;
-  // buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
-  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
-  buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return buf;
-  pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-  if (pRsp->schema == NULL) return NULL;
-  buf = taosDecodeSSchemaWrapper(buf, pRsp->schema);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock block = {0};
-    tDecodeDataBlock(buf, &block);
-    taosArrayPush(pRsp->pBlockData, &block);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
-  if (pRsp->schema) {
-    if (pRsp->schema->nCols) {
-      taosMemoryFreeClear(pRsp->schema->pSchema);
-    }
-    taosMemoryFree(pRsp->schema);
-  }
-  taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
-  pRsp->pBlockData = NULL;
-}
-
 //======================================================================================================================
 // the following structure shared by parser and executor
 typedef struct SColumn {
@@ -229,7 +189,6 @@ typedef struct SResSchame {
   char    name[TSDB_COL_NAME_LEN];
 } SResSchema;
 
-// TODO move away to executor.h
 typedef struct SExprBasicInfo {
   SResSchema   resSchema;
   int16_t      numOfParams;  // argument value of each function
@@ -241,6 +200,20 @@ typedef struct SExprInfo {
   struct tExprNode*     pExpr;
 } SExprInfo;
 
+typedef struct {
+  const char*   key;
+  int32_t       keyLen;
+  uint8_t       type;
+  union{
+    const char* value;
+    int64_t     i;
+    uint64_t    u;
+    double      d;
+    float       f;
+  };
+  int32_t       length;
+} SSmlKv;
+
 #define QUERY_ASC_FORWARD_STEP  1
 #define QUERY_DESC_FORWARD_STEP -1
 

include/common/tdata.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDATA_H_
+#define _TD_TDATA_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+typedef struct STaosData TDATA, tdata_t;
+
+typedef enum {
+  TAOS_META_STABLE_DATA = 0,  // super table meta
+  TAOS_META_TABLE_DATA,       // non-super table meta
+  TAOS_TS_ROW_DATA,           // row time-series data
+  TAOS_TS_COL_DATA,           // col time-series data
+  TAOS_DATA_MAX
+} ETaosDataT;
+
+struct STaosData {
+  ETaosDataT type;
+  uint32_t   nPayload;
+  uint8_t   *pPayload;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDATA_H_*/

include/common/tdatablock.h

@@ -54,6 +54,11 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataSetNotNull_f(bm_, r_)                  \
+  do {                                                \
+    BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
+  } while (0)
+
 #define colDataIsNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] == -1)
 #define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1)
 
@@ -64,8 +69,7 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
 // SColumnInfoData, rowNumber
 #define colDataGetData(p1_, r_) \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
-                                        :  colDataGetNumData(p1_, r_))
 
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
   if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
@@ -183,8 +187,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
 }
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
-                        uint32_t numOfRow2);
+                        const SColumnInfoData* pSource, uint32_t numOfRow2);
 int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
 
@@ -225,6 +229,11 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
 
 void blockDebugShowData(const SArray* dataBlocks);
 
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
+                                    tb_uid_t uid, tb_uid_t suid);
+
+SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, int32_t vgId);
+
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
   return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
 }
@@ -238,10 +247,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32
 
 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                                              int8_t needCompress) {
-  int32_t* colSizes = (int32_t*)data;
+  int32_t* actualLen = (int32_t*)data;
+  data += sizeof(int32_t);
 
+  uint64_t* groupId = (uint64_t*)data;
+  data += sizeof(uint64_t);
+
+  int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
-  *dataLen = (numOfCols * sizeof(int32_t));
+
+  *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
@@ -273,6 +288,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
 
     colSizes[col] = htonl(colSizes[col]);
   }
+
+  *actualLen = *dataLen;
+  *groupId = pBlock->info.groupId;
 }
 
 #ifdef __cplusplus

include/common/tdataformat.h

@@ -18,6 +18,7 @@
 
 #include "os.h"
 #include "talgo.h"
+#include "tencode.h"
 #include "ttypes.h"
 #include "tutil.h"
 
@@ -25,6 +26,104 @@
 extern "C" {
 #endif
 
+typedef struct SSchema       SSchema;
+typedef struct STColumn      STColumn;
+typedef struct STSchema      STSchema;
+typedef struct SColVal       SColVal;
+typedef struct STSRow2       STSRow2;
+typedef struct STSRowBuilder STSRowBuilder;
+typedef struct STagVal       STagVal;
+typedef struct STag          STag;
+
+// STSchema
+int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
+void    tTSchemaDestroy(STSchema *pTSchema);
+
+// SColVal
+#define ColValNONE               ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL})
+#define ColValNULL               ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL})
+#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)})
+
+// STSRow2
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow);
+void    tTSRowFree(STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+
+// STSRowBuilder
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema);
+void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
+void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
+int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
+
+// STag
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag);
+void    tTagFree(STag *pTag);
+void    tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData);
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag);
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag);
+
+// STRUCT =================
+struct STColumn {
+  col_id_t colId;
+  int8_t   type;
+  int8_t   flags;
+  int32_t  bytes;
+  int32_t  offset;
+};
+
+struct STSchema {
+  int32_t  numOfCols;
+  int32_t  version;
+  int32_t  flen;
+  int32_t  vlen;
+  int32_t  tlen;
+  STColumn columns[];
+};
+
+#define TSROW_HAS_NONE ((uint8_t)0x1)
+#define TSROW_HAS_NULL ((uint8_t)0x2U)
+#define TSROW_HAS_VAL  ((uint8_t)0x4U)
+#define TSROW_KV_ROW   ((uint8_t)0x10U)
+struct STSRow2 {
+  TSKEY    ts;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+struct STSRowBuilder {
+  STSchema *pTSchema;
+  int32_t   szBitMap1;
+  int32_t   szBitMap2;
+  int32_t   szKVBuf;
+  uint8_t  *pKVBuf;
+  int32_t   szTPBuf;
+  uint8_t  *pTPBuf;
+  int32_t   iCol;
+  int32_t   vlenKV;
+  int32_t   vlenTP;
+  STSRow2   row;
+};
+
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT;
+struct SColVal {
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+struct STagVal {
+  int16_t  cid;
+  int8_t   type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+#if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
 #define TD_SUPPORT_READ2
@@ -59,38 +158,20 @@ extern "C" {
   } while (0);
 
 // ----------------- TSDB COLUMN DEFINITION
-#pragma pack(push, 1)
-typedef struct {
-  col_id_t colId;        // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
-  int32_t  type : 8;     // column type
-  int32_t  bytes : 24;   // column bytes (0~16M)
-  int32_t  sma : 8;      // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
-  int32_t  offset : 24;  // point offset in STpRow after the header part.
-} STColumn;
-#pragma pack(pop)
 
 #define colType(col)   ((col)->type)
-#define colSma(col)    ((col)->sma)
+#define colFlags(col)  ((col)->flags)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
-#define colSetSma(col, s)    (colSma(col) = (s))
+#define colSetFlags(col, f)  (colFlags(col) = (f))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
 
 // ----------------- TSDB SCHEMA DEFINITION
-typedef struct {
-  int32_t      numOfCols;  // Number of columns appended
-  schema_ver_t version;    // schema version
-  uint16_t     flen;       // First part length in a STpRow after the header part
-  int32_t      vlen;       // pure value part length, excluded the overhead (bytes only)
-  int32_t      tlen;       // maximum length of a STpRow without the header part
-                           // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
-  STColumn columns[];
-} STSchema;
 
 #define schemaNCols(s)    ((s)->numOfCols)
 #define schemaVersion(s)  ((s)->version)
@@ -146,33 +227,10 @@ typedef struct {
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition
-#ifdef TD_2_0
-
-typedef uint64_t TKEY;
-
-#define TKEY_INVALID       UINT64_MAX
-#define TKEY_NULL          TKEY_INVALID
-#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
-#define TKEY_DELETE_FLAG   (((TKEY)1) << 62)
-#define TKEY_VALUE_FILTER  (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
-
-#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
-#define TKEY_IS_DELETED(tkey)  (((tkey)&TKEY_DELETE_FLAG) != 0)
-#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
-#define tdGetTKEY(key)         (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
-#define tdGetKey(tkey)         (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
-
-#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
-#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)
-
-#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
-
-#else
-
 // typedef uint64_t TKEY;
 #define TKEY TSKEY
 
@@ -192,8 +250,6 @@ typedef uint64_t TKEY;
 
 #define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
 
-#endif
-
 static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
   TSKEY lkey = key;
   if (key > MAX_TS_KEY) {
@@ -218,164 +274,12 @@ static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
   }
 }
 
-#if 0
-// ----------------- Data row structure
-
-/* A data row, the format is like below:
- * |<------------------------------------------------ len ---------------------------------->|
- * |<--     Head      -->|<---------   flen -------------->|                                 |
- * +---------------------+---------------------------------+---------------------------------+
- * | uint16_t |  int16_t |                                 |                                 |
- * +----------+----------+---------------------------------+---------------------------------+
- * |   len    | sversion |           First part            |             Second part         |
- * +----------+----------+---------------------------------+---------------------------------+
- *
- * NOTE: timestamp in this row structure is TKEY instead of TSKEY
- */
-typedef void *SDataRow;
-
-#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
-
-#define dataRowLen(r)                (*(TDRowLenT *)(r))  // 0~65535
-#define dataRowEnd(r)                POINTER_SHIFT(r, dataRowLen(r))
-#define dataRowVersion(r)            (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
-#define dataRowTuple(r)              POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowTKey(r)               (*(TKEY *)(dataRowTuple(r)))
-#define dataRowKey(r)                tdGetKey(dataRowTKey(r))
-#define dataRowSetLen(r, l)          (dataRowLen(r) = (l))
-#define dataRowSetVersion(r, v)      (dataRowVersion(r) = (v))
-#define dataRowCpy(dst, r)           memcpy((dst), (r), dataRowLen(r))
-#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
-#define dataRowDeleted(r)            TKEY_IS_DELETED(dataRowTKey(r))
-
-SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
-void     tdFreeDataRow(SDataRow row);
-void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-SDataRow tdDataRowDup(SDataRow row);
-
-// offset here not include dataRow header length
-static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
-                                           int32_t offset) {
-  assert(value != NULL);
-  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-
-  if (IS_VAR_DATA_TYPE(type)) {
-    *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-    if (isCopyVarData) {
-      memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
-    }
-    dataRowLen(row) += varDataTLen(value);
-  } else {
-    if (offset == 0) {
-      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
-      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
-      memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]);
-    } else {
-      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-    }
-  }
-
-  return 0;
-}
-
-// offset here not include dataRow header length
-static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
-  return tdAppendDataColVal(row, value, true, type, offset);
-}
-
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
-  if (IS_VAR_DATA_TYPE(type)) {
-    return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-  } else {
-    return POINTER_SHIFT(row, offset);
-  }
-}
-
-static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) {
-  return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset);
-}
-
-static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-
-  return tdGetRowDataOfCol(row, type, offset);
-}
-
-static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-
-  return isNull(tdGetRowDataOfCol(row, type, offset), type);
-}
-
-static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-  int16_t bytes = pSchema->columns[idx].bytes;
-
-  setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
-}
-
-static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src,
-                                                STSchema *pSrcSchema, int32_t srcIdx) {
-  int8_t type = pDstSchema->columns[dstIdx].type;
-  assert(type == pSrcSchema->columns[srcIdx].type);
-  void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
-  void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx);
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value;
-      pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData);
-      value = POINTER_SHIFT(src, *(VarDataOffsetT *)value);
-      memcpy(pData, value, varDataTLen(value));
-      break;
-    case TSDB_DATA_TYPE_NULL:
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_TINYINT:
-    case TSDB_DATA_TYPE_UTINYINT:
-      *(uint8_t *)pData = *(uint8_t *)value;
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_USMALLINT:
-      *(uint16_t *)pData = *(uint16_t *)value;
-      break;
-    case TSDB_DATA_TYPE_INT:
-    case TSDB_DATA_TYPE_UINT:
-      *(uint32_t *)pData = *(uint32_t *)value;
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_UBIGINT:
-      *(uint64_t *)pData = *(uint64_t *)value;
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-      SET_FLOAT_PTR(pData, value);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      SET_DOUBLE_PTR(pData, value);
-      break;
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
-      } else {
-        *(TSKEY *)pData = *(TSKEY *)value;
-      }
-      break;
-    default:
-      memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes);
-  }
-}
-#endif
 // ----------------- Data column structure
 // SDataCol arrangement: data => bitmap => dataOffset
 typedef struct SDataCol {
   int8_t          type;        // column type
   uint8_t         bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
-  uint8_t         bitmapMode : 1;  // default is 0(2 bits), otherwise 1(1 bit)
+  uint8_t         reserve : 7;
-  uint8_t         reserve : 6;
   int16_t         colId;      // column ID
   int32_t         bytes;      // column data bytes defined
   int32_t         offset;     // data offset in a SDataRow (including the header size)
@@ -387,8 +291,6 @@ typedef struct SDataCol {
   TSKEY           ts;         // only used in last NULL column
 } SDataCol;
 
-
-
 #define isAllRowsNull(pCol) ((pCol)->len == 0)
 #define isAllRowsNone(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
@@ -401,29 +303,6 @@ void   *dataColSetOffset(SDataCol *pCol, int32_t nEle);
 
 bool isNEleNull(SDataCol *pCol, int32_t nEle);
 
-#if 0
-// Get the data pointer from a column-wised data
-static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) {
-  if (isAllRowsNull(pCol)) {
-    return getNullValue(pCol->type);
-  }
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
-  } else {
-    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
-  }
-}
-
-static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
-  assert(rows > 0);
-
-  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-    return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
-  } else {
-    return TYPE_BYTES[pDataCol->type] * rows;
-  }
-}
-#endif
 typedef struct {
   col_id_t  maxCols;    // max number of columns
   col_id_t  numOfCols;  // Total number of cols
@@ -482,7 +361,8 @@ void       tdResetDataCols(SDataCols *pCols);
 int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer);
+int32_t    tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update,
+                           TDRowVerT maxVer);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|
@@ -535,7 +415,7 @@ static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) {
   }
 }
 
-static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) {
+static FORCE_INLINE void *tdGetKVRowValOfCol(const SKVRow row, int16_t colId) {
   void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
   if (ret == NULL) return NULL;
   return kvRowColVal(row, (SColIdx *)ret);
@@ -545,54 +425,6 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
   return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
 }
 
-#if 0
-// offset here not include kvRow header length
-static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
-                                         int32_t offset) {
-  assert(value != NULL);
-  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
-  SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
-  char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
-
-  pColIdx->colId = colId;
-  pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
-
-  if (IS_VAR_DATA_TYPE(type)) {
-    if (isCopyValData) {
-      memcpy(ptr, value, varDataTLen(value));
-    }
-    kvRowLen(row) += varDataTLen(value);
-  } else {
-    if (offset == 0) {
-      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
-      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
-      memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
-    } else {
-      memcpy(ptr, value, TYPE_BYTES[type]);
-    }
-    kvRowLen(row) += TYPE_BYTES[type];
-  }
-
-  return 0;
-}
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); }
-
-static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) {
-  while (*nIdx < kvRowNCols(row)) {
-    SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx);
-    if (pColIdx->colId == colId) {
-      ++(*nIdx);
-      return tdGetKvRowDataOfCol(row, pColIdx->offset);
-    } else if (pColIdx->colId > colId) {
-      return NULL;
-    } else {
-      ++(*nIdx);
-    }
-  }
-  return NULL;
-}
-#endif
 // ----------------- K-V data row builder
 typedef struct {
   int16_t  tCols;
@@ -635,165 +467,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   return 0;
 }
-#if 0
-// ----------------- SMemRow appended with tuple row structure
-/*
- * |---------|------------------------------------------------- len ---------------------------------->|
- * |<--------     Head      ------>|<---------   flen -------------->|                                 |
- * |---------+---------------------+---------------------------------+---------------------------------+
- * | uint8_t | uint16_t |  int16_t |                                 |                                 |
- * |---------+----------+----------+---------------------------------+---------------------------------+
- * |  flag   |   len    | sversion |           First part            |             Second part         |
- * +---------+----------+----------+---------------------------------+---------------------------------+
- *
- * NOTE: timestamp in this row structure is TKEY instead of TSKEY
- */
-
-// ----------------- SMemRow appended with extended K-V data row structure
-/* |--------------------|------------------------------------------------  len ---------------------------------->|
- * |<-------------     Head      ------------>|<---------   flen -------------->|                                 |
- * |--------------------+----------+--------------------------------------------+---------------------------------+
- * | uint8_t | int16_t  | uint16_t |  int16_t |                                 |                                 |
- * |---------+----------+----------+----------+---------------------------------+---------------------------------+
- * |   flag  | sversion |   len    |   ncols  |           cols index            |             data part           |
- * |---------+----------+----------+----------+---------------------------------+---------------------------------+
- */
-
-typedef void *SMemRow;
-
-#define TD_MEM_ROW_TYPE_SIZE        sizeof(uint8_t)
-#define TD_MEM_ROW_KV_VER_SIZE      sizeof(int16_t)
-#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE)
-#define TD_MEM_ROW_DATA_HEAD_SIZE   (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
-#define TD_MEM_ROW_KV_HEAD_SIZE     (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
-
-#define SMEM_ROW_DATA 0x0U   // SDataRow
-#define SMEM_ROW_KV   0x01U  // SKVRow
-
-#define KVRatioConvert (0.9f)
-
-#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
-
-#define memRowSetType(r, t)  ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
-#define isDataRowT(t)        (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
-#define isDataRow(r)         (SMEM_ROW_DATA == memRowType(r))
-#define isKvRowT(t)          (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
-#define isKvRow(r)           (SMEM_ROW_KV == memRowType(r))
-#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
-
-#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
-#define memRowKvBody(r) \
-  POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE)  // section after flag + sversion as to reuse SKVRow
-
-#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r))  //  0~65535
-#define memRowKvLen(r)   (*(TDRowLenT *)memRowKvBody(r))    //  0~65535
-
-#define memRowDataTLen(r) \
-  ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
-
-#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
-
-#define memRowLen(r)  (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
-#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
-
-static FORCE_INLINE char *memRowEnd(SMemRow row) {
-  if (isDataRow(row)) {
-    return (char *)dataRowEnd(memRowDataBody(row));
-  } else {
-    return (char *)kvRowEnd(memRowKvBody(row));
-  }
-}
-
-#define memRowDataVersion(r)     dataRowVersion(memRowDataBody(r))
-#define memRowKvVersion(r)       (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
-#define memRowVersion(r)         (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
-#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
-#define memRowTuple(r)           (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
-
-#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
-#define memRowKey(r)  (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
-#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
-#define memRowSetTKey(r, k)                 \
-  do {                                      \
-    if (isDataRow(r)) {                     \
-      dataRowTKey(memRowDataBody(r)) = (k); \
-    } else {                                \
-      kvRowTKey(memRowKvBody(r)) = (k);     \
-    }                                       \
-  } while (0)
-
-#define memRowSetLen(r, l)          (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
-#define memRowSetVersion(r, v)      (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
-#define memRowCpy(dst, r)           memcpy((dst), (r), memRowTLen(r))
-#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE)
-#define memRowDeleted(r)            TKEY_IS_DELETED(memRowTKey(r))
-
-SMemRow tdMemRowDup(SMemRow row);
-void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
-
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) {
-  if (isDataRow(row)) {
-    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
-  } else {
-    return tdGetKVRowValOfCol(memRowKvBody(row), colId);
-  }
-}
-
-/**
- * NOTE:
- *  1. Applicable to scan columns one by one
- *  2. offset here including the header size
- */
-static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset,
-                                                 int32_t *kvNIdx) {
-  if (isDataRow(row)) {
-    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
-  } else {
-    return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx);
-  }
-}
-
-static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
-                                             int8_t type, int32_t offset) {
-  if (isDataRow(row)) {
-    tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
-  } else {
-    tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
-  }
-  return 0;
-}
-
-// make sure schema->flen appended for SDataRow
-static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) {
-  int32_t len = 0;
-  if (IS_VAR_DATA_TYPE(colType)) {
-    len += varDataTLen(value);
-    if (rowType == SMEM_ROW_KV) {
-      len += sizeof(SColIdx);
-    }
-  } else {
-    if (rowType == SMEM_ROW_KV) {
-      len += TYPE_BYTES[colType];
-      len += sizeof(SColIdx);
-    }
-  }
-  return len;
-}
-
-typedef struct {
-  int16_t colId;
-  uint8_t colType;
-  char *  colVal;
-} SColInfo;
-
-static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) {
-  colInfo->colId = colId;
-  colInfo->colType = colType;
-  colInfo->colVal = colVal;
-}
-
-SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
 #endif
 
 #ifdef __cplusplus

include/common/tglobal.h

@@ -17,8 +17,8 @@
 #define _TD_COMMON_GLOBAL_H_
 
 #include "tarray.h"
-#include "tdef.h"
 #include "tconfig.h"
+#include "tdef.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -32,31 +32,27 @@ extern char     tsLocalEp[];
 extern uint16_t tsServerPort;
 extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
-extern bool     tsEnableTelemetryReporting;
+extern int32_t  tsNumOfSupportVnodes;
 
 // common
-extern int32_t tsRpcTimer;
-extern int32_t tsRpcMaxTime;
-extern bool    tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
-extern int32_t tsMaxConnections;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
-extern int32_t tsMaxTmrCtrl;
 extern int32_t tsCompressMsgSize;
 extern int32_t tsCompressColData;
 extern int32_t tsMaxNumOfDistinctResults;
 extern int32_t tsCompatibleModel;
 extern bool    tsEnableSlaveQuery;
 extern bool    tsPrintAuth;
-extern int64_t tsTickPerDay[3];
+extern int64_t tsTickPerMin[3];
 
 // multi-process
-extern bool    tsMultiProcess;
+extern int32_t tsMultiProcess;
 extern int32_t tsMnodeShmSize;
 extern int32_t tsVnodeShmSize;
 extern int32_t tsQnodeShmSize;
 extern int32_t tsSnodeShmSize;
 extern int32_t tsBnodeShmSize;
+extern int32_t tsNumOfShmThreads;
 
 // queue & threads
 extern int32_t tsNumOfRpcThreads;
@@ -73,6 +69,7 @@ extern int32_t tsNumOfQnodeQueryThreads;
 extern int32_t tsNumOfQnodeFetchThreads;
 extern int32_t tsNumOfSnodeSharedThreads;
 extern int32_t tsNumOfSnodeUniqueThreads;
+extern int64_t tsRpcQueueMemoryAllowed;
 
 // monitor
 extern bool     tsEnableMonitor;
@@ -82,6 +79,12 @@ extern uint16_t tsMonitorPort;
 extern int32_t  tsMonitorMaxLogs;
 extern bool     tsMonitorComp;
 
+// telem
+extern bool     tsEnableTelem;
+extern int32_t  tsTelemInterval;
+extern char     tsTelemServer[];
+extern uint16_t tsTelemPort;
+
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
 extern int64_t tsQueryBufferSizeBytes;   // maximum allowed usage buffer size in byte for each data node
@@ -89,10 +92,10 @@ extern bool    tsRetrieveBlockingModel;  // retrieve threads will be blocked
 extern bool    tsKeepOriginalColumnName;
 extern bool    tsDeadLockKillQuery;
 
+// query client
+extern int32_t tsQueryPolicy;
+
 // client
-extern int32_t tsMaxWildCardsLen;
-extern int32_t tsMaxRegexStringLen;
-extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
 extern int32_t tsMaxStreamComputDelay;
@@ -120,11 +123,23 @@ extern char     tsCompressor[];
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
+// udf
+extern bool tsStartUdfd;
+
+// schemaless
+extern char tsSmlChildTableName[];
+extern bool tsSmlDataFormat;
+
+// internal
+extern int32_t tsTransPullupInterval;
+extern int32_t tsMqRebalanceInterval;
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
-                      const char *apolloUrl, SArray *pArgs, bool tsc);
+                      const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
-int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc);
+int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
+                    bool tsc);
 void    taosCleanupCfg();
 void    taosCfgDynamicOptions(const char *option, const char *value);
 void    taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);

include/common/tmsgcb.h

@@ -25,6 +25,7 @@ extern "C" {
 typedef struct SRpcMsg        SRpcMsg;
 typedef struct SEpSet         SEpSet;
 typedef struct SMgmtWrapper   SMgmtWrapper;
+typedef struct SRpcHandleInfo SRpcHandleInfo;
 
 typedef enum {
   QUERY_QUEUE,
@@ -37,33 +38,36 @@ typedef enum {
   QUEUE_MAX,
 } EQueueType;
 
-typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
+typedef int32_t (*PutToQueueFp)(void* pMgmt, SRpcMsg* pMsg);
-typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
+typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
-typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
+typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
-typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
+typedef void (*SendRspFp)(SRpcMsg* pMsg);
-typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
+typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
-typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
+typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
-typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
+typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
-typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc);
+typedef void (*ReportStartup)(const char* name, const char* desc);
 
 typedef struct {
-  SMgmtWrapper*           pWrapper;
+  void*                   mgmt;
+  void*                   clientRpc;
   PutToQueueFp            queueFps[QUEUE_MAX];
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
   SendRspFp               sendRspFp;
+  SendRedirectRspFp       sendRedirectRspFp;
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
   ReportStartup           reportStartupFp;
 } SMsgCb;
 
 void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
-int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
+int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
-int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
+int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
-void    tmsgSendRsp(const SRpcMsg* pRsp);
+void    tmsgSendRsp(SRpcMsg* pMsg);
-void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
+void    tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
-void    tmsgReleaseHandle(void* handle, int8_t type);
+void    tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
+void    tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
 void    tmsgReportStartup(const char* name, const char* desc);
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -85,6 +85,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
 
   // Requests handled by MNODE
   TD_NEW_MSG_SEG(TDMT_MND_MSG)
@@ -144,9 +145,11 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
@@ -167,17 +170,15 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateTbReq, SVCreateTbRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONSUME, "vnode-mq-consume", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@@ -202,6 +203,22 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp)
+
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL)
   
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)

include/common/tname.h

@@ -16,6 +16,7 @@
 #ifndef _TD_COMMON_NAME_H_
 #define _TD_COMMON_NAME_H_
 
+#include "tarray.h"
 #include "tdef.h"
 
 #ifdef __cplusplus
@@ -62,6 +63,18 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
 bool tNameDBNameEqual(SName* left, SName* right);
 
+typedef struct {
+  // input
+  SArray*     tags;           // element is SSmlKv
+  const char* sTableName;     // super table name
+  uint8_t     sTableNameLen;  // the length of super table name
+
+  // output
+  char*    childTableName;  // must have size of TSDB_TABLE_NAME_LEN;
+  uint64_t uid;             // child table uid, may be useful
+} RandTableName;
+
+void buildChildTableName(RandTableName* rName);
 
 #ifdef __cplusplus
 }

include/common/tschema.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_COMMON_SCHEMA_H_
-#define _TD_COMMON_SCHEMA_H_
-
-#include "os.h"
-#include "tarray.h"
-#include "ttypes.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if 0
-typedef struct STColumn {
-  /// column name
-  char *cname;
-  union {
-    /// for encode purpose
-    uint64_t info;
-    struct {
-      uint64_t sma : 1;
-      /// column data type
-      uint64_t type : 7;
-      /// column id
-      uint64_t cid : 16;
-      /// max bytes of the column
-      uint64_t bytes : 32;
-      /// reserved
-      uint64_t reserve : 8;
-    };
-  };
-  /// comment about the column
-  char *comment;
-} STColumn;
-
-typedef struct STSchema {
-  /// schema version
-  uint16_t sver;
-  /// number of columns
-  uint16_t ncols;
-  /// sma attributes
-  struct {
-    bool    sma;
-    SArray *smaArray;
-  };
-  /// column info
-  STColumn cols[];
-} STSchema;
-
-typedef struct {
-  uint64_t  size;
-  STSchema *pSchema;
-} STShemaBuilder;
-
-#define tSchemaBuilderInit(target, capacity) \
-  { .size = (capacity), .pSchema = (target) }
-void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
-void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
-int32_t  tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
-
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_COMMON_SCHEMA_H_*/

include/common/ttime.h

@@ -59,8 +59,9 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
  *    precision == TSDB_TIME_PRECISION_NANO,  it returns timestamp in nanosecond.
  */
 static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
-  int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 :
+  int64_t    factor = (precision == TSDB_TIME_PRECISION_MILLI)   ? 1000
-                   (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000;
+                      : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
+                                                                 : 1000000000;
   time_t     t = taosTime(NULL);
   struct tm* tm = taosLocalTime(&t, NULL);
   tm->tm_hour = 0;

include/common/ttokendef.h

@@ -51,191 +51,208 @@
 #define TK_USER          33
 #define TK_PRIVILEGE     34
 #define TK_DROP          35
-#define TK_DNODE                           36
+#define TK_GRANT         36
-#define TK_PORT                            37
+#define TK_ON            37
-#define TK_NK_INTEGER                      38
+#define TK_TO            38
-#define TK_DNODES                          39
+#define TK_REVOKE        39
-#define TK_NK_IPTOKEN                      40
+#define TK_FROM          40
-#define TK_LOCAL                           41
+#define TK_NK_COMMA      41
-#define TK_QNODE                           42
+#define TK_READ          42
-#define TK_ON                              43
+#define TK_WRITE         43
-#define TK_BNODE                           44
+#define TK_NK_DOT        44
-#define TK_SNODE                           45
+#define TK_DNODE         45
-#define TK_MNODE                           46
+#define TK_PORT          46
-#define TK_DATABASE                        47
+#define TK_NK_INTEGER    47
-#define TK_USE                             48
+#define TK_DNODES        48
-#define TK_IF                              49
+#define TK_NK_IPTOKEN    49
-#define TK_NOT                             50
+#define TK_LOCAL         50
-#define TK_EXISTS                          51
+#define TK_QNODE         51
-#define TK_BLOCKS                          52
+#define TK_BNODE         52
-#define TK_CACHE                           53
+#define TK_SNODE         53
-#define TK_CACHELAST                       54
+#define TK_MNODE         54
-#define TK_COMP                            55
+#define TK_DATABASE      55
-#define TK_DAYS                            56
+#define TK_USE           56
-#define TK_NK_VARIABLE                     57
+#define TK_IF            57
-#define TK_FSYNC                           58
+#define TK_NOT           58
-#define TK_MAXROWS                         59
+#define TK_EXISTS        59
-#define TK_MINROWS                         60
+#define TK_BUFFER        60
-#define TK_KEEP                            61
+#define TK_CACHELAST     61
-#define TK_PRECISION                       62
+#define TK_COMP          62
-#define TK_QUORUM                          63
+#define TK_DAYS          63
-#define TK_REPLICA                         64
+#define TK_NK_VARIABLE   64
-#define TK_TTL                             65
+#define TK_FSYNC         65
-#define TK_WAL                             66
+#define TK_MAXROWS       66
-#define TK_VGROUPS                         67
+#define TK_MINROWS       67
-#define TK_SINGLE_STABLE                   68
+#define TK_KEEP          68
-#define TK_STREAM_MODE                     69
+#define TK_PAGES         69
-#define TK_RETENTIONS                      70
+#define TK_PAGESIZE      70
-#define TK_STRICT                          71
+#define TK_PRECISION     71
-#define TK_NK_COMMA                        72
+#define TK_REPLICA       72
-#define TK_NK_COLON                        73
+#define TK_STRICT        73
-#define TK_TABLE                           74
+#define TK_WAL           74
-#define TK_NK_LP                           75
+#define TK_VGROUPS       75
-#define TK_NK_RP                           76
+#define TK_SINGLE_STABLE 76
-#define TK_STABLE                          77
+#define TK_RETENTIONS    77
-#define TK_ADD                             78
+#define TK_NK_COLON      78
-#define TK_COLUMN                          79
+#define TK_TABLE         79
-#define TK_MODIFY                          80
+#define TK_NK_LP         80
-#define TK_RENAME                          81
+#define TK_NK_RP         81
-#define TK_TAG                             82
+#define TK_STABLE        82
-#define TK_SET                             83
+#define TK_ADD           83
-#define TK_NK_EQ                           84
+#define TK_COLUMN        84
-#define TK_USING                           85
+#define TK_MODIFY        85
-#define TK_TAGS                            86
+#define TK_RENAME        86
-#define TK_NK_DOT                          87
+#define TK_TAG           87
-#define TK_COMMENT                         88
+#define TK_SET           88
-#define TK_BOOL                            89
+#define TK_NK_EQ         89
-#define TK_TINYINT                         90
+#define TK_USING         90
-#define TK_SMALLINT                        91
+#define TK_TAGS          91
-#define TK_INT                             92
+#define TK_COMMENT       92
-#define TK_INTEGER                         93
+#define TK_BOOL          93
-#define TK_BIGINT                          94
+#define TK_TINYINT       94
-#define TK_FLOAT                           95
+#define TK_SMALLINT      95
-#define TK_DOUBLE                          96
+#define TK_INT           96
-#define TK_BINARY                          97
+#define TK_INTEGER       97
-#define TK_TIMESTAMP                       98
+#define TK_BIGINT        98
-#define TK_NCHAR                           99
+#define TK_FLOAT         99
-#define TK_UNSIGNED                       100
+#define TK_DOUBLE        100
-#define TK_JSON                           101
+#define TK_BINARY        101
-#define TK_VARCHAR                        102
+#define TK_TIMESTAMP     102
-#define TK_MEDIUMBLOB                     103
+#define TK_NCHAR         103
-#define TK_BLOB                           104
+#define TK_UNSIGNED      104
-#define TK_VARBINARY                      105
+#define TK_JSON          105
-#define TK_DECIMAL                        106
+#define TK_VARCHAR       106
-#define TK_SMA                            107
+#define TK_MEDIUMBLOB    107
-#define TK_ROLLUP                         108
+#define TK_BLOB          108
-#define TK_FILE_FACTOR                    109
+#define TK_VARBINARY     109
-#define TK_NK_FLOAT                       110
+#define TK_DECIMAL       110
 #define TK_DELAY         111
-#define TK_SHOW                           112
+#define TK_FILE_FACTOR   112
-#define TK_DATABASES                      113
+#define TK_NK_FLOAT      113
-#define TK_TABLES                         114
+#define TK_ROLLUP        114
-#define TK_STABLES                        115
+#define TK_TTL           115
-#define TK_MNODES                         116
+#define TK_SMA           116
-#define TK_MODULES                        117
+#define TK_SHOW          117
-#define TK_QNODES                         118
+#define TK_DATABASES     118
-#define TK_FUNCTIONS                      119
+#define TK_TABLES        119
-#define TK_INDEXES                        120
+#define TK_STABLES       120
-#define TK_FROM                           121
+#define TK_MNODES        121
-#define TK_ACCOUNTS                       122
+#define TK_MODULES       122
-#define TK_APPS                           123
+#define TK_QNODES        123
-#define TK_CONNECTIONS                    124
+#define TK_FUNCTIONS     124
-#define TK_LICENCE                        125
+#define TK_INDEXES       125
-#define TK_GRANTS                         126
+#define TK_ACCOUNTS      126
-#define TK_QUERIES                        127
+#define TK_APPS          127
-#define TK_SCORES                         128
+#define TK_CONNECTIONS   128
-#define TK_TOPICS                         129
+#define TK_LICENCE       129
-#define TK_VARIABLES                      130
+#define TK_GRANTS        130
-#define TK_BNODES                         131
+#define TK_QUERIES       131
-#define TK_SNODES                         132
+#define TK_SCORES        132
-#define TK_LIKE                           133
+#define TK_TOPICS        133
-#define TK_INDEX                          134
+#define TK_VARIABLES     134
-#define TK_FULLTEXT                       135
+#define TK_BNODES        135
-#define TK_FUNCTION                       136
+#define TK_SNODES        136
-#define TK_INTERVAL                       137
+#define TK_CLUSTER       137
-#define TK_TOPIC                          138
+#define TK_TRANSACTIONS  138
-#define TK_AS                             139
+#define TK_LIKE          139
-#define TK_DESC                           140
+#define TK_INDEX         140
-#define TK_DESCRIBE                       141
+#define TK_FULLTEXT      141
-#define TK_RESET                          142
+#define TK_FUNCTION      142
-#define TK_QUERY                          143
+#define TK_INTERVAL      143
-#define TK_EXPLAIN                        144
+#define TK_TOPIC         144
-#define TK_ANALYZE                        145
+#define TK_AS            145
-#define TK_VERBOSE                        146
+#define TK_WITH          146
-#define TK_NK_BOOL                        147
+#define TK_SCHEMA        147
-#define TK_RATIO                          148
+#define TK_DESC          148
-#define TK_COMPACT                        149
+#define TK_DESCRIBE      149
-#define TK_VNODES                         150
+#define TK_RESET         150
-#define TK_IN                             151
+#define TK_QUERY         151
-#define TK_OUTPUTTYPE                     152
+#define TK_CACHE         152
-#define TK_AGGREGATE                      153
+#define TK_EXPLAIN       153
-#define TK_BUFSIZE                        154
+#define TK_ANALYZE       154
-#define TK_STREAM                         155
+#define TK_VERBOSE       155
-#define TK_INTO                           156
+#define TK_NK_BOOL       156
-#define TK_TRIGGER                        157
+#define TK_RATIO         157
-#define TK_AT_ONCE                        158
+#define TK_COMPACT       158
-#define TK_WINDOW_CLOSE                   159
+#define TK_VNODES        159
-#define TK_WATERMARK                      160
+#define TK_IN            160
-#define TK_KILL                           161
+#define TK_OUTPUTTYPE    161
-#define TK_CONNECTION                     162
+#define TK_AGGREGATE     162
-#define TK_MERGE                          163
+#define TK_BUFSIZE       163
-#define TK_VGROUP                         164
+#define TK_STREAM        164
-#define TK_REDISTRIBUTE                   165
+#define TK_INTO          165
-#define TK_SPLIT                          166
+#define TK_TRIGGER       166
-#define TK_SYNCDB                         167
+#define TK_AT_ONCE       167
-#define TK_NULL                           168
+#define TK_WINDOW_CLOSE  168
-#define TK_NK_QUESTION                    169
+#define TK_WATERMARK     169
-#define TK_NK_ARROW                       170
+#define TK_KILL          170
-#define TK_ROWTS                          171
+#define TK_CONNECTION    171
-#define TK_TBNAME                         172
+#define TK_TRANSACTION   172
-#define TK_QSTARTTS                       173
+#define TK_MERGE         173
-#define TK_QENDTS                         174
+#define TK_VGROUP        174
-#define TK_WSTARTTS                       175
+#define TK_REDISTRIBUTE  175
-#define TK_WENDTS                         176
+#define TK_SPLIT         176
-#define TK_WDURATION                      177
+#define TK_SYNCDB        177
-#define TK_CAST                           178
+#define TK_NULL          178
-#define TK_NOW                            179
+#define TK_NK_QUESTION   179
-#define TK_TODAY                          180
+#define TK_NK_ARROW      180
-#define TK_TIMEZONE                       181
+#define TK_ROWTS         181
-#define TK_COUNT                          182
+#define TK_TBNAME        182
-#define TK_FIRST                          183
+#define TK_QSTARTTS      183
-#define TK_LAST                           184
+#define TK_QENDTS        184
-#define TK_LAST_ROW                       185
+#define TK_WSTARTTS      185
-#define TK_BETWEEN                        186
+#define TK_WENDTS        186
-#define TK_IS                             187
+#define TK_WDURATION     187
-#define TK_NK_LT                          188
+#define TK_CAST          188
-#define TK_NK_GT                          189
+#define TK_NOW           189
-#define TK_NK_LE                          190
+#define TK_TODAY         190
-#define TK_NK_GE                          191
+#define TK_TIMEZONE      191
-#define TK_NK_NE                          192
+#define TK_COUNT         192
-#define TK_MATCH                          193
+#define TK_FIRST         193
-#define TK_NMATCH                         194
+#define TK_LAST          194
-#define TK_CONTAINS                       195
+#define TK_LAST_ROW      195
-#define TK_JOIN                           196
+#define TK_BETWEEN       196
-#define TK_INNER                          197
+#define TK_IS            197
-#define TK_SELECT                         198
+#define TK_NK_LT         198
-#define TK_DISTINCT                       199
+#define TK_NK_GT         199
-#define TK_WHERE                          200
+#define TK_NK_LE         200
-#define TK_PARTITION                      201
+#define TK_NK_GE         201
-#define TK_BY                             202
+#define TK_NK_NE         202
-#define TK_SESSION                        203
+#define TK_MATCH         203
-#define TK_STATE_WINDOW                   204
+#define TK_NMATCH        204
-#define TK_SLIDING                        205
+#define TK_CONTAINS      205
-#define TK_FILL                           206
+#define TK_JOIN          206
-#define TK_VALUE                          207
+#define TK_INNER         207
-#define TK_NONE                           208
+#define TK_SELECT        208
-#define TK_PREV                           209
+#define TK_DISTINCT      209
-#define TK_LINEAR                         210
+#define TK_WHERE         210
-#define TK_NEXT                           211
+#define TK_PARTITION     211
-#define TK_GROUP                          212
+#define TK_BY            212
-#define TK_HAVING                         213
+#define TK_SESSION       213
-#define TK_ORDER                          214
+#define TK_STATE_WINDOW  214
-#define TK_SLIMIT                         215
+#define TK_SLIDING       215
-#define TK_SOFFSET                        216
+#define TK_FILL          216
-#define TK_LIMIT                          217
+#define TK_VALUE         217
-#define TK_OFFSET                         218
+#define TK_NONE          218
-#define TK_ASC                            219
+#define TK_PREV          219
-#define TK_NULLS                          220
+#define TK_LINEAR        220
+#define TK_NEXT          221
+#define TK_GROUP         222
+#define TK_HAVING        223
+#define TK_ORDER         224
+#define TK_SLIMIT        225
+#define TK_SOFFSET       226
+#define TK_LIMIT         227
+#define TK_OFFSET        228
+#define TK_ASC           229
+#define TK_NULLS         230
+#define TK_ID            231
+#define TK_NK_BITNOT     232
+#define TK_INSERT        233
+#define TK_VALUES        234
+#define TK_IMPORT        235
+#define TK_NK_SEMI       236
+#define TK_FILE          237
 
 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301
@@ -243,13 +260,6 @@
 #define TK_NK_HEX     303  // hex number  0x123
 #define TK_NK_OCT     304  // oct number
 #define TK_NK_BIN     305  // bin format data 0b111
-#define TK_NK_FILE                        306
-
-#define TK_NK_BITNOT                      501
-#define TK_INSERT                         502
-#define TK_VALUES                         507
-#define TK_IMPORT                         509
-#define TK_NK_SEMI                        508
 
 #define TK_NK_NIL 65535
 

include/common/ttypes.h

@@ -50,6 +50,7 @@ typedef struct {
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
 #define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
+#define IS_STR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -186,14 +187,14 @@ typedef struct {
 #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
 #define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
 
-#define IS_VALID_TINYINT(_t)   ((_t) > INT8_MIN && (_t) <= INT8_MAX)
+#define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
-#define IS_VALID_SMALLINT(_t)  ((_t) > INT16_MIN && (_t) <= INT16_MAX)
+#define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
-#define IS_VALID_INT(_t)       ((_t) > INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
-#define IS_VALID_BIGINT(_t)    ((_t) > INT64_MIN && (_t) <= INT64_MAX)
+#define IS_VALID_BIGINT(_t)    ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
-#define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) < UINT8_MAX)
+#define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
-#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX)
+#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
-#define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) < UINT32_MAX)
+#define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
-#define IS_VALID_UBIGINT(_t)   ((_t) >= 0 && (_t) < UINT64_MAX)
+#define IS_VALID_UBIGINT(_t)   ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -36,12 +36,11 @@ typedef struct SVariant {
   };
 } SVariant;
 
-int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *issigned);
+int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
+int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 
 bool taosVariantIsValid(SVariant *pVar);
 
-void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type);
-
 void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
 
 void taosVariantDestroy(SVariant *pV);

include/dnode/bnode/bnode.h

@@ -26,6 +26,9 @@ extern "C" {
 typedef struct SBnode SBnode;
 
 typedef struct {
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 } SBnodeLoad;
 
 typedef struct {

include/dnode/mgmt/dnode.h

@@ -22,66 +22,28 @@
 extern "C" {
 #endif
 
-/* ------------------------ TYPES EXPOSED ---------------- */
-typedef struct SDnode SDnode;
-
 /**
- * @brief Initialize the environment
+ * @brief Initialize the dnode
  *
+ * @param rtype for internal debug usage, default is 0
  * @return int32_t 0 for success and -1 for failure
  */
-int32_t dmInit();
+int32_t dmInit(int8_t rtype);
 
 /**
- * @brief Clear the environment
+ * @brief Cleanup the dnode
  */
 void dmCleanup();
 
-/* ------------------------ SDnode ----------------------- */
+/**
-typedef struct {
+ * @brief Run dnode.
-  int32_t   numOfSupportVnodes;
+ */
-  uint16_t  serverPort;
+int32_t dmRun();
-  char      dataDir[PATH_MAX];
-  char      localEp[TSDB_EP_LEN];
-  char      localFqdn[TSDB_FQDN_LEN];
-  char      firstEp[TSDB_EP_LEN];
-  char      secondEp[TSDB_EP_LEN];
-  SDiskCfg *disks;
-  int32_t   numOfDisks;
-  int8_t    ntype;
-} SDnodeOpt;
-
-typedef enum { DND_EVENT_START = 0, DND_EVENT_STOP = 1, DND_EVENT_CHILD = 2 } EDndEvent;
 
 /**
- * @brief Initialize and start the dnode.
+ * @brief Stop dnode.
- *
- * @param pOption Option of the dnode.
- * @return SDnode* The dnode object.
  */
-SDnode *dmCreate(const SDnodeOpt *pOption);
+void dmStop();
-
-/**
- * @brief Stop and cleanup the dnode.
- *
- * @param pDnode The dnode object to close.
- */
-void dmClose(SDnode *pDnode);
-
-/**
- * @brief Run dnode until specific event is receive.
- *
- * @param pDnode The dnode object to run.
- */
-int32_t dmRun(SDnode *pDnode);
-
-/**
- * @brief Handle event in the dnode.
- *
- * @param pDnode The dnode object to close.
- * @param event The event to handle.
- */
-void dmSetEvent(SDnode *pDnode, EDndEvent event);
 
 #ifdef __cplusplus
 }

include/dnode/mnode/mnode.h

@@ -88,7 +88,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
  * @param pMsg The request msg.
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t mndProcessMsg(SNodeMsg *pMsg);
+int32_t mndProcessMsg(SRpcMsg *pMsg);
 
 /**
  * @brief Generate machine code

include/dnode/mnode/sdb/sdb.h

@@ -18,6 +18,11 @@
 
 #include "os.h"
 
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -92,14 +97,19 @@ extern "C" {
 typedef struct SMnode  SMnode;
 typedef struct SSdbRaw SSdbRaw;
 typedef struct SSdbRow SSdbRow;
-typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
+
+typedef enum {
+  SDB_KEY_BINARY = 1,
+  SDB_KEY_INT32 = 2,
+  SDB_KEY_INT64 = 3,
+} EKeyType;
+
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,
-  SDB_STATUS_UPDATING = 2,
+  SDB_STATUS_DROPPING = 2,
-  SDB_STATUS_DROPPING = 3,
+  SDB_STATUS_DROPPED = 3,
   SDB_STATUS_READY = 4,
-  SDB_STATUS_DROPPED = 5
 } ESdbStatus;
 
 typedef enum {
@@ -129,7 +139,7 @@ typedef enum {
 typedef struct SSdb SSdb;
 typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
-typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj);
+typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
@@ -215,7 +225,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw);
  * @param pRaw The raw data.
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw);
+int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw);
 
 /**
  * @brief Acquire a row from sdb
@@ -320,9 +330,28 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
+typedef struct SSdb {
+  SMnode     *pMnode;
+  char       *currDir;
+  char       *syncDir;
+  char       *tmpDir;
+  int64_t     lastCommitVer;
+  int64_t     curVer;
+  int64_t     tableVer[SDB_MAX];
+  int64_t     maxId[SDB_MAX];
+  EKeyType    keyTypes[SDB_MAX];
+  SHashObj   *hashObjs[SDB_MAX];
+  SRWLatch    locks[SDB_MAX];
+  SdbInsertFp insertFps[SDB_MAX];
+  SdbUpdateFp updateFps[SDB_MAX];
+  SdbDeleteFp deleteFps[SDB_MAX];
+  SdbDeployFp deployFps[SDB_MAX];
+  SdbEncodeFp encodeFps[SDB_MAX];
+  SdbDecodeFp decodeFps[SDB_MAX];
+} SSdb;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/catalog/catalog.h

@@ -40,6 +40,11 @@ enum {
   CTG_DBG_STB_RENT_NUM,
 };
 
+typedef enum {
+  AUTH_TYPE_READ = 1,
+  AUTH_TYPE_WRITE,
+  AUTH_TYPE_OTHER,
+} AUTH_TYPE;
 
 typedef struct SCatalogReq {
   SArray *pTableName;     // element is SNAME
@@ -51,12 +56,13 @@ typedef struct SMetaData {
   SArray    *pTableMeta;  // STableMeta array
   SArray    *pVgroupInfo; // SVgroupInfo list
   SArray    *pUdfList;    // udf info list
-  SArray    *pEpSetList;  // qnode epset list, SArray<SEpSet>
+  SArray    *pQnodeList;  // qnode list, SArray<SQueryNodeAddr>
 } SMetaData;
 
 typedef struct SCatalogCfg {
   uint32_t maxTblCacheNum;
   uint32_t maxDBCacheNum;
+  uint32_t maxUserCacheNum;
   uint32_t dbRentSec;
   uint32_t stbRentSec;
 } SCatalogCfg;
@@ -77,6 +83,11 @@ typedef struct SDbVgVersion {
   int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
 } SDbVgVersion;
 
+typedef struct SUserAuthVersion {
+  char    user[TSDB_USER_LEN];
+  int32_t version;
+} SUserAuthVersion;
+
 typedef SDbCfgRsp SDbCfgInfo;
 typedef SUserIndexRsp SIndexInfo;
 
@@ -213,17 +224,24 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const
  */
 int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
-
 int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
 
 int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);
 
 int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
 
+int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num);
+
 int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg);
 
 int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
 
+int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo);
+
+int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass);
+
+int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);
+
 
 /**
  * Destroy catalog and relase all resources

include/libs/executor/dataSinkMgt.h

@@ -45,6 +45,7 @@ typedef struct SInputData {
 
 typedef struct SOutputData {
   int32_t numOfRows;
+  int32_t numOfCols;
   int8_t  compressed;
   char*   pData;
   bool    queryEnd;

include/libs/executor/executor.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "query.h"
 #include "tcommon.h"
+#include "tmsgcb.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -32,6 +33,9 @@ typedef struct SReadHandle {
   void*   reader;
   void*   meta;
   void*   config;
+  void*   vnode;
+  void*   mnd;
+  SMsgCb* pMsgCb;
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
@@ -163,15 +167,6 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
  */
 int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
 
-/**
- * release the query handle and decrease the reference count in cache
- * @param pMgmt
- * @param pQInfo
- * @param freeHandle
- * @return
- */
-void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
-
 void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
 
 int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes);

include/libs/function/function.h

@@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
 typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -126,7 +126,7 @@ enum {
 
 enum {
   MAIN_SCAN     = 0x0u,
-  REVERSE_SCAN  = 0x1u,
+  REVERSE_SCAN  = 0x1u,  // todo remove it
   REPEAT_SCAN   = 0x2u,  //repeat scan belongs to the master scan
   MERGE_STAGE   = 0x20u,
 };
@@ -141,8 +141,7 @@ struct SResultRowEntryInfo;
 
 //for selectivity query, the corresponding tag value is assigned if the data is qualified
 typedef struct SSubsidiaryResInfo {
-  int16_t                 bufLen;      // keep the tags data for top/bottom query result
+  int16_t num;
-  int16_t                 numOfCols;
   struct SqlFunctionCtx **pCtx;
 } SSubsidiaryResInfo;
 
@@ -166,6 +165,7 @@ typedef struct SInputColumnInfoData {
   SColumnInfoData  *pPTS;           // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;
+  uint64_t          uid;            // table uid, used to set the tag value when building the final query result for selectivity functions.
 } SInputColumnInfoData;
 
 // sql function runtime context
@@ -173,35 +173,27 @@ typedef struct SqlFunctionCtx {
   SInputColumnInfoData   input;
   SResultDataInfo        resDataInfo;
   uint32_t               order;  // data block scanner order: asc|desc
-  ////////////////////////////////////////////////////////////////
+  uint8_t                scanFlag;  // record current running step, default: 0
-  int32_t          startRow;   // start row index
-  int32_t          size;       // handled processed row number
-  SColumnInfoData* pInput;
-  SColumnDataAgg   agg;
-  int16_t          inputType;    // TODO remove it
-  int16_t          inputBytes;   // TODO remove it
-  bool             hasNull;      // null value exist in current block, TODO remove it
-  bool             requireNull;  // require null in some function, TODO remove it
-  int32_t          columnIndex;  // TODO remove it
-  uint8_t          currentStage;  // record current running step, default: 0
-  bool             isAggSet;
-  int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block, TODO remove it
-  /////////////////////////////////////////////////////////////////
-  bool             stableQuery;
   int16_t                functionId;    // function id
   char                  *pOutput;       // final result output buffer, point to sdata->data
   int32_t                numOfParams;
-  SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  SFunctParam           *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t               *ptsList;       // corresponding timestamp array list
   SColumnInfoData       *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
   int32_t                offset;
   SVariant               tag;
   struct  SResultRowEntryInfo *resultInfo;
-  SSubsidiaryResInfo     subsidiaryRes;
+  SSubsidiaryResInfo     subsidiaries;
   SPoint1                start;
   SPoint1                end;
   SFuncExecFuncs         fpSet;
   SScalarFuncExecFuncs   sfp;
+  struct SExprInfo      *pExpr;
+  struct SDiskbasedBuf  *pBuf;
+  struct SSDataBlock    *pSrcBlock;
+  int32_t                curBufPage;
+
+  char                   udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 enum {
@@ -216,13 +208,6 @@ enum {
 typedef struct tExprNode {
   int32_t nodeType;
   union {
-    struct {
-      int32_t           optr;   // binary operator
-      void             *info;   // support filter operation on this expression only available for leaf node
-      struct tExprNode *pLeft;  // left child pointer
-      struct tExprNode *pRight; // right child pointer
-    } _node;
-
     SSchema            *pSchema;// column node
     struct SVariant    *pVal;   // value node
 
@@ -231,12 +216,6 @@ typedef struct tExprNode {
       int32_t           functionId;
       int32_t           num;
       struct SFunctionNode    *pFunctNode;
-      // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
-      // calculation instead.
-      // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
-      //  The concat function, concat(col1, col2), is a binary scalar
-      //  operator and is kept in the attribute of _node.
-      struct tExprNode **pChild;
     } _function;
 
     struct {
@@ -267,6 +246,7 @@ typedef struct SAggFunctionInfo {
 struct SScalarParam {
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;
+  void            *param;  // other parameter, such as meta handle from vnode, to extract table name/tag value
   int32_t          numOfRows;
 };
 
@@ -275,10 +255,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
 
 bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId);
 
-tExprNode* exprTreeFromBinary(const void* data, size_t size);
-
-tExprNode* exprdup(tExprNode* pTree);
-
 void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);
 void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell);
 int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock);
@@ -318,6 +294,29 @@ struct SUdfInfo;
 void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 
+/**
+ * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
+ * @return error code
+ */
+int32_t udfcOpen();
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t udfcClose();
+
+/**
+ * start udfd that serves udf function invocation under dnode startDnodeId
+ * @param startDnodeId
+ * @return
+ */
+int32_t udfStartUdfd(int32_t startDnodeId);
+/**
+ * stop udfd
+ * @return
+ */
+int32_t udfStopUdfd();
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "function.h"
+#include "querynodes.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -31,15 +31,17 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ELAPSED,
   FUNCTION_TYPE_IRATE,
   FUNCTION_TYPE_LAST_ROW,
-  FUNCTION_TYPE_LEASTSQUARES,
   FUNCTION_TYPE_MAX,
   FUNCTION_TYPE_MIN,
   FUNCTION_TYPE_MODE,
   FUNCTION_TYPE_PERCENTILE,
   FUNCTION_TYPE_SPREAD,
   FUNCTION_TYPE_STDDEV,
+  FUNCTION_TYPE_LEASTSQUARES,
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
+  FUNCTION_TYPE_HISTOGRAM,
+  FUNCTION_TYPE_HYPERLOGLOG,
 
   // nonstandard SQL function
   FUNCTION_TYPE_BOTTOM = 500,
@@ -54,6 +56,8 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_TAIL,
   FUNCTION_TYPE_TOP,
   FUNCTION_TYPE_UNIQUE,
+  FUNCTION_TYPE_STATE_COUNT,
+  FUNCTION_TYPE_STATE_DURATION,
 
   // math function
   FUNCTION_TYPE_ABS = 1000,
@@ -110,34 +114,51 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_QENDTS,
   FUNCTION_TYPE_WSTARTTS,
   FUNCTION_TYPE_WENDTS,
-  FUNCTION_TYPE_WDURATION
+  FUNCTION_TYPE_WDURATION,
+
+  // internal function
+  FUNCTION_TYPE_SELECT_VALUE,
+
+  // user defined funcion
+  FUNCTION_TYPE_UDF = 10000
 } EFunctionType;
 
 struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
+struct SCatalog;
+
+typedef struct SFmGetFuncInfoParam {
+  struct SCatalog* pCtg;
+  void*            pRpc;
+  const SEpSet*    pMgmtEps;
+  char*            pErrBuf;
+  int32_t          errBufLen;
+} SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
-
-int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
 
 bool fmIsAggFunc(int32_t funcId);
 bool fmIsScalarFunc(int32_t funcId);
 bool fmIsNonstandardSQLFunc(int32_t funcId);
 bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
+bool fmIsSelectFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
+bool fmIsScanPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);
 bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
 bool fmIsMultiResFunc(int32_t funcId);
+bool fmIsRepeatScanFunc(int32_t funcId);
+bool fmIsUserDefinedFunc(int32_t funcId);
 
 typedef enum EFuncDataRequired {
   FUNC_DATA_REQUIRED_DATA_LOAD = 1,
@@ -150,6 +171,10 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
+int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
+bool    fmIsInvertible(int32_t funcId);
 
 #ifdef __cplusplus
 }

include/libs/function/tudf.h

@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TUDF_H
+#define TDENGINE_TUDF_H
+
+
+#include <stdint.h>
+#include <stdbool.h>
+#include "tmsg.h"
+#include "tcommon.h"
+#include "function.h"
+#include "tdatablock.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UDF_LISTEN_PIPE_NAME_LEN 32
+#ifdef _WIN32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock"
+#else
+#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock."
+#endif
+#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
+
+//======================================================================================
+//begin API to taosd and qworker
+
+typedef struct SUdfColumnMeta {
+  int16_t type;
+  int32_t bytes;
+  uint8_t precision;
+  uint8_t scale;
+} SUdfColumnMeta;
+
+typedef struct SUdfColumnData {
+  int32_t numOfRows;
+  int32_t rowsAlloc;
+  union {
+    struct {
+      int32_t nullBitmapLen;
+      char   *nullBitmap;
+      int32_t dataLen;
+      char   *data;
+    } fixLenCol;
+
+    struct {
+      int32_t varOffsetsLen;
+      int32_t   *varOffsets;
+      int32_t payloadLen;
+      char   *payload;
+      int32_t payloadAllocLen;
+    } varLenCol;
+  };
+} SUdfColumnData;
+
+
+typedef struct SUdfColumn {
+  SUdfColumnMeta colMeta;
+  bool           hasNull;
+  SUdfColumnData colData;
+} SUdfColumn;
+
+typedef struct SUdfDataBlock {
+  int32_t numOfRows;
+  int32_t numOfCols;
+  SUdfColumn **udfCols;
+} SUdfDataBlock;
+
+typedef struct SUdfInterBuf {
+  int32_t bufLen;
+  char* buf;
+  int8_t numOfResult; //zero or one
+} SUdfInterBuf;
+typedef void *UdfcFuncHandle;
+
+//low level APIs
+/**
+ * setup udf
+ * @param udf, in
+ * @param funcHandle, out
+ * @return error code
+ */
+int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle);
+// output: interBuf
+int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
+// input: block, state
+// output: newState
+int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
+// input: interBuf
+// output: resultData
+int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
+// input: interbuf1, interbuf2
+// output: resultBuf
+int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
+// input: block
+// output: resultData
+int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+/**
+ * tearn down udf
+ * @param handle
+ * @return
+ */
+int32_t doTeardownUdf(UdfcFuncHandle handle);
+
+void freeUdfInterBuf(SUdfInterBuf *buf);
+
+//high level APIs
+bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
+int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
+
+int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+
+int32_t cleanUpUdfs();
+// end API to taosd and qworker
+//=============================================================================================================================
+// begin API to UDF writer.
+
+// dynamic lib init and destroy
+typedef int32_t (*TUdfInitFunc)();
+typedef int32_t (*TUdfDestroyFunc)();
+
+//TODO: add API to check function arguments type, number etc.
+
+#define UDF_MEMORY_EXP_GROWTH 1.5
+
+#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1)
+#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row))))
+#define udfColDataSetNull_f(pColumn, row)                                                \
+  do {                                                                                   \
+    BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \
+  } while (0)
+
+#define udfColDataSetNotNull_f(pColumn, r_)                                               \
+  do {                                                                                    \
+    BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_)));   \
+  } while (0)
+#define udfColDataSetNull_var(pColumn, row)  ((pColumn->colData.varLenCol.varOffsets)[row] = -1)
+
+
+static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) {
+  if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
+    return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row];
+  } else {
+    return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row;
+  }
+}
+
+static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) {
+  if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
+    if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) {
+      if (udfColDataIsNull_var(pColumn, row)) {
+        return true;
+      }
+      char* data = udfColDataGetData(pColumn, row);
+      return (*data == TSDB_DATA_TYPE_NULL);
+    } else {
+      return udfColDataIsNull_var(pColumn, row);
+    }
+  } else {
+    return udfColDataIsNull_f(pColumn, row);
+  }
+}
+
+static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) {
+  SUdfColumnMeta *meta = &pColumn->colMeta;
+  SUdfColumnData *data = &pColumn->colData;
+
+  if (newCapacity== 0 || newCapacity <= data->rowsAlloc) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int allocCapacity = TMAX(data->rowsAlloc, 8);
+  while (allocCapacity < newCapacity) {
+    allocCapacity *= UDF_MEMORY_EXP_GROWTH;
+  }
+
+  if (IS_VAR_DATA_TYPE(meta->type)) {
+    char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity);
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    data->varLenCol.varOffsets = (int32_t*)tmp;
+    data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity;
+    // for payload, add data in udfColDataAppend
+  } else {
+    char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity));
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    data->fixLenCol.nullBitmap = tmp;
+    data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity);
+    if (meta->type == TSDB_DATA_TYPE_NULL) {
+      return TSDB_CODE_SUCCESS;
+    }
+
+    tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes);
+    if (tmp == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    data->fixLenCol.data = tmp;
+    data->fixLenCol.dataLen = allocCapacity* meta->bytes;
+  }
+
+  data->rowsAlloc = allocCapacity;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) {
+  udfColEnsureCapacity(pColumn, row+1);
+  if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
+    udfColDataSetNull_var(pColumn, row);
+  } else {
+    udfColDataSetNull_f(pColumn, row);
+  }
+  pColumn->hasNull = true;
+}
+
+static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) {
+  SUdfColumnMeta *meta = &pColumn->colMeta;
+  SUdfColumnData *data = &pColumn->colData;
+  udfColEnsureCapacity(pColumn, currentRow+1);
+  bool isVarCol = IS_VAR_DATA_TYPE(meta->type);
+  if (isNull) {
+      udfColDataSetNull(pColumn, currentRow);
+  } else {
+    if (!isVarCol) {
+      colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow);
+      memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes);
+    } else {
+      int32_t dataLen = varDataTLen(pData);
+      if (meta->type == TSDB_DATA_TYPE_JSON) {
+        if (*pData == TSDB_DATA_TYPE_NULL) {
+          dataLen = 0;
+        } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+          dataLen = varDataTLen(pData + CHAR_BYTES);
+        } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+          dataLen = LONG_BYTES;
+        } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+          dataLen = CHAR_BYTES;
+        }
+        dataLen += CHAR_BYTES;
+      }
+
+      if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) {
+        uint32_t newSize = data->varLenCol.payloadAllocLen;
+        if (newSize <= 1) {
+          newSize = 8;
+        }
+
+        while (newSize < data->varLenCol.payloadLen + dataLen) {
+          newSize = newSize * UDF_MEMORY_EXP_GROWTH;
+        }
+
+        char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize);
+        if (buf == NULL) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+
+        data->varLenCol.payload = buf;
+        data->varLenCol.payloadAllocLen = newSize;
+      }
+
+      uint32_t len = data->varLenCol.payloadLen;
+      data->varLenCol.varOffsets[currentRow] = len;
+
+      memcpy(data->varLenCol.payload + len, pData, dataLen);
+      data->varLenCol.payloadLen += dataLen;
+    }
+  }
+  data->numOfRows = TMAX(currentRow + 1, data->numOfRows);
+  return 0;
+}
+
+typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
+
+typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
+typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
+typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
+
+
+// end API to UDF writer
+//=======================================================================================================================
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TUDF_H

include/libs/index/index.h

@@ -17,6 +17,7 @@
 #define _TD_INDEX_H_
 
 #include "os.h"
+#include "taoserror.h"
 #include "tarray.h"
 
 #ifdef __cplusplus
@@ -41,11 +42,23 @@ typedef enum {
   UPDATE_VALUE,  // update index column value
   ADD_INDEX,     // add index on specify column
   DROP_INDEX,    // drop existed index
-  DROP_SATBLE    // drop stable
+  DROP_SATBLE,   // drop stable
+  DEFAULT        // query
 } SIndexOperOnColumn;
 
-typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
+typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType;
-typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
+typedef enum {
+  QUERY_TERM = 0,
+  QUERY_PREFIX,
+  QUERY_SUFFIX,
+  QUERY_REGEX,
+  QUERY_LESS_THAN,
+  QUERY_LESS_EQUAL,
+  QUERY_GREATER_THAN,
+  QUERY_GREATER_EQUAL,
+  QUERY_RANGE,
+  QUERY_MAX
+} EIndexQueryType;
 
 /*
  * create multi query

include/libs/nodes/cmdnodes.h

@@ -30,24 +30,25 @@ extern "C" {
 
 typedef struct SDatabaseOptions {
   ENodeType   type;
-  SValueNode* pNumOfBlocks;
+  int32_t     buffer;
-  SValueNode* pCacheBlockSize;
+  int8_t      cachelast;
-  SValueNode* pCachelast;
+  int8_t      compressionLevel;
-  SValueNode* pCompressionLevel;
+  int32_t     daysPerFile;
   SValueNode* pDaysPerFile;
-  SValueNode* pFsyncPeriod;
+  int32_t     fsyncPeriod;
-  SValueNode* pMaxRowsPerBlock;
+  int32_t     maxRowsPerBlock;
-  SValueNode* pMinRowsPerBlock;
+  int32_t     minRowsPerBlock;
   SNodeList*  pKeep;
-  SValueNode* pPrecision;
+  int32_t     keep[3];
-  SValueNode* pQuorum;
+  int32_t     pages;
-  SValueNode* pReplica;
+  int32_t     pagesize;
-  SValueNode* pTtl;
+  char        precisionStr[3];
-  SValueNode* pWalLevel;
+  int8_t      precision;
-  SValueNode* pNumOfVgroups;
+  int8_t      replica;
-  SValueNode* pSingleStable;
+  int8_t      strict;
-  SValueNode* pStreamMode;
+  int8_t      walLevel;
-  SValueNode* pStrict;
+  int32_t     numOfVgroups;
+  int8_t      singleStable;
   SNodeList*  pRetentions;
 } SDatabaseOptions;
 
@@ -77,13 +78,12 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType  type;
-  SNodeList* pKeep;
+  char       comment[TSDB_STB_COMMENT_LEN];
-  SValueNode* pTtl;
+  int32_t    delay;
-  SValueNode* pComments;
+  float      filesFactor;
+  SNodeList* pRollupFuncs;
+  int32_t    ttl;
   SNodeList* pSma;
-  SNodeList* pFuncs;
-  SValueNode* pFilesFactor;
-  SValueNode* pDelay;
 } STableOptions;
 
 typedef struct SColumnDefNode {
@@ -201,10 +201,7 @@ typedef struct SShowCreatStmt {
   char      tableName[TSDB_TABLE_NAME_LEN];
 } SShowCreatStmt;
 
-typedef enum EIndexType {
+typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType;
-  INDEX_TYPE_SMA = 1,
-  INDEX_TYPE_FULLTEXT
-} EIndexType;
 
 typedef struct SIndexOptions {
   ENodeType  type;
@@ -241,12 +238,20 @@ typedef struct SDropComponentNodeStmt {
   int32_t   dnodeId;
 } SDropComponentNodeStmt;
 
+typedef struct STopicOptions {
+  ENodeType type;
+  bool      withTable;
+  bool      withSchema;
+  bool      withTag;
+} STopicOptions;
+
 typedef struct SCreateTopicStmt {
   ENodeType      type;
   char           topicName[TSDB_TABLE_NAME_LEN];
   char           subscribeDbName[TSDB_DB_NAME_LEN];
   bool           ignoreExists;
   SNode*         pQuery;
+  STopicOptions* pOptions;
 } SCreateTopicStmt;
 
 typedef struct SDropTopicStmt {
@@ -295,6 +300,39 @@ typedef struct SDropStreamStmt {
   bool      ignoreNotExists;
 } SDropStreamStmt;
 
+typedef struct SCreateFunctionStmt {
+  ENodeType type;
+  bool      ignoreExists;
+  char      funcName[TSDB_FUNC_NAME_LEN];
+  bool      isAgg;
+  char      libraryPath[PATH_MAX];
+  SDataType outputDt;
+  int32_t   bufSize;
+} SCreateFunctionStmt;
+
+typedef struct SDropFunctionStmt {
+  ENodeType type;
+  char      funcName[TSDB_FUNC_NAME_LEN];
+  bool      ignoreNotExists;
+} SDropFunctionStmt;
+
+#define PRIVILEGE_TYPE_MASK(n) (1 << n)
+
+#define PRIVILEGE_TYPE_ALL   PRIVILEGE_TYPE_MASK(0)
+#define PRIVILEGE_TYPE_READ  PRIVILEGE_TYPE_MASK(1)
+#define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2)
+
+#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
+
+typedef struct SGrantStmt {
+  ENodeType type;
+  char      userName[TSDB_USER_LEN];
+  char      dbName[TSDB_DB_NAME_LEN];
+  int64_t   privileges;
+} SGrantStmt;
+
+typedef SGrantStmt SRevokeStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -28,7 +28,8 @@ extern "C" {
 #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0)
 
 #define FOREACH(node, list)                                       \
-  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
+  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
+       (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
 
 #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
 
@@ -44,12 +45,25 @@ extern "C" {
 #define ERASE_NODE(list) cell = nodesListErase((list), cell)
 
 #define FORBOTH(node1, list1, node2, list2)                                               \
-  for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \
+  for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL),                      \
-    (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
+                  *cell2 = (NULL != (list2) ? (list2)->pHead : NULL);                     \
+       (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)),            \
+                  (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), \
+                  (node1 != NULL && node2 != NULL);                                       \
        cell1 = cell1->pNext, cell2 = cell2->pNext)
 
+#define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode)
+#define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode)
+
 #define FOREACH_FOR_REWRITE(node, list)                           \
-  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
+  for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
+       (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
+
+#define DESTORY_LIST(list)  \
+  do {                      \
+    nodesDestroyList(list); \
+    list = NULL;            \
+  } while (0)
 
 typedef enum ENodeType {
   // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
@@ -81,6 +95,7 @@ typedef enum ENodeType {
   QUERY_NODE_INDEX_OPTIONS,
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
+  QUERY_NODE_TOPIC_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
@@ -128,12 +143,15 @@ typedef enum ENodeType {
   QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
   QUERY_NODE_SPLIT_VGROUP_STMT,
   QUERY_NODE_SYNCDB_STMT,
+  QUERY_NODE_GRANT_STMT,
+  QUERY_NODE_REVOKE_STMT,
   QUERY_NODE_SHOW_DNODES_STMT,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
   QUERY_NODE_SHOW_SNODES_STMT,
   QUERY_NODE_SHOW_BNODES_STMT,
+  QUERY_NODE_SHOW_CLUSTER_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
@@ -146,7 +164,6 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_TOPICS_STMT,
   QUERY_NODE_SHOW_CONSUMERS_STMT,
   QUERY_NODE_SHOW_SUBSCRIBES_STMT,
-  QUERY_NODE_SHOW_TRANS_STMT,
   QUERY_NODE_SHOW_SMAS_STMT,
   QUERY_NODE_SHOW_CONFIGS_STMT,
   QUERY_NODE_SHOW_CONNECTIONS_STMT,
@@ -158,8 +175,11 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
   QUERY_NODE_SHOW_CREATE_TABLE_STMT,
   QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TRANSACTIONS_STMT,
   QUERY_NODE_KILL_CONNECTION_STMT,
   QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_QUERY,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -169,6 +189,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
   QUERY_NODE_LOGIC_PLAN_EXCHANGE,
   QUERY_NODE_LOGIC_PLAN_WINDOW,
+  QUERY_NODE_LOGIC_PLAN_FILL,
   QUERY_NODE_LOGIC_PLAN_SORT,
   QUERY_NODE_LOGIC_PLAN_PARTITION,
   QUERY_NODE_LOGIC_SUBPLAN,
@@ -186,6 +207,8 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
   QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_FILL,
   QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_PARTITION,
@@ -217,6 +240,7 @@ typedef struct SNodeList {
 
 #define SNodeptr void*
 
+int32_t  nodesNodeSize(ENodeType type);
 SNodeptr nodesMakeNode(ENodeType type);
 void     nodesDestroyNode(SNodeptr pNode);
 
@@ -235,12 +259,7 @@ void nodesDestroyList(SNodeList* pList);
 // Only clear the linked list structure, without releasing the elements inside
 void nodesClearList(SNodeList* pList);
 
-typedef enum EDealRes {
+typedef enum EDealRes { DEAL_RES_CONTINUE = 1, DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, DEAL_RES_END } EDealRes;
-  DEAL_RES_CONTINUE = 1,
-  DEAL_RES_IGNORE_CHILD,
-  DEAL_RES_ERROR,
-  DEAL_RES_END
-} EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
 void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext);

include/libs/nodes/plannodes.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 #include "tname.h"
 
 typedef struct SLogicNode {
@@ -33,20 +33,16 @@ typedef struct SLogicNode {
   int32_t            optimizedFlag;
 } SLogicNode;
 
-typedef enum EScanType {
+typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
-  SCAN_TYPE_TAG,
-  SCAN_TYPE_TABLE,
-  SCAN_TYPE_SYSTEM_TABLE,
-  SCAN_TYPE_STREAM
-} EScanType;
 
 typedef struct SScanLogicNode {
   SLogicNode         node;
   SNodeList*         pScanCols;
+  SNodeList*         pScanPseudoCols;
   struct STableMeta* pMeta;
   SVgroupsInfo*      pVgroupList;
   EScanType          scanType;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t            scanSeq[2];  // first is scan count, and second is reverse scan count
   STimeWindow        scanRange;
   SName              tableName;
   bool               showRewrite;
@@ -95,11 +91,7 @@ typedef struct SExchangeLogicNode {
   uint8_t    precision;
 } SExchangeLogicNode;
 
-typedef enum EWindowType {
+typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
-  WINDOW_TYPE_INTERVAL = 1,
-  WINDOW_TYPE_SESSION,
-  WINDOW_TYPE_STATE
-} EWindowType;
 
 typedef struct SWindowLogicNode {
   SLogicNode  node;
@@ -110,7 +102,6 @@ typedef struct SWindowLogicNode {
   int64_t     sliding;
   int8_t      intervalUnit;
   int8_t      slidingUnit;
-  SFillNode* pFill;
   int64_t     sessionGap;
   SNode*      pTspk;
   SNode*      pStateExpr;
@@ -118,6 +109,14 @@ typedef struct SWindowLogicNode {
   int64_t     watermark;
 } SWindowLogicNode;
 
+typedef struct SFillLogicNode {
+  SLogicNode  node;
+  EFillMode   mode;
+  SNode*      pWStartTs;
+  SNode*      pValues;  // SNodeListNode
+  STimeWindow timeRange;
+} SFillLogicNode;
+
 typedef struct SSortLogicNode {
   SLogicNode node;
   SNodeList* pSortKeys;
@@ -155,7 +154,6 @@ typedef struct SLogicSubplan {
 
 typedef struct SQueryLogicPlan {
   ENodeType  type;
-  int32_t totalLevel;
   SNodeList* pTopSubplans;
 } SQueryLogicPlan;
 
@@ -188,16 +186,13 @@ typedef struct SPhysiNode {
 typedef struct SScanPhysiNode {
   SPhysiNode node;
   SNodeList* pScanCols;
+  SNodeList* pScanPseudoCols;
   uint64_t   uid;  // unique id of the table
   int8_t     tableType;
-  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
-  int32_t count;         // repeat count
-  int32_t reverse;       // reverse scan count
   SName      tableName;
 } SScanPhysiNode;
 
 typedef SScanPhysiNode STagScanPhysiNode;
-typedef SScanPhysiNode SStreamScanPhysiNode;
 
 typedef struct SSystemTableScanPhysiNode {
   SScanPhysiNode scan;
@@ -208,7 +203,7 @@ typedef struct SSystemTableScanPhysiNode {
 
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t        scanSeq[2];  // first is scan count, and second is reverse scan count
   STimeWindow    scanRange;
   double         ratio;
   int32_t        dataRequired;
@@ -221,6 +216,7 @@ typedef struct STableScanPhysiNode {
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
+typedef STableScanPhysiNode SStreamScanPhysiNode;
 
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
@@ -234,10 +230,12 @@ typedef struct SProjectPhysiNode {
 typedef struct SJoinPhysiNode {
   SPhysiNode node;
   EJoinType  joinType;
-  SNode* pOnConditions; // in or out tuple ?
+  SNode*     pOnConditions;
   SNodeList* pTargets;
 } SJoinPhysiNode;
 
+typedef SJoinPhysiNode SSortMergeJoinPhysiNode;
+
 typedef struct SAggPhysiNode {
   SPhysiNode node;
   SNodeList* pExprs;  // these are expression list of group_by_clause and parameter expression of aggregate function
@@ -274,9 +272,19 @@ typedef struct SIntervalPhysiNode {
   int64_t          sliding;
   int8_t           intervalUnit;
   int8_t           slidingUnit;
-  SFillNode* pFill;
 } SIntervalPhysiNode;
 
+typedef SIntervalPhysiNode SStreamIntervalPhysiNode;
+
+typedef struct SFillPhysiNode {
+  SPhysiNode  node;
+  EFillMode   mode;
+  SNode*      pWStartTs;  // SColumnNode
+  SNode*      pValues;    // SNodeListNode
+  SNodeList*  pTargets;
+  STimeWindow timeRange;
+} SFillPhysiNode;
+
 typedef struct SMultiTableIntervalPhysiNode {
   SIntervalPhysiNode interval;
   SNodeList*         pPartitionKeys;
@@ -337,11 +345,7 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;     // data of the subplan flow into the datasink
 } SSubplan;
 
-typedef enum EExplainMode {
+typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
-  EXPLAIN_MODE_DISABLE = 1,
-  EXPLAIN_MODE_STATIC,
-  EXPLAIN_MODE_ANALYZE
-} EExplainMode;
 
 typedef struct SExplainInfo {
   EExplainMode mode;
@@ -355,7 +359,7 @@ typedef struct SQueryPlan {
   int32_t      numOfSubplans;
   SNodeList*   pSubplans;  // Element is SNodeListNode. The execution level of subplan, starting from 0.
   SExplainInfo explainInfo;
-  SNodeList* pPlaceholderValues;
+  SArray*      pPlaceholderValues;
 } SQueryPlan;
 
 void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

include/libs/nodes/querynodes.h

@@ -25,8 +25,10 @@ extern "C" {
 #include "tvariant.h"
 
 #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
-#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define TABLE_META_SIZE(pMeta) \
-#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
+  (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define VGROUPS_INFO_SIZE(pInfo) \
+  (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
 
 typedef struct SRawExprNode {
   ENodeType nodeType;
@@ -46,13 +48,11 @@ typedef struct SExprNode {
   ENodeType type;
   SDataType resType;
   char      aliasName[TSDB_COL_NAME_LEN];
-  SNodeList* pAssociationList;
+  char      userAlias[TSDB_COL_NAME_LEN];
+  SArray*   pAssociation;
 } SExprNode;
 
-typedef enum EColumnType {
+typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType;
-  COLUMN_TYPE_COLUMN = 1,
-  COLUMN_TYPE_TAG
-} EColumnType;
 
 typedef struct SColumnNode {
   SExprNode   node;  // QUERY_NODE_COLUMN
@@ -81,7 +81,6 @@ typedef struct SValueNode {
   char*     literal;
   bool      isDuration;
   bool      translate;
-  bool genByCalc;
   int16_t   placeholderNo;
   union {
     bool     b;
@@ -90,6 +89,7 @@ typedef struct SValueNode {
     double   d;
     char*    p;
   } datum;
+  int64_t typeData;
   char    unit;
 } SValueNode;
 
@@ -100,7 +100,6 @@ typedef struct SOperatorNode {
   SNode*        pRight;
 } SOperatorNode;
 
-
 typedef struct SLogicConditionNode {
   SExprNode           node;  // QUERY_NODE_LOGIC_CONDITION
   ELogicConditionType condType;
@@ -119,6 +118,7 @@ typedef struct SFunctionNode {
   int32_t    funcId;
   int32_t    funcType;
   SNodeList* pParameterList;
+  int32_t    udfBufSize;
 } SFunctionNode;
 
 typedef struct STableNode {
@@ -135,7 +135,7 @@ typedef struct SRealTableNode {
   STableNode         table;  // QUERY_NODE_REAL_TABLE
   struct STableMeta* pMeta;
   SVgroupsInfo*      pVgroupList;
-  char useDbName[TSDB_DB_NAME_LEN];
+  char               qualDbName[TSDB_DB_NAME_LEN];  // SHOW qualDbName.TABLES
   double             ratio;
 } SRealTableNode;
 
@@ -144,9 +144,7 @@ typedef struct STempTableNode {
   SNode*     pSubquery;
 } STempTableNode;
 
-typedef enum EJoinType {
+typedef enum EJoinType { JOIN_TYPE_INNER = 1 } EJoinType;
-  JOIN_TYPE_INNER = 1
-} EJoinType;
 
 typedef struct SJoinTableNode {
   STableNode table;  // QUERY_NODE_JOIN_TABLE
@@ -156,9 +154,7 @@ typedef struct SJoinTableNode {
   SNode*     pOnCond;
 } SJoinTableNode;
 
-typedef enum EGroupingSetType {
+typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
-  GP_TYPE_NORMAL = 1
-} EGroupingSetType;
 
 typedef struct SGroupingSetNode {
   ENodeType        type;  // QUERY_NODE_GROUPING_SET
@@ -166,16 +162,9 @@ typedef struct SGroupingSetNode {
   SNodeList*       pParameterList;
 } SGroupingSetNode;
 
-typedef enum EOrder {
+typedef enum EOrder { ORDER_ASC = 1, ORDER_DESC } EOrder;
-  ORDER_ASC = 1,
-  ORDER_DESC
-} EOrder;
 
-typedef enum ENullOrder {
+typedef enum ENullOrder { NULL_ORDER_DEFAULT = 1, NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder;
-  NULL_ORDER_DEFAULT = 1,
-  NULL_ORDER_FIRST,
-  NULL_ORDER_LAST
-} ENullOrder;
 
 typedef struct SOrderByExprNode {
   ENodeType  type;  // QUERY_NODE_ORDER_BY_EXPR
@@ -224,33 +213,37 @@ typedef struct SFillNode {
   ENodeType   type;  // QUERY_NODE_FILL
   EFillMode   mode;
   SNode*      pValues;    // SNodeListNode
+  SNode*      pWStartTs;  // _wstartts pseudo column
+  STimeWindow timeRange;
 } SFillNode;
 
 typedef struct SSelectStmt {
   ENodeType   type;  // QUERY_NODE_SELECT_STMT
   bool        isDistinct;
-  SNodeList* pProjectionList; // SNode
+  SNodeList*  pProjectionList;
   SNode*      pFromTable;
   SNode*      pWhere;
-  SNodeList* pPartitionByList; // SNode
+  SNodeList*  pPartitionByList;
   SNode*      pWindow;
   SNodeList*  pGroupByList;  // SGroupingSetNode
   SNode*      pHaving;
   SNodeList*  pOrderByList;  // SOrderByExprNode
-  SNode* pLimit;
+  SLimitNode* pLimit;
-  SNode* pSlimit;
+  SLimitNode* pSlimit;
   char        stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t     precision;
   bool        isEmptyResult;
+  bool        isTimeOrderQuery;
+  bool        hasAggFuncs;
+  bool        hasRepeatScanFuncs;
 } SSelectStmt;
 
-typedef enum ESetOperatorType {
+typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
-  SET_OP_TYPE_UNION_ALL = 1
-} ESetOperatorType;
 
 typedef struct SSetOperator {
   ENodeType        type;  // QUERY_NODE_SET_OPERATOR
   ESetOperatorType opType;
+  SNodeList*       pProjectionList;
   SNode*           pLeft;
   SNode*           pRight;
   SNodeList*       pOrderByList;  // SOrderByExprNode
@@ -269,7 +262,6 @@ typedef enum ESqlClause {
   SQL_CLAUSE_ORDER_BY
 } ESqlClause;
 
-
 typedef enum {
   PAYLOAD_TYPE_KV = 0,
   PAYLOAD_TYPE_RAW = 1,
@@ -304,13 +296,50 @@ typedef struct SExplainStmt {
   SNode*           pQuery;
 } SExplainStmt;
 
+typedef struct SCmdMsgInfo {
+  int16_t msgType;
+  SEpSet  epSet;
+  void*   pMsg;
+  int32_t msgLen;
+  void*   pExtension;  // todo remove it soon
+} SCmdMsgInfo;
+
+typedef enum EQueryExecMode {
+  QUERY_EXEC_MODE_LOCAL = 1,
+  QUERY_EXEC_MODE_RPC,
+  QUERY_EXEC_MODE_SCHEDULE,
+  QUERY_EXEC_MODE_EMPTY_RESULT
+} EQueryExecMode;
+
+typedef struct SQuery {
+  ENodeType      type;
+  EQueryExecMode execMode;
+  bool           haveResultSet;
+  SNode*         pRoot;
+  int32_t        numOfResCols;
+  SSchema*       pResSchema;
+  int8_t         precision;
+  SCmdMsgInfo*   pCmdMsg;
+  int32_t        msgType;
+  SArray*        pDbList;
+  SArray*        pTableList;
+  bool           showRewrite;
+  int32_t        placeholderNum;
+  SArray*        pPlaceholderValues;
+  SNode*         pPrepareRoot;
+} SQuery;
+
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
+typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
+                            SNodeList** pCols);
 
 typedef bool (*FFuncClassifier)(int32_t funcId);
-int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);
+int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs);
+
+int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes);
 
 bool nodesIsExprNode(const SNode* pNode);
 
@@ -318,11 +347,13 @@ bool nodesIsUnaryOp(const SOperatorNode* pOp);
 bool nodesIsArithmeticOp(const SOperatorNode* pOp);
 bool nodesIsComparisonOp(const SOperatorNode* pOp);
 bool nodesIsJsonOp(const SOperatorNode* pOp);
+bool nodesIsRegularOp(const SOperatorNode* pOp);
 
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
 void*   nodesGetValueFromNode(SValueNode* pNode);
+int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value);
 char*   nodesGetStrValueFromNode(SValueNode* pNode);
 char*   getFillModeString(EFillMode mode);
 void    valueNodeToVariant(const SValueNode* pNode, SVariant* pVal);

include/libs/parser/parser.h

@@ -20,8 +20,16 @@
 extern "C" {
 #endif
 
+#include "query.h"
 #include "querynodes.h"
 
+typedef struct SStmtCallback {
+  TAOS_STMT* pStmt;
+  int32_t (*getTbNameFn)(TAOS_STMT*, char**);
+  int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, char*, bool, SHashObj*, SHashObj*);
+  int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
+} SStmtCallback;
+
 typedef struct SParseContext {
   uint64_t         requestId;
   int32_t          acctId;
@@ -34,43 +42,45 @@ typedef struct SParseContext {
   char*            pMsg;    // extended error message if exists to help identifying the problem in sql statement.
   int32_t          msgLen;  // max length of the msg
   struct SCatalog* pCatalog;
+  SStmtCallback*   pStmtCb;
+  const char*      pUser;
+  bool             isSuperUser;
 } SParseContext;
 
-typedef struct SCmdMsgInfo {
+int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
-  int16_t msgType;
+bool    isInsertSql(const char* pStr, size_t length);
-  SEpSet epSet;
-  void* pMsg;
-  int32_t msgLen;
-  void* pExtension;  // todo remove it soon
-} SCmdMsgInfo;
-
-typedef enum EQueryExecMode {
-  QUERY_EXEC_MODE_LOCAL = 1,
-  QUERY_EXEC_MODE_RPC,
-  QUERY_EXEC_MODE_SCHEDULE,
-  QUERY_EXEC_MODE_EMPTY_RESULT
-} EQueryExecMode;
-
-typedef struct SQuery {
-  EQueryExecMode execMode;
-  bool haveResultSet;
-  SNode* pRoot;
-  int32_t numOfResCols;
-  SSchema* pResSchema;
-  int8_t   precision;
-  SCmdMsgInfo* pCmdMsg;
-  int32_t msgType;
-  SArray* pDbList;
-  SArray* pTableList;
-  bool showRewrite;
-} SQuery;
-
-int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
 
 void qDestroyQuery(SQuery* pQueryNode);
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
 
+int32_t     qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
+int32_t     qResetStmtDataBlock(void* block, bool keepBuf);
+int32_t     qCloneStmtDataBlock(void** pDst, void* pSrc);
+void        qFreeStmtDataBlock(void* pDataBlock);
+int32_t     qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId);
+void        qDestroyStmtDataBlock(void* pBlock);
+STableMeta* qGetTableMetaInDataBlock(void* pDataBlock);
+
+int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
+int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
+int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
+int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
+                                int32_t rowNum);
+int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind,
+                           char* msgBuf, int32_t msgBufLen);
+void    destroyBoundColumnInfo(void* pBoundInfo);
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
+                     int32_t msgBufLen);
+
+void*   smlInitHandle(SQuery* pQuery);
+void    smlDestroyHandle(void* pHandle);
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+                    char* tableName, char* msgBuf, int16_t msgBufLen);
+int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/planner/planner.h

@@ -30,10 +30,12 @@ typedef struct SPlanContext {
   SNode*   pAstRoot;
   bool     topicQuery;
   bool     streamQuery;
+  bool     rSmaQuery;
   bool     showRewrite;
   int8_t   triggerType;
   int64_t  watermark;
-  bool isStmtQuery;
+  char*    pMsg;
+  int32_t  msgLen;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.
@@ -45,9 +47,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-typedef TAOS_MULTI_BIND TAOS_BIND_v2; // todo remove
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams);
-
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
 int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);

include/libs/qcom/query.h

@@ -24,6 +24,7 @@ extern "C" {
 #include "thash.h"
 #include "tlog.h"
 #include "tmsg.h"
+#include "tmsgcb.h"
 
 typedef enum {
   JOB_TASK_STATUS_NULL = 0,
@@ -50,10 +51,12 @@ typedef struct STableComInfo {
 } STableComInfo;
 
 typedef struct SIndexMeta {
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 
 } SIndexMeta;
 
-
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -129,12 +132,6 @@ typedef struct SMsgSendInfo {
   SDataBuf             msgInfo;
 } SMsgSendInfo;
 
-typedef struct SQueryNodeAddr {
-  int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epSet;
-} SQueryNodeAddr;
-
-
 typedef struct SQueryNodeStat {
   int32_t tableNum;  // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;
@@ -151,7 +148,8 @@ int32_t cleanupTaskQueue();
  */
 int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
 
-int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx);
+int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo,
+                                bool persistHandle, void* ctx);
 
 /**
  * Asynchronously send message to server, after the response received, the callback will be incured.
@@ -184,62 +182,87 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
+#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
-#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
+  ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
+#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
+  ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
-#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
+#define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
+  (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
+   NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
 
-#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
+#define NEED_SCHEDULER_RETRY_ERROR(_code) \
+  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
 
 #define REQUEST_MAX_TRY_TIMES 5
 
 #define qFatal(...)                                                                           \
   do {                                                                                        \
     if (qDebugFlag & DEBUG_FATAL) {                                                           \
-      taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                         \
   } while (0)
 #define qError(...)                                                                           \
   do {                                                                                        \
     if (qDebugFlag & DEBUG_ERROR) {                                                           \
-      taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                         \
   } while (0)
 #define qWarn(...)                                                                          \
   do {                                                                                      \
     if (qDebugFlag & DEBUG_WARN) {                                                          \
-      taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                       \
   } while (0)
 #define qInfo(...)                                                                     \
   do {                                                                                 \
     if (qDebugFlag & DEBUG_INFO) {                                                     \
-      taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                  \
   } while (0)
 #define qDebug(...)                                                                     \
   do {                                                                                  \
     if (qDebugFlag & DEBUG_DEBUG) {                                                     \
-      taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                   \
   } while (0)
 #define qTrace(...)                                                                     \
   do {                                                                                  \
     if (qDebugFlag & DEBUG_TRACE) {                                                     \
-      taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
+      taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                   \
   } while (0)
 #define qDebugL(...)                                                                           \
   do {                                                                                         \
     if (qDebugFlag & DEBUG_DEBUG) {                                                            \
-      taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+      taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
     }                                                                                          \
   } while (0)
 
-#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define QRY_ERR_RET(c)                \
-#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+  do {                                \
-#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+    int32_t _code = c;                \
-
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+      return _code;                   \
+    }                                 \
+  } while (0)
+#define QRY_RET(c)                    \
+  do {                                \
+    int32_t _code = c;                \
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+    }                                 \
+    return _code;                     \
+  } while (0)
+#define QRY_ERR_JRET(c)              \
+  do {                               \
+    code = c;                        \
+    if (code != TSDB_CODE_SUCCESS) { \
+      terrno = code;                 \
+      goto _return;                  \
+    }                                \
+  } while (0)
 
 #ifdef __cplusplus
 }

include/libs/scalar/scalar.h

@@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/scheduler/scheduler.h

@@ -56,6 +56,7 @@ typedef struct SQueryResult {
   uint64_t        numOfRows;
   int32_t         msgSize;
   char           *msg;
+  void           *res;
 } SQueryResult;
 
 typedef struct STaskInfo {
@@ -71,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
  * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes);
+int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes);
 
 /**
  * Process the query job, generated according to the query physical plan.

include/libs/stream/tstream.h

@@ -13,9 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "os.h"
 #include "tdatablock.h"
 #include "tmsg.h"
 #include "tmsgcb.h"
+#include "tqueue.h"
 #include "trpc.h"
 
 #ifdef __cplusplus
@@ -25,9 +27,26 @@ extern "C" {
 #ifndef _TSTREAM_H_
 #define _TSTREAM_H_
 
+typedef struct SStreamTask SStreamTask;
+
 enum {
-  STREAM_TASK_STATUS__RUNNING = 1,
+  TASK_STATUS__IDLE = 1,
-  STREAM_TASK_STATUS__STOP,
+  TASK_STATUS__EXECUTING,
+  TASK_STATUS__CLOSING,
+};
+
+enum {
+  TASK_INPUT_STATUS__NORMAL = 1,
+  TASK_INPUT_STATUS__BLOCKED,
+  TASK_INPUT_STATUS__RECOVER,
+  TASK_INPUT_STATUS__STOP,
+  TASK_INPUT_STATUS__FAILED,
+};
+
+enum {
+  TASK_OUTPUT_STATUS__NORMAL = 1,
+  TASK_OUTPUT_STATUS__WAIT,
+  TASK_OUTPUT_STATUS__BLOCKED,
 };
 
 enum {
@@ -35,23 +54,64 @@ enum {
   STREAM_CREATED_BY__SMA,
 };
 
-#if 0
-// pipe  -> fetch/pipe queue
-// merge -> merge      queue
-// write -> write      queue
 enum {
-  TASK_DISPATCH_MSG__SND_PIPE = 1,
+  STREAM_INPUT__DATA_SUBMIT = 1,
-  TASK_DISPATCH_MSG__SND_MERGE,
+  STREAM_INPUT__DATA_BLOCK,
-  TASK_DISPATCH_MSG__VND_PIPE,
+  STREAM_INPUT__CHECKPOINT,
-  TASK_DISPATCH_MSG__VND_MERGE,
-  TASK_DISPATCH_MSG__VND_WRITE,
 };
-#endif
 
 typedef struct {
-  int32_t nodeId;  // 0 for snode
+  int8_t type;
-  SEpSet  epSet;
+
-} SStreamTaskEp;
+  int32_t sourceVg;
+  int64_t sourceVer;
+
+  int32_t*    dataRef;
+  SSubmitReq* data;
+} SStreamDataSubmit;
+
+typedef struct {
+  int8_t type;
+
+  int32_t sourceVg;
+  int64_t sourceVer;
+
+  SArray* blocks;  // SArray<SSDataBlock*>
+} SStreamDataBlock;
+
+typedef struct {
+  int8_t type;
+} SStreamCheckpoint;
+
+static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
+  SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosMemoryCalloc(1, sizeof(SStreamDataSubmit));
+  if (pDataSubmit == NULL) return NULL;
+  pDataSubmit->data = pReq;
+  pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
+  if (pDataSubmit->data == NULL) goto FAIL;
+  *pDataSubmit->dataRef = 1;
+  return pDataSubmit;
+FAIL:
+  taosMemoryFree(pDataSubmit);
+  return NULL;
+}
+
+static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
+  //
+  atomic_add_fetch_32(pDataSubmit->dataRef, 1);
+}
+
+static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) {
+  int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1);
+  ASSERT(ref >= 0);
+  if (ref == 0) {
+    taosMemoryFree(pDataSubmit->data);
+    taosMemoryFree(pDataSubmit->dataRef);
+  }
+}
+
+int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
+void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
 
 typedef struct {
   void* inputHandle;
@@ -82,28 +142,30 @@ typedef struct {
   SUseDbRsp dbInfo;
 } STaskDispatcherShuffle;
 
+typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
+
 typedef struct {
-  int8_t reserved;
+  int64_t         stbUid;
+  SSchemaWrapper* pSchemaWrapper;
   // not applicable to encoder and decoder
+  void*     vnode;
+  FTbSink*  tbSinkFunc;
+  STSchema* pTSchema;
   SHashObj* pHash;  // groupId to tbuid
 } STaskSinkTb;
 
-typedef void FSmaHandle(void* vnode, int64_t smaId, const SArray* data);
+typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);
 
 typedef struct {
   int64_t smaId;
   // following are not applicable to encoder and decoder
-  FSmaHandle* smaHandle;
+  FSmaSink* smaSink;
 } STaskSinkSma;
 
 typedef struct {
   int8_t reserved;
 } STaskSinkFetch;
 
-typedef struct {
-  int8_t reserved;
-} STaskSinkShow;
-
 enum {
   TASK_SOURCE__SCAN = 1,
   TASK_SOURCE__PIPE,
@@ -128,12 +190,17 @@ enum {
   TASK_SINK__TABLE,
   TASK_SINK__SMA,
   TASK_SINK__FETCH,
-  TASK_SINK__SHOW,
 };
 
-typedef struct {
+enum {
+  TASK_INPUT_TYPE__SUMBIT_BLOCK = 1,
+  TASK_INPUT_TYPE__DATA_BLOCK,
+};
+
+struct SStreamTask {
   int64_t streamId;
   int32_t taskId;
+  int8_t  inputType;
   int8_t  status;
 
   int8_t  sourceType;
@@ -155,7 +222,6 @@ typedef struct {
     STaskSinkTb    tbSink;
     STaskSinkSma   smaSink;
     STaskSinkFetch fetchSink;
-    STaskSinkShow  showSink;
   };
 
   // dispatch
@@ -165,14 +231,21 @@ typedef struct {
     STaskDispatcherShuffle shuffleDispatcher;
   };
 
+  int8_t inputStatus;
+  int8_t outputStatus;
+
+  STaosQueue* inputQ;
+  STaosQall*  inputQAll;
+  STaosQueue* outputQ;
+  STaosQall*  outputQAll;
+
   // application storage
   void* ahandle;
-
+};
-} SStreamTask;
 
 SStreamTask* tNewSStreamTask(int64_t streamId);
-int32_t      tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask);
+int32_t      tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
-int32_t      tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask);
+int32_t      tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
 void         tFreeSStreamTask(SStreamTask* pTask);
 
 typedef struct {
@@ -206,8 +279,16 @@ typedef struct {
   SArray* res;  // SArray<SSDataBlock>
 } SStreamSinkReq;
 
+int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
+int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
+int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
+
 int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
 
+int32_t streamTaskRun(SStreamTask* pTask);
+
+int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/stream/tstreamUpdate.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TSTREAMUPDATE_H_
+#define _TSTREAMUPDATE_H_
+
+#include "taosdef.h"
+#include "tarray.h"
+#include "tmsg.h"
+#include "tscalablebf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SUpdateInfo {
+  SArray *pTsBuckets;
+  uint64_t numBuckets;
+  SArray *pTsSBFs;
+  uint64_t numSBFs;
+  int64_t interval;
+  int64_t watermark;
+  TSKEY minTS;
+} SUpdateInfo;
+
+SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
+SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
+bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
+void updateInfoDestroy(SUpdateInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef _TSTREAMUPDATE_H_ */

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
+//#include <tdatablock.h>
-#include "taosdef.h"
+#include "cJSON.h"
-#include "trpc.h"
+#include "tdef.h"
-#include "wal.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
+  uint16_t nodePort;
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
+
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
+
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
+
 typedef struct SSyncFSM {
   void* data;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-                     ESyncState state);
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
-
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
-
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
-
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN 0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
+struct SWal;
-typedef struct SStateMgr {
+typedef struct SWal SWal;
-  void* data;
 
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+struct SEpSet;
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+typedef struct SEpSet SEpSet;
-
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,25 +140,38 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
+int32_t     syncInit();
+void        syncCleanUp();
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
+void        syncStartStandBy(int64_t rid);
+void        syncStop(int64_t rid);
+int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
+ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
+void        syncGetEpSet(int64_t rid, SEpSet* pEpSet);
+int32_t     syncGetVgId(int64_t rid);
+
+typedef enum {
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
+
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+bool syncEnvIsStart();
+
+extern int32_t sDebugFlag;
+
+//-----------------------------------------
 struct SSyncNode;
 typedef struct SSyncNode SSyncNode;
 
-int32_t syncInit();
+struct SSyncBuffer;
-void    syncCleanUp();
+typedef struct SSyncBuffer SSyncBuffer;
-
+//-----------------------------------------
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
-void       syncStop(int64_t rid);
-int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-ESyncState syncGetMyRole(int64_t rid);
-
-// propose with sequence number, to implement linearizable semantics
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
-
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-
-extern int32_t sDebugFlag;
 
 const char* syncStr(ESyncState state);
 

include/libs/sync/syncTools.h

@@ -0,0 +1,414 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -26,42 +26,52 @@ extern "C" {
 
 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
+#define IsReq(pMsg)      (pMsg->msgType & 1U)
 
 extern int tsRpcHeadSize;
 
-typedef struct SRpcConnInfo {
+typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
-  uint32_t serverIp;
   char     user[TSDB_USER_LEN];
 } SRpcConnInfo;
 
+typedef struct SRpcHandleInfo {
+  // rpc info
+  void   *handle;         // rpc handle returned to app
+  int64_t refId;          // refid, used by server
+  int32_t noResp;         // has response or not(default 0, 0: resp, 1: no resp);
+  int32_t persistHandle;  // persist handle or not
+
+  // app info
+  void *ahandle;  // app handle set by client
+  void *wrapper;  // wrapper handle
+  void *node;     // node mgmt handle
+
+  // resp info
+  void   *rsp;
+  int32_t rspLen;
+} SRpcHandleInfo;
+
 typedef struct SRpcMsg {
   tmsg_t         msgType;
   void          *pCont;
-  int     contLen;
+  int32_t        contLen;
   int32_t        code;
-  void *  handle;         // rpc handle returned to app
+  SRpcHandleInfo info;
-  void *  ahandle;        // app handle set by client
+  SRpcConnInfo   conn;
-  int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
-  int     persistHandle;  // persist handle or not
-
 } SRpcMsg;
 
-typedef struct {
+typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
-  char     user[TSDB_USER_LEN];
-  uint32_t clientIp;
-  uint16_t clientPort;
-  SRpcMsg  rpcMsg;
-  int32_t  rspLen;
-  void    *pRsp;
-  void    *pNode;
-} SNodeMsg;
-
-typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
 typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+///
+// // SRpcMsg code
+// REDIERE,
+// NOT READY, EpSet
+typedef bool (*RpcRfp)(int32_t code);
 
 typedef struct SRpcInit {
+  char     localFqdn[TSDB_FQDN_LEN];
   uint16_t localPort;     // local port
   char *   label;         // for debug purpose
   int      numOfThreads;  // number of threads to handle connections
@@ -80,7 +90,10 @@ typedef struct SRpcInit {
   RpcCfp cfp;
 
   // call back to retrieve the client auth info, for server app only
-  RpcAfp afp;;
+  RpcAfp afp;
+
+  // user defined retry func
+  RpcRfp rfp;
 
   void *parent;
 } SRpcInit;

include/libs/wal/wal.h

@@ -72,8 +72,6 @@ extern "C" {
 #define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
 #define WAL_MAGIC        0xFAFBFCFDULL
 
-#define WAL_CUR_FAILED 1
-
 #pragma pack(push, 1)
 typedef enum {
   TAOS_WAL_NOLOG = 0,
@@ -92,7 +90,7 @@ typedef struct SWalReadHead {
   int8_t  headVer;
   int8_t  reserved;
   int16_t msgType;
-  int32_t len;
+  int32_t bodyLen;
   int64_t ingestTs;  // not implemented
   int64_t version;
 
@@ -192,7 +190,13 @@ int32_t walEndSnapshot(SWal *);
 SWalReadHandle *walOpenReadHandle(SWal *);
 void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
-int32_t         walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
+
+// only for tq usage
+// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
+void    walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
+int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
+int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
+int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
 
 // deprecated
 #if 0
@@ -204,6 +208,7 @@ int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readN
 int64_t walGetFirstVer(SWal *);
 int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);
+int64_t walGetCommittedVer(SWal *);
 
 #ifdef __cplusplus
 }

include/os/os.h

@@ -22,7 +22,6 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
-#include <semaphore.h>
 
 #include <regex.h>
 
@@ -51,7 +50,14 @@ extern "C" {
 #include <sys/prctl.h>
 #include <argp.h>
 #endif
+#else
 
+#include <time.h>
+#ifndef TD_USE_WINSOCK
+#include <winsock2.h>
+#else
+#include <winsock.h>
+#endif
 #endif
 
 #include <errno.h>

include/os/osAtomic.h

@@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, int32_t val);
+void   *atomic_add_fetch_ptr(void *ptr, void *val);
 int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, int32_t val);
+void   *atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
+void   *atomic_sub_fetch_ptr(void *ptr, void *val);
 int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
+void   *atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t  atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);

include/os/osDef.h

@@ -62,7 +62,7 @@ extern "C" {
   #define strncasecmp _strnicmp
   #define wcsncasecmp _wcsnicmp
   #define strtok_r strtok_s
-  #define snprintf _snprintf
+  // #define snprintf _snprintf
   #define in_addr_t unsigned long
 //  #define socklen_t int
 
@@ -191,7 +191,7 @@ extern "C" {
   #define threadlocal __declspec( thread )
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define PRIzu "ld"  
 #else
   #define PRIzu "zu"  

include/os/osDir.h

@@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr;
 
 
 void    taosRemoveDir(const char *dirname);
-bool    taosDirExist(char *dirname);
+bool    taosDirExist(const char *dirname);
 int32_t taosMkDir(const char *dirname);
 int32_t taosMulMkDir(const char *dirname);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
-int32_t taosRealPath(char *dirname, int32_t maxlen);
+int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
 char*   taosDirName(char *dirname);
 char*   taosDirEntryBaseName(char *dirname);
@@ -50,7 +50,7 @@ TdDirPtr      taosOpenDir(const char *dirname);
 TdDirEntryPtr taosReadDir(TdDirPtr pDir);
 bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
 char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
-int32_t       taosCloseDir(TdDirPtr pDir);
+int32_t       taosCloseDir(TdDirPtr *ppDir);
 
 #ifdef __cplusplus
 }

include/os/osEnv.h

@@ -34,6 +34,7 @@ extern int64_t         tsOpenMax;
 extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
+extern char*           tsProcPath;
 
 extern char configDir[];
 extern char tsDataDir[];

include/os/osFile.h

@@ -66,7 +66,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
 int32_t taosUmaskFile(int32_t maskVal);
  
 int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
-int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
+int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
 int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
 bool    taosCheckExistFile(const char *pathname);
  
@@ -80,6 +80,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
 void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
 
 int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf);
+int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf);
 
 int32_t taosEOFFile(TdFilePtr pFile);
  
@@ -93,7 +94,6 @@ void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix,
 
 int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
 
-void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
 bool taosValidFile(TdFilePtr pFile);
 
 int32_t taosGetErrorFile(TdFilePtr pFile);

include/os/osLz4.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   int32_t BUILDIN_CLZL(uint64_t val);
   int32_t BUILDIN_CLZ(uint32_t val);
   int32_t BUILDIN_CTZL(uint64_t val);

include/os/osMath.h

@@ -23,26 +23,23 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#define TSWAP(a, b)                              \
-
-  #define TSWAP(a, b, c) \
   do {                                           \
-      c __tmp = (c)(a); \
+    char *__tmp = taosMemoryMalloc(sizeof(a));   \
-      (a) = (c)(b);     \
+    memcpy(__tmp, &(a), sizeof(a));              \
-      (b) = __tmp;      \
+    memcpy(&(a), &(b), sizeof(a));               \
+    memcpy(&(b), __tmp, sizeof(a));              \
+    taosMemoryFree(__tmp);                       \
   } while (0)
+
+#ifdef WINDOWS
+
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
+  #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 
-  #define TSWAP(a, b, c)       \
-    do {                       \
-      __typeof(a) __tmp = (a); \
-      (a) = (b);               \
-      (b) = __tmp;             \
-    } while (0)
-
   #define TMAX(a, b)             \
     ({                           \
       __typeof(a) __a = (a);     \

include/os/osMemory.h

@@ -35,6 +35,7 @@ void *taosMemoryRealloc(void *ptr, int32_t size);
 void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(void *ptr);
 int32_t taosMemorySize(void *ptr);
+void taosPrintBackTrace();
 
 #define taosMemoryFreeClear(ptr)   \
   do {                             \

include/os/osSemaphore.h

@@ -23,17 +23,24 @@ extern "C" {
 #include <semaphore.h>
 
 #if defined(_TD_DARWIN_64)
+
 typedef struct tsem_s *tsem_t;
+
 int tsem_init(tsem_t *sem, int pshared, unsigned int value);
 int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
 int tsem_post(tsem_t *sem);
 int tsem_destroy(tsem_t *sem);
+
 #else
+
 #define tsem_t       sem_t
 #define tsem_init    sem_init
 int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
 #define tsem_post    sem_post
 #define tsem_destroy sem_destroy
+
 #endif
 
 #if defined(_TD_DARWIN_64)

include/os/osSocket.h

@@ -31,10 +31,31 @@
 #endif
 
 #if defined(WINDOWS)
-  #include "winsock2.h"
+#if BYTE_ORDER == LITTLE_ENDIAN
-  #include <WS2tcpip.h>
+#include <stdlib.h>
-  #include <winbase.h>
+#define htobe16(x) _byteswap_ushort(x)
-  #include <Winsock2.h>
+#define htole16(x) (x)
+#define be16toh(x) _byteswap_ushort(x)
+#define le16toh(x) (x)
+
+#define htobe32(x) _byteswap_ulong(x)
+#define htole32(x) (x)
+#define be32toh(x) _byteswap_ulong(x)
+#define le32toh(x) (x)
+
+#define htobe64(x) _byteswap_uint64(x)
+#define htole64(x) (x)
+#define be64toh(x) _byteswap_uint64(x)
+#define le64toh(x) (x)
+#else
+#error byte order not supported
+#endif
+
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
+
 #else
 #include <netinet/in.h>
 #include <sys/socket.h>
@@ -52,14 +73,10 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  #define htobe64 htonll
+typedef int socklen_t;
-#endif
-
-#if defined(WINDOWS)
 #define TAOS_EPOLL_WAIT_TIME 100
 typedef SOCKET eventfd_t;
 #define eventfd(a, b)      -1
-  typedef SOCKET EpollFd;
 #define EpollClose(pollFd) epoll_close(pollFd)
 #ifndef EPOLLWAKEUP
 #define EPOLLWAKEUP (1u << 29)
@@ -102,8 +119,6 @@ extern "C" {
 #define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
-
 typedef int32_t  SocketFd;
 typedef SocketFd EpollFd;
 
@@ -122,7 +137,8 @@ typedef struct TdEpoll *TdEpollPtr;
 int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr,
+                           int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -139,14 +155,18 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+void    taosWinSocketInit();
+
+int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
 
 TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+bool              taosValidIpAndPort(uint32_t ip, uint16_t port);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int32_t           taosKeepTcpAlive(TdSocketPtr pSocket);
-TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+TdSocketPtr       taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
 
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
 
 void        taosBlockSIGPIPE();
 uint32_t    taosGetIpv4FromFqdn(const char *);

include/os/osString.h

@@ -37,9 +37,10 @@ typedef int32_t TdUcs4;
     #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
     #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
     #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
+    #define strcasestr STR_CASE_STR_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define tstrdup(str) _strdup(str)
 #else
   #define tstrdup(str) strdup(str)
@@ -59,6 +60,8 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
+int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);
 int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);
@@ -67,6 +70,8 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
 int32_t taosWcharToMb(char *pStr, TdWchar wchar);
 int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
 
+char *taosStrCaseStr(const char *str, const char *pattern);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osSystem.h

@@ -41,9 +41,9 @@ void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
-void setTerminalMode();
+void    taosSetTerminalMode();
-int32_t getOldTerminalMode();
+int32_t taosGetOldTerminalMode();
-void resetTerminalMode();
+void    taosResetTerminalMode();
 
 #ifdef __cplusplus
 }

include/os/osThread.h

@@ -22,9 +22,12 @@
 extern "C" {
 #endif
 
+#ifndef WINDOWS
 #ifndef __USE_XOPEN2K
+#define TD_USE_SPINLOCK_AS_MUTEX
 typedef pthread_mutex_t pthread_spinlock_t;
 #endif
+#endif
 
 typedef pthread_t TdThread;
 typedef pthread_spinlock_t TdThreadSpinlock;
@@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey;
 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop pthread_cleanup_pop
 
+
+#ifdef WINDOWS
+#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
+#else
+#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#endif
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
 void taosThreadTestCancel(void);
-int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
+void taosThreadClear(TdThread *thread);
-int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -20,8 +20,6 @@
 extern "C" {
 #endif
 
-#include <time.h>
-
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -34,7 +32,7 @@ extern "C" {
     #define mktime MKTIME_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define CLOCK_REALTIME 	0
 
@@ -58,6 +56,8 @@ extern "C" {
 
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
+int32_t taosClockGetTime(int clock_id, struct timespec *pTS);
+
 //@return timestamp in second
 int32_t taosGetTimestampSec();
 
@@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() {
 //@return timestamp in nanosecond
 static FORCE_INLINE int64_t taosGetTimestampNs() {
   struct timespec systemTime = {0};
-  clock_gettime(CLOCK_REALTIME, &systemTime);
+  taosClockGetTime(CLOCK_REALTIME, &systemTime);
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 

include/util/talgo.h

@@ -27,6 +27,11 @@ extern "C" {
 typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
+typedef void *(*FCopy)(void *);
+typedef void (*FDelete)(void *);
+typedef int32_t (*FEncode)(void **buf, const void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst);
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4
@@ -77,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
  * @return
  */
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                    __ext_compar_fn_t compar, char* buf, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -93,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @return
  */
 void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                  bool maxroot);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -62,6 +62,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_APP_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0014)
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0015)
 #define TSDB_CODE_RPC_INVALID_VERSION           TAOS_DEF_ERROR_CODE(0, 0x0016)
+#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017)
 
 //common & util
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0100)
@@ -86,6 +87,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0113)
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x0114)
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x0115)
+#define TSDB_CODE_DUP_KEY                       TAOS_DEF_ERROR_CODE(0, 0x0116)
+#define TSDB_CODE_NEED_RETRY                    TAOS_DEF_ERROR_CODE(0, 0x0117)
+#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE       TAOS_DEF_ERROR_CODE(0, 0x0118)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0140)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0141)
@@ -132,20 +136,16 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TSC_INVALID_JSON_TYPE         TAOS_DEF_ERROR_CODE(0, 0x0222)
 #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE        TAOS_DEF_ERROR_CODE(0, 0x0223)
 #define TSDB_CODE_TSC_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0X0224)
+#define TSDB_CODE_TSC_STMT_API_ERROR            TAOS_DEF_ERROR_CODE(0, 0X0225)
+#define TSDB_CODE_TSC_STMT_TBNAME_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0226)
+#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0227)
 
 // mnode-common
 #define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
 #define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0301)
-#define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0302)
+#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0302)
-#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0303)
+#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0303)
-#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED   TAOS_DEF_ERROR_CODE(0, 0x0304)
+#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0304)
-#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0305)
-#define TSDB_CODE_MND_INVALID_OPTIONS           TAOS_DEF_ERROR_CODE(0, 0x0306)
-#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0307)
-#define TSDB_CODE_MND_INVALID_MSG_VERSION       TAOS_DEF_ERROR_CODE(0, 0x0308)
-#define TSDB_CODE_MND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0309)
-#define TSDB_CODE_MND_INVALID_MSG_TYPE          TAOS_DEF_ERROR_CODE(0, 0x030A)
-#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS      TAOS_DEF_ERROR_CODE(0, 0x030B)
 
 // mnode-show
 #define TSDB_CODE_MND_INVALID_SHOWOBJ           TAOS_DEF_ERROR_CODE(0, 0x0310)
@@ -247,9 +247,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
 #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
+#define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_INFOS_TBL         TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
@@ -265,20 +266,22 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TRANS_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03D0)
 #define TSDB_CODE_MND_TRANS_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03D1)
 #define TSDB_CODE_MND_TRANS_INVALID_STAGE       TAOS_DEF_ERROR_CODE(0, 0x03D2)
-#define TSDB_CODE_MND_TRANS_CANT_PARALLEL       TAOS_DEF_ERROR_CODE(0, 0x03D4)
+#define TSDB_CODE_MND_TRANS_CONFLICT            TAOS_DEF_ERROR_CODE(0, 0x03D3)
+#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR        TAOS_DEF_ERROR_CODE(0, 0x03D4)
 
 // mnode-mq
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E0)
 #define TSDB_CODE_MND_TOPIC_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03E1)
 #define TSDB_CODE_MND_TOO_MANY_TOPICS           TAOS_DEF_ERROR_CODE(0, 0x03E2)
 #define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x03E3)
-#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E4)
+#define TSDB_CODE_MND_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x03E4)
-#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E5)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E5)
-#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB    TAOS_DEF_ERROR_CODE(0, 0x03E6)
+#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E6)
-#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
+#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E7)
-#define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
+#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
-#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
-#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_TOPIC_SUBSCRIBED          TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -320,35 +323,43 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_TB_NOT_EXIST              TAOS_DEF_ERROR_CODE(0, 0x0515)
 #define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
 #define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
+#define TSDB_CODE_VND_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0518)
+#define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
+#define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
+#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
 #define TSDB_CODE_TDB_INVALID_TABLE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x0601)
 #define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION     TAOS_DEF_ERROR_CODE(0, 0x0602)
 #define TSDB_CODE_TDB_TABLE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0603)
-#define TSDB_CODE_TDB_INVALID_CONFIG            TAOS_DEF_ERROR_CODE(0, 0x0604)
+#define TSDB_CODE_TDB_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0604)
-#define TSDB_CODE_TDB_INIT_FAILED               TAOS_DEF_ERROR_CODE(0, 0x0605)
+#define TSDB_CODE_TDB_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0605)
-#define TSDB_CODE_TDB_NO_DISKSPACE              TAOS_DEF_ERROR_CODE(0, 0x0606)
+#define TSDB_CODE_TDB_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0606)
-#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS       TAOS_DEF_ERROR_CODE(0, 0x0607)
+#define TSDB_CODE_TDB_INVALID_CONFIG            TAOS_DEF_ERROR_CODE(0, 0x0607)
-#define TSDB_CODE_TDB_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x0608)
+#define TSDB_CODE_TDB_INIT_FAILED               TAOS_DEF_ERROR_CODE(0, 0x0608)
-#define TSDB_CODE_TDB_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x0609)
+#define TSDB_CODE_TDB_NO_DISKSPACE              TAOS_DEF_ERROR_CODE(0, 0x0609)
-#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE       TAOS_DEF_ERROR_CODE(0, 0x060A)
+#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS       TAOS_DEF_ERROR_CODE(0, 0x060A)
-#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE    TAOS_DEF_ERROR_CODE(0, 0x060B)
+#define TSDB_CODE_TDB_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x060B)
-#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP       TAOS_DEF_ERROR_CODE(0, 0x060C)
+#define TSDB_CODE_TDB_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x060C)
-#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x060D)
+#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE       TAOS_DEF_ERROR_CODE(0, 0x060D)
-#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x060E)
+#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE    TAOS_DEF_ERROR_CODE(0, 0x060E)
-#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x060F)
+#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP       TAOS_DEF_ERROR_CODE(0, 0x060F)
-#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS       TAOS_DEF_ERROR_CODE(0, 0x0610)
+#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0600)
-#define TSDB_CODE_TDB_TABLE_RECONFIGURE         TAOS_DEF_ERROR_CODE(0, 0x0611)
+#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0601)
-#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0612)
+#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0602)
-#define TSDB_CODE_TDB_NO_AVAIL_DISK             TAOS_DEF_ERROR_CODE(0, 0x0613)
+#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS       TAOS_DEF_ERROR_CODE(0, 0x0613)
-#define TSDB_CODE_TDB_MESSED_MSG                TAOS_DEF_ERROR_CODE(0, 0x0614)
+#define TSDB_CODE_TDB_TABLE_RECONFIGURE         TAOS_DEF_ERROR_CODE(0, 0x0614)
-#define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0615)
+#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0615)
-#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0616)
+#define TSDB_CODE_TDB_NO_AVAIL_DISK             TAOS_DEF_ERROR_CODE(0, 0x0616)
-#define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x0617)
+#define TSDB_CODE_TDB_MESSED_MSG                TAOS_DEF_ERROR_CODE(0, 0x0617)
-#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x0618)
+#define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0618)
-#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x0619)
+#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0619)
-#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x0620)
+#define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061A)
+#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061B)
+#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x061C)
+#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x061D)
+#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x061E)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)
@@ -407,6 +418,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)
@@ -565,7 +579,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x2602)
 #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN          TAOS_DEF_ERROR_CODE(0, 0x2603)
 #define TSDB_CODE_PAR_WRONG_VALUE_TYPE          TAOS_DEF_ERROR_CODE(0, 0x2604)
-#define TSDB_CODE_PAR_INVALID_FUNTION           TAOS_DEF_ERROR_CODE(0, 0x2605)
 #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION  TAOS_DEF_ERROR_CODE(0, 0x2608)
 #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT    TAOS_DEF_ERROR_CODE(0, 0x2609)
 #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION   TAOS_DEF_ERROR_CODE(0, 0x260A)
@@ -586,7 +599,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_RANGE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x2619)
 #define TSDB_CODE_PAR_INVALID_STR_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261A)
 #define TSDB_CODE_PAR_INVALID_ENUM_OPTION       TAOS_DEF_ERROR_CODE(0, 0x261B)
-#define TSDB_CODE_PAR_INVALID_TTL_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261C)
 #define TSDB_CODE_PAR_INVALID_KEEP_NUM          TAOS_DEF_ERROR_CODE(0, 0x261D)
 #define TSDB_CODE_PAR_INVALID_KEEP_ORDER        TAOS_DEF_ERROR_CODE(0, 0x261E)
 #define TSDB_CODE_PAR_INVALID_KEEP_VALUE        TAOS_DEF_ERROR_CODE(0, 0x261F)
@@ -610,15 +622,58 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG     TAOS_DEF_ERROR_CODE(0, 0x2631)
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
 #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2633)
+#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL      TAOS_DEF_ERROR_CODE(0, 0x2634)
+#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL   TAOS_DEF_ERROR_CODE(0, 0x2635)
+#define TSDB_CODE_PAR_INVALID_DAYS_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2636)
+#define TSDB_CODE_PAR_OFFSET_LESS_ZERO          TAOS_DEF_ERROR_CODE(0, 0x2637)
+#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY  TAOS_DEF_ERROR_CODE(0, 0x2638)
+#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x2639)
+#define TSDB_CODE_PAR_INVALID_DROP_STABLE       TAOS_DEF_ERROR_CODE(0, 0x263A)
+#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE   TAOS_DEF_ERROR_CODE(0, 0x263B)
+#define TSDB_CODE_PAR_DUPLICATED_COLUMN         TAOS_DEF_ERROR_CODE(0, 0x263C)
+#define TSDB_CODE_PAR_INVALID_TAGS_LENGTH       TAOS_DEF_ERROR_CODE(0, 0x263D)
+#define TSDB_CODE_PAR_INVALID_ROW_LENGTH        TAOS_DEF_ERROR_CODE(0, 0x263E)
+#define TSDB_CODE_PAR_INVALID_COLUMNS_NUM       TAOS_DEF_ERROR_CODE(0, 0x263F)
+#define TSDB_CODE_PAR_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x2640)
+#define TSDB_CODE_PAR_INVALID_FIRST_COLUMN      TAOS_DEF_ERROR_CODE(0, 0x2641)
+#define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN    TAOS_DEF_ERROR_CODE(0, 0x2642)
+#define TSDB_CODE_PAR_INVALID_TAGS_NUM          TAOS_DEF_ERROR_CODE(0, 0x2643)
+#define TSDB_CODE_PAR_PERMISSION_DENIED         TAOS_DEF_ERROR_CODE(0, 0x2644)
+#define TSDB_CODE_PAR_INVALID_STREAM_QUERY      TAOS_DEF_ERROR_CODE(0, 0x2645)
+#define TSDB_CODE_PAR_INVALID_INTERNAL_PK       TAOS_DEF_ERROR_CODE(0, 0x2646)
+#define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC     TAOS_DEF_ERROR_CODE(0, 0x2647)
+#define TSDB_CODE_PAR_INVALID_PASSWD            TAOS_DEF_ERROR_CODE(0, 0x2648)
+#define TSDB_CODE_PAR_INVALID_ALTER_TABLE       TAOS_DEF_ERROR_CODE(0, 0x2649)
+#define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY   TAOS_DEF_ERROR_CODE(0, 0x264A)
+#define TSDB_CODE_PAR_INVALID_MODIFY_COL        TAOS_DEF_ERROR_CODE(0, 0x264B)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
+#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL        TAOS_DEF_ERROR_CODE(0, 0x2701)
+#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN   TAOS_DEF_ERROR_CODE(0, 0x2702)
 
 //function
 #define TSDB_CODE_FUNC_FUNTION_ERROR            TAOS_DEF_ERROR_CODE(0, 0x2800)
 #define TSDB_CODE_FUNC_FUNTION_PARA_NUM         TAOS_DEF_ERROR_CODE(0, 0x2801)
 #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE        TAOS_DEF_ERROR_CODE(0, 0x2802)
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
+#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2804)
+
+//udf
+#define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
+#define TSDB_CODE_UDF_PIPE_READ_ERR             TAOS_DEF_ERROR_CODE(0, 0x2902)
+#define TSDB_CODE_UDF_PIPE_CONNECT_ERR          TAOS_DEF_ERROR_CODE(0, 0x2903)
+#define TSDB_CODE_UDF_PIPE_NO_PIPE              TAOS_DEF_ERROR_CODE(0, 0x2904)
+#define TSDB_CODE_UDF_LOAD_UDF_FAILURE          TAOS_DEF_ERROR_CODE(0, 0x2905)
+#define TSDB_CODE_UDF_INVALID_STATE             TAOS_DEF_ERROR_CODE(0, 0x2906)
+#define TSDB_CODE_UDF_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0x2907)
+#define TSDB_CODE_UDF_NO_FUNC_HANDLE            TAOS_DEF_ERROR_CODE(0, 0x2908)
+#define TSDB_CODE_UDF_INVALID_BUFSIZE           TAOS_DEF_ERROR_CODE(0, 0x2909)
+#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE       TAOS_DEF_ERROR_CODE(0, 0x290A)
+
+#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
+#define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)
+#define TSDB_CODE_SML_INVALID_DATA              TAOS_DEF_ERROR_CODE(0, 0x3002)
 
 #ifdef __cplusplus
 }

include/util/tarray.h

@@ -199,6 +199,12 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize);
  */
 SArray* taosArrayDup(const SArray* pSrc);
 
+/**
+ * deep copy a new array
+ * @param pSrc
+ */
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
+
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -212,19 +218,16 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
-
 /**
- * destroy array list
+ * clear the array (remove all element)
- * @param pArray
- */
-void* taosArrayDestroy(SArray* pArray);
-
-/**
- *
  * @param pArray
  * @param fp
  */
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
+void taosArrayClearP(SArray* pArray, FDelete fp);
+
+void* taosArrayDestroy(SArray* pArray);
+void  taosArrayDestroyP(SArray* pArray, FDelete fp);
+void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
 
 /**
  * sort the array
@@ -272,6 +275,11 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
 
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
+void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+
+char* taosShowStrArray(const SArray* pArray);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tbloomfilter.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_BLOOMFILTER_H_
+#define _TD_UTIL_BLOOMFILTER_H_
+
+#include "os.h"
+#include "thash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBloomFilter {
+  uint32_t hashFunctions;
+  uint64_t expectedEntries;
+  uint64_t numUnits;
+  uint64_t numBits;
+  uint64_t size;
+  _hash_fn_t hashFn1;
+  _hash_fn_t hashFn2;
+  void *buffer;
+  double errorRate;
+} SBloomFilter;
+
+SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
+int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
+int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
+                              uint32_t len);
+void tBloomFilterDestroy(SBloomFilter *pBF);
+void tBloomFilterDump(const SBloomFilter *pBF);
+bool tBloomFilterIsFull(const SBloomFilter *pBF);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_BLOOMFILTER_H_*/

include/util/tcache.h

@@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
 /**
  * initialize the cache object
  * @param keyType              key type
- * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
+ * @param refreshTimeInMs      refresh operation interval time, the maximum survival time when one element is expired
  *                             and not referenced by other objects
  * @param extendLifespan       auto extend lifespan, if accessed
  * @param fn                   free resource callback function
  * @return
  */
-SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
+SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName);
 
 /**

include/util/tcoding.h

@@ -59,6 +59,21 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
 
 static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
 
+// --- Bool
+
+static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
+  if (buf != NULL) {
+    ((int8_t *)(*buf))[0] = (value ? 1 : 0);
+    *buf = POINTER_SHIFT(*buf, sizeof(int8_t));
+  }
+  return (int32_t)sizeof(int8_t);
+}
+
+static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
+  *value = ( (((int8_t *)buf)[0] == 0) ? false : true );
+  return POINTER_SHIFT(buf, sizeof(int8_t));
+}
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {

include/util/tconfig.h

@@ -30,6 +30,7 @@ typedef enum {
   CFG_STYPE_CFG_FILE,
   CFG_STYPE_ENV_FILE,
   CFG_STYPE_ENV_VAR,
+  CFG_STYPE_ENV_CMD,
   CFG_STYPE_APOLLO_URL,
   CFG_STYPE_ARG_LIST,
   CFG_STYPE_TAOS_OPTIONS
@@ -82,7 +83,7 @@ typedef struct SConfig {
 } SConfig;
 
 SConfig *cfgInit();
-int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
+int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
 int32_t  cfgLoadFromArray(SConfig *pCfg, SArray *pArgs);  // SConfigPair
 void     cfgCleanup(SConfig *pCfg);
 
@@ -105,6 +106,8 @@ const char *cfgDtypeStr(ECfgDataType type);
 
 void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
 
+int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -29,6 +29,8 @@ extern "C" {
 #define TSKEY_MAX         (INT64_MAX - 1)
 #define TSKEY_INITIAL_VAL TSKEY_MIN
 
+#define TD_VER_MAX UINT64_MAX  // TODO: use the real max version from query handle
+
 // Bytes for each type.
 extern const int32_t TYPE_BYTES[15];
 
@@ -76,8 +78,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DEFAULT_PASS "taosdata"
 #endif
 
-#define SHELL_MAX_PASSWORD_LEN 20
-
 #define TSDB_TRUE  1
 #define TSDB_FALSE 0
 #define TSDB_OK    0
@@ -89,6 +89,9 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MILLI   0
 #define TSDB_TIME_PRECISION_MICRO   1
 #define TSDB_TIME_PRECISION_NANO    2
+#define TSDB_TIME_PRECISION_HOURS   3
+#define TSDB_TIME_PRECISION_MINUTES 4
+#define TSDB_TIME_PRECISION_SECONDS 5
 
 #define TSDB_TIME_PRECISION_MILLI_STR "ms"
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
@@ -99,42 +102,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MICRO_DIGITS 16
 #define TSDB_TIME_PRECISION_NANO_DIGITS  19
 
-#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
-#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
-#define TSDB_INS_TABLE_DNODES                 "dnodes"
-#define TSDB_INS_TABLE_MNODES                 "mnodes"
-#define TSDB_INS_TABLE_MODULES                "modules"
-#define TSDB_INS_TABLE_QNODES                 "qnodes"
-#define TSDB_INS_TABLE_BNODES                 "bnodes"
-#define TSDB_INS_TABLE_SNODES                 "snodes"
-#define TSDB_INS_TABLE_CLUSTER                "cluster"
-#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
-#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
-#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
-#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
-#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
-#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
-#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
-#define TSDB_INS_TABLE_USER_USERS             "user_users"
-#define TSDB_INS_TABLE_LICENCES               "grants"
-#define TSDB_INS_TABLE_VGROUPS                "vgroups"
-#define TSDB_INS_TABLE_TOPICS                 "topics"
-#define TSDB_INS_TABLE_CONSUMERS              "consumers"
-#define TSDB_INS_TABLE_SUBSCRIBES             "subscribes"
-#define TSDB_INS_TABLE_TRANS                  "trans"
-#define TSDB_INS_TABLE_SMAS                   "smas"
-#define TSDB_INS_TABLE_CONFIGS                "configs"
-#define TSDB_INS_TABLE_CONNS                  "connections"
-#define TSDB_INS_TABLE_QUERIES                "queries"
-#define TSDB_INS_TABLE_VNODES                 "vnodes"
-
-#define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
-#define TSDB_PERFS_TABLE_CONNECTIONS "connections"
-#define TSDB_PERFS_TABLE_QUERIES     "queries"
-#define TSDB_PERFS_TABLE_TOPICS      "topics"
-#define TSDB_PERFS_TABLE_CONSUMERS   "consumers"
-#define TSDB_PERFS_TABLE_SUBSCRIBES  "subscribes"
-
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
 
@@ -198,22 +165,14 @@ typedef enum EOperatorType {
   OP_TYPE_JSON_CONTAINS
 } EOperatorType;
 
+#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
+
 typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_AND = 1,
   LOGIC_COND_TYPE_OR,
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
-#define FUNCTION_CEIL  4500
-#define FUNCTION_FLOOR 4501
-#define FUNCTION_ABS   4502
-#define FUNCTION_ROUND 4503
-
-#define FUNCTION_LENGTH 4800
-#define FUNCTION_CONCAT 4801
-#define FUNCTION_LTRIM  4802
-#define FUNCTION_RTRIM  4803
-
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24
@@ -239,6 +198,8 @@ typedef enum ELogicConditionType {
 #define TSDB_FUNC_BUF_SIZE       512
 #define TSDB_FUNC_TYPE_SCALAR    1
 #define TSDB_FUNC_TYPE_AGGREGATE 2
+#define TSDB_FUNC_SCRIPT_BIN_LIB 0
+#define TSDB_FUNC_SCRIPT_LUA     1
 #define TSDB_FUNC_MAX_RETRIEVE   1024
 
 #define TSDB_INDEX_NAME_LEN      65  // 64 + 1 '\0'
@@ -285,8 +246,8 @@ typedef enum ELogicConditionType {
 #define TSDB_IPv4ADDR_LEN       16
 #define TSDB_FILENAME_LEN       128
 #define TSDB_SHOW_SQL_LEN       512
-#define TSDB_SHOW_SUBQUERY_LEN  1000
 #define TSDB_SLOW_QUERY_SQL_LEN 512
+#define TSDB_SHOW_SUBQUERY_LEN  1000
 
 #define TSDB_TRANS_STAGE_LEN 12
 #define TSDB_TRANS_TYPE_LEN  16
@@ -323,15 +284,21 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_VNODES_PER_DB          1
 #define TSDB_MAX_VNODES_PER_DB          4096
 #define TSDB_DEFAULT_VN_PER_DB          2
-#define TSDB_MIN_CACHE_BLOCK_SIZE     1
+#define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
-#define TSDB_MAX_CACHE_BLOCK_SIZE     128  // 128MB for each vnode
+#define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB
-#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
+#define TSDB_DEFAULT_BUFFER_PER_VNODE   96
-#define TSDB_MIN_TOTAL_BLOCKS         3
+#define TSDB_MIN_PAGES_PER_VNODE        64
-#define TSDB_MAX_TOTAL_BLOCKS         10000
+#define TSDB_MAX_PAGES_PER_VNODE        16384
-#define TSDB_DEFAULT_TOTAL_BLOCKS     6
+#define TSDB_DEFAULT_PAGES_PER_VNODE    256
+#define TSDB_MIN_PAGESIZE_PER_VNODE     1  // unit KB
+#define TSDB_MAX_PAGESIZE_PER_VNODE     16384
+#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
 #define TSDB_MIN_DAYS_PER_FILE          60  // unit minute
 #define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
 #define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
+#define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
+#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
+#define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
 #define TSDB_MIN_KEEP                   (1 * 1440)       // data in db to be reserved. unit minute
 #define TSDB_MAX_KEEP                   (365000 * 1440)  // data in db to be reserved.
 #define TSDB_DEFAULT_KEEP               (3650 * 1440)    // ten years
@@ -341,14 +308,9 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_MAXROWS_FBLOCK         200
 #define TSDB_MAX_MAXROWS_FBLOCK         10000
 #define TSDB_DEFAULT_MAXROWS_FBLOCK     4096
-#define TSDB_MIN_COMMIT_TIME          30
-#define TSDB_MAX_COMMIT_TIME          40960
-#define TSDB_DEFAULT_COMMIT_TIME      3600
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD       3000    // three second
-#define TSDB_MIN_DB_TTL               1
-#define TSDB_DEFAULT_DB_TTL           1
 #define TSDB_MIN_WAL_LEVEL              1
 #define TSDB_MAX_WAL_LEVEL              2
 #define TSDB_DEFAULT_WAL_LEVEL          1
@@ -364,9 +326,6 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_STRICT_OFF              0
 #define TSDB_DB_STRICT_ON               1
 #define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MIN_DB_UPDATE            0
-#define TSDB_MAX_DB_UPDATE            2
-#define TSDB_DEFAULT_DB_UPDATE        0
 #define TSDB_MIN_DB_CACHE_LAST_ROW      0
 #define TSDB_MAX_DB_CACHE_LAST_ROW      3
 #define TSDB_DEFAULT_CACHE_LAST_ROW     0
@@ -377,12 +336,15 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SINGLE_STABLE_OFF       1
 #define TSDB_DEFAULT_DB_SINGLE_STABLE   0
 
-#define TSDB_MIN_DB_FILE_FACTOR     0
+#define TSDB_MIN_ROLLUP_FILE_FACTOR     0
-#define TSDB_MAX_DB_FILE_FACTOR     1
+#define TSDB_MAX_ROLLUP_FILE_FACTOR     1
-#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1
+#define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1
-#define TSDB_MIN_DB_DELAY           1
+#define TSDB_MIN_ROLLUP_DELAY           1
-#define TSDB_MAX_DB_DELAY           10
+#define TSDB_MAX_ROLLUP_DELAY           10
-#define TSDB_DEFAULT_DB_DELAY       2
+#define TSDB_DEFAULT_ROLLUP_DELAY       2
+#define TSDB_MIN_TABLE_TTL              0
+#define TSDB_DEFAULT_TABLE_TTL          0
+
 #define TSDB_MIN_EXPLAIN_RATIO     0
 #define TSDB_MAX_EXPLAIN_RATIO     1
 #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
@@ -412,21 +374,9 @@ typedef enum ELogicConditionType {
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag
  */
-#define TSDB_QUERY_TYPE_SUBQUERY        0x02u
-#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u  // two-stage subquery for super table
-
-#define TSDB_QUERY_TYPE_TABLE_QUERY      0x08u  // query ordinary table; below only apply to client side
-#define TSDB_QUERY_TYPE_STABLE_QUERY     0x10u  // query on super table
-#define TSDB_QUERY_TYPE_JOIN_QUERY       0x20u  // join query
-#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u  // select *,columns... query
-#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE   0x80u  // join sub query at the second stage
-
-#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
 #define TSDB_QUERY_TYPE_INSERT      0x100u  // insert type
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
 #define TSDB_QUERY_TYPE_FILE_INSERT 0x400u  // insert data from file
 #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u  // stmt insert type
-#define TSDB_QUERY_TYPE_NEST_SUBQUERY    0x1000u  // nested sub query
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)   (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)   ((x) |= (_type))
@@ -475,9 +425,12 @@ enum {
   SND_WORKER_TYPE__UNIQUE,
 };
 
-#define MNODE_HANDLE   -1
-#define QNODE_HANDLE   1
 #define DEFAULT_HANDLE 0
+#define MNODE_HANDLE   -1
+#define QNODE_HANDLE   -2
+#define SNODE_HANDLE   -3
+#define VNODE_HANDLE   -4
+#define BNODE_HANDLE   -5
 
 #define TSDB_CONFIG_OPTION_LEN 16
 #define TSDB_CONIIG_VALUE_LEN  48

include/util/tencode.h

@@ -17,12 +17,35 @@
 #define _TD_UTIL_ENCODE_H_
 
 #include "tcoding.h"
-#include "tfreelist.h"
+#include "tlist.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SEncoderNode SEncoderNode;
+typedef struct SDecoderNode SDecoderNode;
+
+typedef struct SCoderMem {
+  struct SCoderMem* next;
+} SCoderMem;
+
+typedef struct {
+  uint8_t*      data;
+  uint32_t      size;
+  uint32_t      pos;
+  SCoderMem*    mList;
+  SEncoderNode* eStack;
+} SEncoder;
+
+typedef struct {
+  const uint8_t* data;
+  uint32_t       size;
+  uint32_t       pos;
+  SCoderMem*     mList;
+  SDecoderNode*  dStack;
+} SDecoder;
+
 #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
 #define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0]
 
@@ -50,92 +73,84 @@ extern "C" {
 #define tRGet32 tRPut32
 #define tRGet64 tRPut64
 
-typedef enum { TD_ENCODER, TD_DECODER } td_coder_t;
-
-#define CODER_NODE_FIELDS \
-  uint8_t* data;          \
-  int32_t  size;          \
-  int32_t  pos;
-
-struct SCoderNode {
-  TD_SLIST_NODE(SCoderNode);
-  CODER_NODE_FIELDS
-};
-
-typedef struct {
-  td_coder_t  type;
-  td_endian_t endian;
-  SFreeList   fl;
-  CODER_NODE_FIELDS
-  TD_SLIST(SCoderNode) stack;
-} SCoder;
-
 #define TD_CODER_POS(CODER)                            ((CODER)->pos)
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
-#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER)          TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl))
 
-void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
+#define tEncodeSize(E, S, SIZE, RET) \
-void tCoderClear(SCoder* pCoder);
+  do {                               \
+    SEncoder coder = {0};            \
+    tEncoderInit(&coder, NULL, 0);   \
+    if ((E)(&coder, S) == 0) {       \
+      SIZE = coder.pos;              \
+      RET = 0;                       \
+    } else {                         \
+      RET = -1;                      \
+    }                                \
+    tEncoderClear(&coder);           \
+  } while (0)
+
+static void* tEncoderMalloc(SEncoder* pCoder, int32_t size);
+static void* tDecoderMalloc(SDecoder* pCoder, int32_t size);
 
 /* ------------------------ ENCODE ------------------------ */
-int32_t        tStartEncode(SCoder* pEncoder);
+void           tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size);
-void           tEndEncode(SCoder* pEncoder);
+void           tEncoderClear(SEncoder* pCoder);
-static int32_t tEncodeU8(SCoder* pEncoder, uint8_t val);
+int32_t        tStartEncode(SEncoder* pCoder);
-static int32_t tEncodeI8(SCoder* pEncoder, int8_t val);
+void           tEndEncode(SEncoder* pCoder);
-static int32_t tEncodeU16(SCoder* pEncoder, uint16_t val);
+static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val);
-static int32_t tEncodeI16(SCoder* pEncoder, int16_t val);
+static int32_t tEncodeI8(SEncoder* pCoder, int8_t val);
-static int32_t tEncodeU32(SCoder* pEncoder, uint32_t val);
+static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val);
-static int32_t tEncodeI32(SCoder* pEncoder, int32_t val);
+static int32_t tEncodeI16(SEncoder* pCoder, int16_t val);
-static int32_t tEncodeU64(SCoder* pEncoder, uint64_t val);
+static int32_t tEncodeU32(SEncoder* pCoder, uint32_t val);
-static int32_t tEncodeI64(SCoder* pEncoder, int64_t val);
+static int32_t tEncodeI32(SEncoder* pCoder, int32_t val);
-static int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val);
+static int32_t tEncodeU64(SEncoder* pCoder, uint64_t val);
-static int32_t tEncodeI16v(SCoder* pEncoder, int16_t val);
+static int32_t tEncodeI64(SEncoder* pCoder, int64_t val);
-static int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val);
+static int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val);
-static int32_t tEncodeI32v(SCoder* pEncoder, int32_t val);
+static int32_t tEncodeI16v(SEncoder* pCoder, int16_t val);
-static int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val);
+static int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val);
-static int32_t tEncodeI64v(SCoder* pEncoder, int64_t val);
+static int32_t tEncodeI32v(SEncoder* pCoder, int32_t val);
-static int32_t tEncodeFloat(SCoder* pEncoder, float val);
+static int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val);
-static int32_t tEncodeDouble(SCoder* pEncoder, double val);
+static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val);
-static int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len);
+static int32_t tEncodeFloat(SEncoder* pCoder, float val);
-static int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len);
+static int32_t tEncodeDouble(SEncoder* pCoder, double val);
-static int32_t tEncodeCStr(SCoder* pEncoder, const char* val);
+static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len);
+static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len);
+static int32_t tEncodeCStr(SEncoder* pCoder, const char* val);
 
 /* ------------------------ DECODE ------------------------ */
-int32_t        tStartDecode(SCoder* pDecoder);
+void           tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size);
-void           tEndDecode(SCoder* pDecoder);
+void           tDecoderClear(SDecoder* SDecoder);
-static bool    tDecodeIsEnd(SCoder* pCoder);
+int32_t        tStartDecode(SDecoder* pCoder);
-static int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val);
+void           tEndDecode(SDecoder* pCoder);
-static int32_t tDecodeI8(SCoder* pDecoder, int8_t* val);
+static bool    tDecodeIsEnd(SDecoder* pCoder);
-static int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val);
+static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val);
-static int32_t tDecodeI16(SCoder* pDecoder, int16_t* val);
+static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val);
-static int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val);
+static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val);
-static int32_t tDecodeI32(SCoder* pDecoder, int32_t* val);
+static int32_t tDecodeI16(SDecoder* pCoder, int16_t* val);
-static int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val);
+static int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val);
-static int32_t tDecodeI64(SCoder* pDecoder, int64_t* val);
+static int32_t tDecodeI32(SDecoder* pCoder, int32_t* val);
-static int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val);
+static int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val);
-static int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val);
+static int32_t tDecodeI64(SDecoder* pCoder, int64_t* val);
-static int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val);
+static int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val);
-static int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val);
+static int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val);
-static int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val);
+static int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val);
-static int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val);
+static int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val);
-static int32_t tDecodeFloat(SCoder* pDecoder, float* val);
+static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val);
-static int32_t tDecodeDouble(SCoder* pDecoder, double* val);
+static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val);
-static int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len);
+static int32_t tDecodeFloat(SDecoder* pCoder, float* val);
-static int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len);
+static int32_t tDecodeDouble(SDecoder* pCoder, double* val);
-static int32_t tDecodeCStr(SCoder* pDecoder, const char** val);
+static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len);
-static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val);
+static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len);
+static int32_t tDecodeCStr(SDecoder* pCoder, const char** val);
+static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
 
 /* ------------------------ IMPL ------------------------ */
 #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS)                        \
   if ((CODER)->data) {                                                 \
     if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
-    if (TD_RT_ENDIAN() == (CODER)->endian) {                           \
     tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL));                        \
-    } else {                                                           \
-      tRPut##BITS(TD_CODER_CURRENT(CODER), &(VAL));                    \
-    }                                                                  \
   }                                                                    \
   TD_CODER_MOVE_POS(CODER, sizeof(VAL));                               \
   return 0;
@@ -160,12 +175,7 @@ static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val);
 
 #define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS)                         \
   if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
-  if (TD_RT_ENDIAN() == (CODER)->endian) {                               \
   tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL));                          \
-  } else {                                                               \
-    tRGet##BITS(PVAL, TD_CODER_CURRENT(CODER));                          \
-  }                                                                      \
-                                                                         \
   TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL)));                             \
   return 0;
 
@@ -177,138 +187,134 @@ static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val);
     TYPE tval = TD_CODER_CURRENT(CODER)[0];                  \
     if (tval < ENCODE_LIMIT) {                               \
       *(PVAL) |= (tval << (7 * i));                          \
-      TD_CODER_MOVE_POS(pDecoder, 1);                        \
+      TD_CODER_MOVE_POS(pCoder, 1);                          \
       break;                                                 \
     } else {                                                 \
       *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
       i++;                                                   \
-      TD_CODER_MOVE_POS(pDecoder, 1);                        \
+      TD_CODER_MOVE_POS(pCoder, 1);                          \
     }                                                        \
   }                                                          \
                                                              \
   return 0;
 
 // 8
-static FORCE_INLINE int32_t tEncodeU8(SCoder* pEncoder, uint8_t val) {
+static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) {
-  if (pEncoder->data) {
+  if (pCoder->data) {
-    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
-    tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val);
+    tPut(uint8_t, TD_CODER_CURRENT(pCoder), val);
   }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
+  TD_CODER_MOVE_POS(pCoder, sizeof(val));
   return 0;
 }
 
-static FORCE_INLINE int32_t tEncodeI8(SCoder* pEncoder, int8_t val) {
+static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) {
-  if (pEncoder->data) {
+  if (pCoder->data) {
-    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
-    tPut(int8_t, TD_CODER_CURRENT(pEncoder), val);
+    tPut(int8_t, TD_CODER_CURRENT(pCoder), val);
   }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
+  TD_CODER_MOVE_POS(pCoder, sizeof(val));
   return 0;
 }
 
 // 16
-static FORCE_INLINE int32_t tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); }
+static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); }
-static FORCE_INLINE int32_t tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); }
+static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); }
 // 32
-static FORCE_INLINE int32_t tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); }
+static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); }
-static FORCE_INLINE int32_t tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); }
+static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); }
 // 64
-static FORCE_INLINE int32_t tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); }
+static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); }
-static FORCE_INLINE int32_t tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); }
+static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); }
 // 16v
-static FORCE_INLINE int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
+static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
-static FORCE_INLINE int32_t tEncodeI16v(SCoder* pEncoder, int16_t val) {
+static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) {
-  return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val));
+  return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val));
 }
 // 32v
-static FORCE_INLINE int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
+static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
-static FORCE_INLINE int32_t tEncodeI32v(SCoder* pEncoder, int32_t val) {
+static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) {
-  return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val));
+  return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val));
 }
 // 64v
-static FORCE_INLINE int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
+static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
-static FORCE_INLINE int32_t tEncodeI64v(SCoder* pEncoder, int64_t val) {
+static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) {
-  return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val));
+  return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val));
 }
 
-static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) {
+static FORCE_INLINE int32_t tEncodeFloat(SEncoder* pCoder, float val) {
   union {
     uint32_t ui;
     float    f;
-  } v = {.f = val};
+  } v;
+  v.f = val;
 
-  return tEncodeU32(pEncoder, v.ui);
+  return tEncodeU32(pCoder, v.ui);
 }
 
-static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) {
+static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
   union {
     uint64_t ui;
     double   d;
-  } v = {.d = val};
+  } v;
+  v.d = val;
 
-  return tEncodeU64(pEncoder, v.ui);
+  return tEncodeU64(pCoder, v.ui);
 }
 
-static FORCE_INLINE int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) {
+static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
-  if (tEncodeU64v(pEncoder, len) < 0) return -1;
+  if (tEncodeU32v(pCoder, len) < 0) return -1;
-  if (pEncoder->data) {
+  if (pCoder->data) {
-    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1;
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1;
-    memcpy(TD_CODER_CURRENT(pEncoder), val, len);
+    memcpy(TD_CODER_CURRENT(pCoder), val, len);
   }
 
-  TD_CODER_MOVE_POS(pEncoder, len);
+  TD_CODER_MOVE_POS(pCoder, len);
   return 0;
 }
 
-static FORCE_INLINE int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) {
+static FORCE_INLINE int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len) {
-  return tEncodeBinary(pEncoder, (void*)val, len + 1);
+  return tEncodeBinary(pCoder, (uint8_t*)val, len + 1);
 }
 
-static FORCE_INLINE int32_t tEncodeCStr(SCoder* pEncoder, const char* val) {
+static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) {
-  return tEncodeCStrWithLen(pEncoder, val, (uint64_t)strlen(val));
+  return tEncodeCStrWithLen(pCoder, val, (uint32_t)strlen(val));
 }
 
 /* ------------------------ FOR DECODER ------------------------ */
 // 8
-static FORCE_INLINE int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val) {
+static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
-  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
-  tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val);
+  tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val);
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
+  TD_CODER_MOVE_POS(pCoder, sizeof(*val));
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeI8(SCoder* pDecoder, int8_t* val) {
+static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) {
-  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
-  tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val);
+  tGet(int8_t, TD_CODER_CURRENT(pCoder), *val);
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
+  TD_CODER_MOVE_POS(pCoder, sizeof(*val));
   return 0;
 }
 
 // 16
-static FORCE_INLINE int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val) {
+static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); }
-  TD_DECODE_MACRO(pDecoder, val, uint16_t, 16);
+static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); }
-}
-static FORCE_INLINE int32_t tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); }
 // 32
-static FORCE_INLINE int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val) {
+static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); }
-  TD_DECODE_MACRO(pDecoder, val, uint32_t, 32);
+static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); }
-}
-static FORCE_INLINE int32_t tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); }
 // 64
-static FORCE_INLINE int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val) {
+static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); }
-  TD_DECODE_MACRO(pDecoder, val, uint64_t, 64);
+static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); }
-}
-static FORCE_INLINE int32_t tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); }
 
 // 16v
-static FORCE_INLINE int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val) {
+static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) {
-  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint16_t);
+  TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t);
 }
 
-static FORCE_INLINE int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val) {
+static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
   uint16_t tval;
-  if (tDecodeU16v(pDecoder, &tval) < 0) {
+  if (tDecodeU16v(pCoder, &tval) < 0) {
     return -1;
   }
   *val = ZIGZAGD(int16_t, tval);
@@ -316,13 +322,13 @@ static FORCE_INLINE int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val) {
 }
 
 // 32v
-static FORCE_INLINE int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val) {
+static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) {
-  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint32_t);
+  TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t);
 }
 
-static FORCE_INLINE int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val) {
+static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
   uint32_t tval;
-  if (tDecodeU32v(pDecoder, &tval) < 0) {
+  if (tDecodeU32v(pCoder, &tval) < 0) {
     return -1;
   }
   *val = ZIGZAGD(int32_t, tval);
@@ -330,26 +336,26 @@ static FORCE_INLINE int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val) {
 }
 
 // 64v
-static FORCE_INLINE int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val) {
+static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) {
-  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint64_t);
+  TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t);
 }
 
-static FORCE_INLINE int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val) {
+static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
   uint64_t tval;
-  if (tDecodeU64v(pDecoder, &tval) < 0) {
+  if (tDecodeU64v(pCoder, &tval) < 0) {
     return -1;
   }
   *val = ZIGZAGD(int64_t, tval);
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeFloat(SCoder* pDecoder, float* val) {
+static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) {
   union {
     uint32_t ui;
     float    f;
   } v;
 
-  if (tDecodeU32(pDecoder, &(v.ui)) < 0) {
+  if (tDecodeU32(pCoder, &(v.ui)) < 0) {
     return -1;
   }
 
@@ -357,13 +363,13 @@ static FORCE_INLINE int32_t tDecodeFloat(SCoder* pDecoder, float* val) {
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeDouble(SCoder* pDecoder, double* val) {
+static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
   union {
     uint64_t ui;
     double   d;
   } v;
 
-  if (tDecodeU64(pDecoder, &(v.ui)) < 0) {
+  if (tDecodeU64(pCoder, &(v.ui)) < 0) {
     return -1;
   }
 
@@ -371,60 +377,270 @@ static FORCE_INLINE int32_t tDecodeDouble(SCoder* pDecoder, double* val) {
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) {
+static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) {
-  if (tDecodeU64v(pDecoder, len) < 0) return -1;
+  if (tDecodeU32v(pCoder, len) < 0) return -1;
 
-  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1;
-  *val = (void*)TD_CODER_CURRENT(pDecoder);
+  if (val) {
+    *val = (uint8_t*)TD_CODER_CURRENT(pCoder);
+  }
 
-  TD_CODER_MOVE_POS(pDecoder, *len);
+  TD_CODER_MOVE_POS(pCoder, *len);
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) {
+static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) {
-  if (tDecodeBinary(pDecoder, (const void**)val, len) < 0) return -1;
+  if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1;
   (*len) -= 1;
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStr(SCoder* pDecoder, const char** val) {
+static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) {
-  uint64_t len;
+  uint32_t len;
-  return tDecodeCStrAndLen(pDecoder, val, &len);
+  return tDecodeCStrAndLen(pCoder, val, &len);
 }
 
-static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val) {
+static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
   const char* pStr;
-  uint64_t    len;
+  uint32_t    len;
-  if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1;
+  if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
 
   memcpy(val, pStr, len + 1);
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uint64_t* len) {
+static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
-  if (tDecodeU64v(pDecoder, len) < 0) return -1;
+  if (tDecodeU64v(pCoder, len) < 0) return -1;
 
-  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1;
   *val = taosMemoryMalloc(*len);
   if (*val == NULL) return -1;
-  memcpy(*val, TD_CODER_CURRENT(pDecoder), *len);
+  memcpy(*val, TD_CODER_CURRENT(pCoder), *len);
 
-  TD_CODER_MOVE_POS(pDecoder, *len);
+  TD_CODER_MOVE_POS(pCoder, *len);
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SCoder* pDecoder, char** val, uint64_t* len) {
+static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) {
-  if (tDecodeBinaryAlloc(pDecoder, (void**)val, len) < 0) return -1;
+  if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1;
   (*len) -= 1;
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStrAlloc(SCoder* pDecoder, char** val) {
+static FORCE_INLINE int32_t tDecodeCStrAlloc(SDecoder* pCoder, char** val) {
   uint64_t len;
-  return tDecodeCStrAndLenAlloc(pDecoder, val, &len);
+  return tDecodeCStrAndLenAlloc(pCoder, val, &len);
 }
 
-static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); }
+static FORCE_INLINE bool tDecodeIsEnd(SDecoder* pCoder) { return (pCoder->size == pCoder->pos); }
+
+static FORCE_INLINE void* tEncoderMalloc(SEncoder* pCoder, int32_t size) {
+  void*      p = NULL;
+  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + size);
+  if (pMem) {
+    pMem->next = pCoder->mList;
+    pCoder->mList = pMem;
+    p = (void*)&pMem[1];
+  }
+  return p;
+}
+
+static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
+  void*      p = NULL;
+  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + size);
+  if (pMem) {
+    pMem->next = pCoder->mList;
+    pCoder->mList = pMem;
+    p = (void*)&pMem[1];
+  }
+  return p;
+}
+
+// ===========================================
+#define tPutV(p, v)                    \
+  do {                                 \
+    int32_t n = 0;                     \
+    for (;;) {                         \
+      if (v <= 0x7f) {                 \
+        if (p) p[n] = v;               \
+        n++;                           \
+        break;                         \
+      }                                \
+      if (p) p[n] = (v & 0x7f) | 0x80; \
+      n++;                             \
+      v >>= 7;                         \
+    }                                  \
+    return n;                          \
+  } while (0)
+
+#define tGetV(p, v)                              \
+  do {                                           \
+    int32_t n = 0;                               \
+    if (v) *v = 0;                               \
+    for (;;) {                                   \
+      if (p[n] <= 0x7f) {                        \
+        if (v) (*v) |= (p[n] << (7 * n));        \
+        n++;                                     \
+        break;                                   \
+      }                                          \
+      if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \
+      n++;                                       \
+    }                                            \
+    return n;                                    \
+  } while (0)
+
+// PUT
+static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) {
+  if (p) ((uint8_t*)p)[0] = v;
+  return sizeof(uint8_t);
+}
+
+static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) {
+  if (p) ((int8_t*)p)[0] = v;
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) {
+  if (p) ((uint16_t*)p)[0] = v;
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) {
+  if (p) ((int16_t*)p)[0] = v;
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) {
+  if (p) ((uint32_t*)p)[0] = v;
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) {
+  if (p) ((int32_t*)p)[0] = v;
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) {
+  if (p) ((uint64_t*)p)[0] = v;
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
+  if (p) ((int64_t*)p)[0] = v;
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); }
+
+static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); }
+
+static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); }
+
+// GET
+static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) {
+  if (v) *v = ((uint8_t*)p)[0];
+  return sizeof(uint8_t);
+}
+
+static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) {
+  if (v) *v = ((int8_t*)p)[0];
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) {
+  if (v) *v = ((uint16_t*)p)[0];
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) {
+  if (v) *v = ((int16_t*)p)[0];
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) {
+  if (v) *v = ((uint32_t*)p)[0];
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) {
+  if (v) *v = ((int32_t*)p)[0];
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) {
+  if (v) *v = ((uint64_t*)p)[0];
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) {
+  if (v) *v = ((int64_t*)p)[0];
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) {
+  int32_t  n;
+  uint16_t tv;
+
+  n = tGetU16v(p, &tv);
+  if (v) *v = ZIGZAGD(int16_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) {
+  int32_t  n;
+  uint32_t tv;
+
+  n = tGetU32v(p, &tv);
+  if (v) *v = ZIGZAGD(int32_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) {
+  int32_t  n;
+  uint64_t tv;
+
+  n = tGetU64v(p, &tv);
+  if (v) *v = ZIGZAGD(int64_t, tv);
+
+  return n;
+}
+
+// =====================
+static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) {
+  int n = 0;
+
+  n += tPutU32v(p ? p + n : p, nData);
+  if (p) memcpy(p + n, pData, nData);
+  n += nData;
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) {
+  int32_t  n = 0;
+  uint32_t nt;
+
+  n += tGetU32v(p, &nt);
+  if (nData) *nData = nt;
+  if (ppData) *ppData = p + n;
+  n += nt;
+
+  return n;
+}
 
 #ifdef __cplusplus
 }

include/util/tenv.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -13,17 +14,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TNETTEST_H
+#ifndef _TD_ENV_H_
-#define TDENGINE_TNETTEST_H
+#define _TD_ENV_H_
+
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
+int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen);
+int32_t taosEnvToCfg(const char *envStr, char *cfgStr);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_TNETTEST_H
+#endif /*_TD_ENV_H_*/

include/util/tfreelist.h

@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_FREELIST_H_
-#define _TD_UTIL_FREELIST_H_
-
-#include "tlist.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct SFreeListNode {
-  TD_SLIST_NODE(SFreeListNode);
-  char payload[];
-};
-
-typedef TD_SLIST(SFreeListNode) SFreeList;
-
-#define TFL_MALLOC(PTR, TYPE, SIZE, LIST)                      \
-  do {                                                         \
-    void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \
-    if (ptr) {                                                 \
-      TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr);      \
-      ptr = ((struct SFreeListNode *)ptr)->payload;            \
-      (PTR) = (TYPE)(ptr);                                     \
-    }else{                                                     \
-      (PTR) = NULL;                                            \
-    }                                                          \
-  }while(0);
-
-#define tFreeListInit(pFL) TD_SLIST_INIT(pFL)
-
-static FORCE_INLINE void tFreeListClear(SFreeList *pFL) {
-  struct SFreeListNode *pNode;
-  for (;;) {
-    pNode = TD_SLIST_HEAD(pFL);
-    if (pNode == NULL) break;
-    TD_SLIST_POP(pFL);
-    taosMemoryFree(pNode);
-  }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_FREELIST_H_*/

include/util/thash.h

@@ -40,6 +40,7 @@ typedef void (*_hash_free_fn_t)(void *);
  */
 uint32_t MurmurHash3_32(const char *key, uint32_t len);
 
+uint64_t MurmurHash3_64(const char *key, uint32_t len);
 /**
  *
  * @param key

include/util/tjson.h

@@ -22,13 +22,12 @@
 extern "C" {
 #endif
 
-#define tjsonGetNumberValue(pJson, pName, val) \
+#define tjsonGetNumberValue(pJson, pName, val, code)  \
-  ({ \
+  do {                                                \
     uint64_t _tmp = 0;                                \
-    int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
+    code = tjsonGetBigIntValue(pJson, pName, &_tmp); \
     val = _tmp;                                       \
-    _code; \
+  } while (0)
-  })
 
 typedef void SJson;
 
@@ -45,6 +44,8 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
 int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
 
 SJson*  tjsonGetObjectItem(const SJson* pJson, const char* pName);
+int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
+int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
 int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
 int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
 int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
@@ -77,6 +78,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
 bool   tjsonValidateJson(const char* pJson);
+const char* tjsonGetError();
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -59,6 +59,9 @@ extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
+extern int32_t metaDebugFlag;
+extern int32_t fnDebugFlag;
+extern int32_t smaDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();
@@ -78,6 +81,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #endif
     ;
 
+// clang-format off
 #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@@ -87,6 +91,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
+// clang-format on
 
 #ifdef __cplusplus
 }

include/util/tprocess.h

@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_PROCESS_H_
-#define _TD_UTIL_PROCESS_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum { PROC_FUNC_REQ = 1, PROC_FUNC_RSP, PROC_FUNC_REGIST, PROC_FUNC_RELEASE } EProcFuncType;
-
-typedef struct SProcObj SProcObj;
-typedef void *(*ProcMallocFp)(int32_t contLen);
-typedef void *(*ProcFreeFp)(void *pCont);
-typedef void (*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
-                              EProcFuncType ftype);
-
-typedef struct {
-  ProcConsumeFp childConsumeFp;
-  ProcMallocFp  childMallocHeadFp;
-  ProcFreeFp    childFreeHeadFp;
-  ProcMallocFp  childMallocBodyFp;
-  ProcFreeFp    childFreeBodyFp;
-  ProcConsumeFp parentConsumeFp;
-  ProcMallocFp  parentMallocHeadFp;
-  ProcFreeFp    parentFreeHeadFp;
-  ProcMallocFp  parentMallocBodyFp;
-  ProcFreeFp    parentFreeBodyFp;
-  SShm          shm;
-  void         *parent;
-  const char   *name;
-  bool          isChild;
-} SProcCfg;
-
-SProcObj *taosProcInit(const SProcCfg *pCfg);
-void      taosProcCleanup(SProcObj *pProc);
-int32_t   taosProcRun(SProcObj *pProc);
-void      taosProcStop(SProcObj *pProc);
-
-int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                            void *handle, EProcFuncType ftype);
-void    taosProcRemoveHandle(SProcObj *pProc, void *handle);
-void    taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
-void    taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                             EProcFuncType ftype);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_PROCESS_H_*/

include/util/tqueue.h

@@ -48,18 +48,24 @@ typedef struct {
   int32_t threadNum;
 } SQueueInfo;
 
+typedef enum {
+  DEF_QITEM = 0,
+  RPC_QITEM = 1,
+} EQItype;
+
 typedef void (*FItem)(SQueueInfo *pInfo, void *pItem);
 typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems);
 
 STaosQueue *taosOpenQueue();
 void        taosCloseQueue(STaosQueue *queue);
 void        taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
-void       *taosAllocateQitem(int32_t size);
+void       *taosAllocateQitem(int32_t size, EQItype itype);
 void        taosFreeQitem(void *pItem);
 void        taosWriteQitem(STaosQueue *queue, void *pItem);
 int32_t     taosReadQitem(STaosQueue *queue, void **ppItem);
 bool        taosQueueEmpty(STaosQueue *queue);
-int32_t     taosQueueSize(STaosQueue *queue);
+int32_t     taosQueueItemSize(STaosQueue *queue);
+int64_t     taosQueueMemorySize(STaosQueue *queue);
 
 STaosQall *taosAllocateQall();
 void       taosFreeQall(STaosQall *qall);
@@ -77,8 +83,8 @@ int32_t    taosGetQueueNumber(STaosQset *qset);
 int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp);
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
-int32_t taosGetQueueItemsNumber(STaosQueue *queue);
+
-int32_t taosGetQsetItemsNumber(STaosQset *qset);
+extern int64_t tsRpcQueueMemoryAllowed;
 
 #ifdef __cplusplus
 }