Merge branch '3.0' into feature/tdb

.gitignore

@@ -24,7 +24,6 @@ mac/
 *.orig
 src/connector/nodejs/node_modules/
 src/connector/nodejs/out/
-tests/test/
 tests/taoshebei/
 tests/taoscsv/
 tests/taosdalipu/
@@ -88,7 +87,9 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/
 tests/examples/JDBC/JDBCDemo/.classpath
 tests/examples/JDBC/JDBCDemo/.project
 tests/examples/JDBC/JDBCDemo/.settings/
-source/libs/parser/inc/new_sql.*
+source/libs/parser/inc/sql.*
+tests/script/tmqResult.txt
+tests/tmqResult.txt
 
 # Emacs
 # -*- mode: gitignore; -*-

.gitmodules

@@ -10,10 +10,9 @@
 [submodule "deps/TSZ"]
 	path = deps/TSZ
 	url = https://github.com/taosdata/TSZ.git
-[submodule "tests"]
-	path = tests
-	url = https://github.com/taosdata/tests
-	branch = 3.0
 [submodule "examples/rust"]
 	path = examples/rust
 	url = https://github.com/songtianyi/tdengine-rust-bindings.git
+[submodule "tools/taos-tools"]
+	path = tools/taos-tools
+	url = https://github.com/taosdata/taos-tools

2.0/src/client/inc/tsclient.h

@@ -116,7 +116,7 @@ typedef struct SParsedDataColInfo {
   uint16_t       allNullLen;  // TODO: get from STSchema(base on SDataRow)
   uint16_t       extendedVarLen;
   uint16_t       boundNullLen;    // bound column len with all NULL value(without VarDataOffsetT/SColIdx part)
-  int32_t *      boundedColumns;  // bound column idx according to schema
+  int32_t       *boundColumns;    // bound column idx according to schema
   SBoundColumn * cols;
   SBoundIdxInfo *colIdxInfo;
   int8_t         orderStatus;  // bound columns
@@ -125,7 +125,7 @@ typedef struct SParsedDataColInfo {
 #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
 
 typedef struct {
-  uint8_t memRowType;  // default is 0, that is SDataRow
+  uint8_t rowType;  // default is 0, that is SDataRow
   int32_t rowSize;
 } SMemRowBuilder;
 
@@ -137,17 +137,17 @@ void destroyMemRowBuilder(SMemRowBuilder *pBuilder);
 /**
  * @brief
  *
- * @param memRowType
+ * @param rowType
  * @param spd
  * @param idx   the absolute bound index of columns
  * @return FORCE_INLINE
  */
-static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd,
+static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx,
-                                                int32_t idx, int32_t *toffset, int16_t *colId) {
+                                                int32_t *toffset, int16_t *colId) {
   int32_t schemaIdx = 0;
   if (IS_DATA_COL_ORDERED(spd)) {
-    schemaIdx = spd->boundedColumns[idx];
+    schemaIdx = spd->boundColumns[idx];
-    if (isDataRowT(memRowType)) {
+    if (isDataRowT(rowType)) {
       *toffset = (spd->cols + schemaIdx)->toffset;  // the offset of firstPart
     } else {
       *toffset = idx * sizeof(SColIdx);  // the offset of SColIdx
@@ -155,7 +155,7 @@ static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRow
   } else {
     ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
     schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
-    if (isDataRowT(memRowType)) {
+    if (isDataRowT(rowType)) {
       *toffset = (spd->cols + schemaIdx)->toffset;
     } else {
       *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);

2.0/src/client/src/tscParseInsert.c

@@ -428,7 +428,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   // 1. set the parsed value from sql string
   for (int i = 0; i < spd->numOfBound; ++i) {
     // the start position in data block buffer of current value in sql
-    int32_t colIndex = spd->boundedColumns[i];
+    int32_t colIndex = spd->boundColumns[i];
 
     char *start = row + spd->cols[colIndex].offset;
 
@@ -495,7 +495,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
     bool    isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
     int32_t toffset = -1;
     int16_t colId = -1;
-    tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId);
+    tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId);
 
     int32_t ret =
         tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId);
@@ -630,7 +630,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32
   pColInfo->numOfCols = numOfCols;
   pColInfo->numOfBound = numOfCols;
   pColInfo->orderStatus = ORDER_STATUS_ORDERED;  // default is ORDERED for non-bound mode
-  pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t));
+  pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t));
   pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn));
   pColInfo->colIdxInfo = NULL;
   pColInfo->flen = 0;
@@ -656,7 +656,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32
       default:
         break;
     }
-    pColInfo->boundedColumns[i] = i;
+    pColInfo->boundColumns[i] = i;
   }
   pColInfo->allNullLen += pColInfo->flen;
   pColInfo->boundNullLen = pColInfo->allNullLen;  // default set allNullLen
@@ -991,7 +991,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
     }
 
     for (int i = 0; i < spd.numOfBound; ++i) {
-      SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]];
+      SSchema *pSchema = &pTagSchema[spd.boundColumns[i]];
 
       index = 0;
       sToken = tStrGetToken(sql, &index, true);
@@ -1158,7 +1158,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
 
   pColInfo->numOfBound = 0;
   pColInfo->boundNullLen = 0;
-  memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols);
+  memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols);
   for (int32_t i = 0; i < nCols; ++i) {
     pColInfo->cols[i].valStat = VAL_STAT_NONE;
   }
@@ -1205,7 +1205,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
         }
 
         pColInfo->cols[t].valStat = VAL_STAT_HAS;
-        pColInfo->boundedColumns[pColInfo->numOfBound] = t;
+        pColInfo->boundColumns[pColInfo->numOfBound] = t;
         ++pColInfo->numOfBound;
         switch (pSchema[t].type) {
           case TSDB_DATA_TYPE_BINARY:
@@ -1239,7 +1239,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
           }
 
           pColInfo->cols[t].valStat = VAL_STAT_HAS;
-          pColInfo->boundedColumns[pColInfo->numOfBound] = t;
+          pColInfo->boundColumns[pColInfo->numOfBound] = t;
           ++pColInfo->numOfBound;
           switch (pSchema[t].type) {
             case TSDB_DATA_TYPE_BINARY:
@@ -1279,7 +1279,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
     }
     SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo;
     for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) {
-      pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i];
+      pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i];
       pColIdx[i].boundIdx = i;
     }
     qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
@@ -1289,7 +1289,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
     qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
-  memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0,
+  memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0,
          sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound));
 
   return TSDB_CODE_SUCCESS;

2.0/src/client/src/tscUtil.c

@@ -1554,7 +1554,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
 }
 
 void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) {
-  tfree(pColInfo->boundedColumns);
+  tfree(pColInfo->boundColumns);
   tfree(pColInfo->cols);
   tfree(pColInfo->colIdxInfo);
 }

2.0/src/query/inc/qExecutor.h

@@ -589,7 +589,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRunt
 SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream);
-SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
+SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
 SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);

2.0/src/query/src/qExecutor.c

@@ -2166,7 +2166,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
       }
       case OP_TimeWindow: {
         pRuntimeEnv->proot =
-            createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput);
+            createIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput);
         int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType;
         if (opType != OP_DummyInput && opType != OP_Join) {
           setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot);
@@ -6756,7 +6756,7 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI
   return pOperator;
 }
 
-SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {
+SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {
   STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
 
   pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset);

CMakeLists.txt

@@ -8,10 +8,11 @@ project(
 
 set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
 set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
-include(${CMAKE_SUPPORT_DIR}/cmake.options)
 
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
+include(${CMAKE_SUPPORT_DIR}/cmake.platform)
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
+include(${CMAKE_SUPPORT_DIR}/cmake.define)
+include(${CMAKE_SUPPORT_DIR}/cmake.options)
+include(${CMAKE_SUPPORT_DIR}/cmake.version)
 
 # contrib
 add_subdirectory(contrib)
@@ -25,6 +26,7 @@ if(${BUILD_TEST})
     include(CTest)
     enable_testing()
 endif(${BUILD_TEST})
+
 add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(tests)

Jenkinsfile2

@@ -35,6 +35,7 @@ def abort_previous(){
 def pre_test(){
     sh'hostname'
     sh '''
+    date
     sudo rmtaos || echo "taosd has not installed"
     '''
     sh '''
@@ -60,6 +61,7 @@ def pre_test(){
         sh '''
         cd ${WKC}
         git checkout 3.0
+        [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../..
         '''
       }
       else{
@@ -74,37 +76,9 @@ def pre_test(){
     git pull >/dev/null
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
-    git submodule update --init --recursive --remote
+    git submodule update --init --recursive
     '''
-    script {
-      if (env.CHANGE_TARGET == 'master') {
     sh'''
-        cd ${WKCT}
-        git checkout master
-        '''
-        }
-      else if(env.CHANGE_TARGET == '2.0'){
-        sh '''
-        cd ${WKCT}
-        git checkout 2.0
-        '''
-      }
-      else if(env.CHANGE_TARGET == '3.0'){
-        sh '''
-        cd ${WKCT}
-        git checkout 3.0
-        '''
-      } 
-      else{
-        sh '''
-        cd ${WKCT}
-        git checkout develop
-        '''
-      }
-    }
-    sh'''
-    cd ${WKCT}
-    git pull >/dev/null
     cd ${WKC}
     export TZ=Asia/Harbin
     date
@@ -123,11 +97,10 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
-      WKCT= '/var/lib/jenkins/workspace/TDengine/tests'
   }
   stages {
       stage('pre_build'){
-          agent{label 'slave3_0'}
+          agent{label " slave3_0 || slave15 || slave16 || slave17 "}
           options { skipDefaultCheckout() } 
           when {
               changeRequest()

cmake/cmake.define

@@ -0,0 +1,49 @@
+cmake_minimum_required(VERSION 3.16)
+
+IF ("${BUILD_TOOLS}" STREQUAL "")
+  IF (TD_LINUX)
+    IF (TD_ARM_32)
+      SET(BUILD_TOOLS "false")
+    ELSEIF (TD_ARM_64)
+      SET(BUILD_TOOLS "false")
+    ELSE ()
+      SET(BUILD_TOOLS "false")
+    ENDIF ()
+  ELSEIF (TD_DARWIN)
+    SET(BUILD_TOOLS "false")
+  ELSE ()
+    SET(BUILD_TOOLS "false")
+  ENDIF ()
+ENDIF ()
+
+IF ("${BUILD_TOOLS}" MATCHES "false")
+    MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}")
+    SET(TD_TAOS_TOOLS FALSE)
+ELSE ()
+    MESSAGE("")
+    MESSAGE("${Green} Will build taos_tools! ${ColourReset}")
+    MESSAGE("")
+    SET(TD_TAOS_TOOLS TRUE)
+ENDIF ()
+
+IF (TD_WINDOWS)
+    MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
+    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+
+    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    ENDIF ()
+
+ELSE ()
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
+
+MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
+IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
+    ADD_DEFINITIONS("-D_TD_ARM_")
+ELSE ()
+    ADD_DEFINITIONS("-msse4.2 -mfma")
+ENDIF ()
+
+ENDIF ()

cmake/cmake.options

@@ -1,12 +1,42 @@
 # =========================================================
 # Deps options
 # =========================================================
+
+IF(${TD_WINDOWS})
+
+    MESSAGE("build pthread Win32")
+    option(
+            BUILD_PTHREAD
+            "If build pthread on Windows"
+            ON
+          )
+
+    MESSAGE("build gnu regex for Windows")
+    option(
+            BUILD_GNUREGEX
+            "If build gnu regex on Windows"
+            ON
+          )
+
+    MESSAGE("build iconv Win32")
+    option(
+            BUILD_WITH_ICONV
+            "If build iconv on Windows"
+            ON
+          )
+
+ENDIF ()
+
+IF(${TD_LINUX} MATCHES TRUE)
+
 option(
     BUILD_TEST 
     "If build unit tests using googletest" 
     ON
 )
 
+ENDIF ()
+
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 
@@ -25,12 +55,17 @@ option(
     OFF
 )
 
+IF(${TD_WINDOWS})
+    MESSAGE("Not build BDB on Windows")
+ELSE ()
     option(
             BUILD_WITH_BDB
             "If build with BerkleyDB"
             ON
           )
 
+ENDIF ()
+
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 
@@ -68,12 +103,16 @@ option(
     OFF
 )
 
+IF(${TD_LINUX} MATCHES TRUE)
+
 option(
     BUILD_DEPENDENCY_TESTS 
     "If build dependency tests" 
     ON
 )
 
+ENDIF ()
+
 option(
     BUILD_DOCS
     "If use doxygen build documents" 

cmake/cmake.platform

@@ -0,0 +1,74 @@
+cmake_minimum_required(VERSION 3.16)
+
+MESSAGE("Current system is ${CMAKE_SYSTEM_NAME}")
+
+# init
+SET(TD_LINUX FALSE)
+SET(TD_WINDOWS FALSE)
+SET(TD_DARWIN FALSE)
+
+MESSAGE("Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
+if(CMAKE_COMPILER_IS_GNUCXX MATCHES 1)
+    set(CXX_COMPILER_IS_GNU TRUE)
+else()
+    set(CXX_COMPILER_IS_GNU FALSE)
+endif()
+
+MESSAGE("Current system name is ${CMAKE_SYSTEM_NAME}.")
+
+IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+
+    IF (${CXX_COMPILER_IS_GNU})
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
+    ELSE ()
+        ADD_DEFINITIONS("-Wno-tautological-constant-out-of-range-compare -Wno-pointer-sign -Wno-unknown-warning-option")
+        set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup")
+        set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
+    ENDIF ()
+
+    IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+
+        SET(TD_LINUX TRUE)
+        SET(OSTYPE "Linux")
+        ADD_DEFINITIONS("-DLINUX")
+
+        IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
+            SET(TD_LINUX_64 TRUE)
+        ELSE ()
+            SET(TD_LINUX_32 TRUE)
+        ENDIF ()
+
+    ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+
+        SET(TD_DARWIN TRUE)
+        SET(OSTYPE "macOS")
+        ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare -Wno-return-type")
+
+        MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
+        IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
+            MESSAGE("Current system arch is arm64")
+            SET(TD_DARWIN_64 TRUE)
+            ADD_DEFINITIONS("-D_TD_DARWIN_64")
+        ENDIF ()
+
+        ADD_DEFINITIONS("-DHAVE_UNISTD_H")
+    ENDIF ()
+
+ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+
+    SET(TD_WINDOWS TRUE)
+    SET(OSTYPE "Windows")
+    ADD_DEFINITIONS("-DWINDOWS")
+
+    IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
+        SET(TD_WINDOWS_64 TRUE)
+    	ADD_DEFINITIONS("-D_TD_WINDOWS_64")
+    ELSE ()
+        SET(TD_WINDOWS_32 TRUE)
+    	ADD_DEFINITIONS("-D_TD_WINDOWS_32")
+    ENDIF ()
+
+ENDIF()
+
+MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}")
+MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}")

cmake/cmake.version

@@ -0,0 +1,77 @@
+
+IF (DEFINED VERNUMBER)
+  SET(TD_VER_NUMBER ${VERNUMBER})
+ELSE ()
+  SET(TD_VER_NUMBER "3.0.0.0")
+ENDIF ()
+
+IF (DEFINED VERCOMPATIBLE)
+  SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE})
+ELSE ()
+  SET(TD_VER_COMPATIBLE "3.0.0.0")
+ENDIF ()
+
+find_program(HAVE_GIT NAMES git)
+
+IF (DEFINED GITINFO)
+  SET(TD_VER_GIT ${GITINFO})
+ELSEIF (HAVE_GIT)
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+  #message(STATUS "git log result:${GIT_COMMITID}")
+  IF (GIT_COMMITID)
+    string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
+    SET(TD_VER_GIT ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
+ELSE ()
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT "no git commit id")
+ENDIF ()
+
+IF (DEFINED VERDATE)
+  SET(TD_VER_DATE ${VERDATE})
+ELSE ()
+  STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") 
+ENDIF ()
+
+IF (DEFINED VERTYPE)
+  SET(TD_VER_VERTYPE ${VERTYPE})
+ELSE ()
+  SET(TD_VER_VERTYPE "stable")
+ENDIF ()
+
+IF (DEFINED CPUTYPE)
+  SET(TD_VER_CPUTYPE ${CPUTYPE})
+ELSE ()
+  IF (TD_WINDOWS_32)
+    SET(TD_VER_CPUTYPE "x86")
+  ELSEIF (TD_LINUX_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSEIF (TD_ARM_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSEIF (TD_MIPS_32)
+    SET(TD_VER_CPUTYPE "x86")  
+  ELSE ()
+    SET(TD_VER_CPUTYPE "x64")
+  ENDIF ()
+ENDIF ()
+
+IF (DEFINED OSTYPE)
+  SET(TD_VER_OSTYPE ${OSTYPE})
+ELSE ()
+  SET(TD_VER_OSTYPE "Linux")
+ENDIF ()
+
+MESSAGE(STATUS "============= compile version parameter information start ============= ")
+MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER})
+MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE})
+MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT})
+MESSAGE(STATUS "build date:" ${TD_VER_DATE})
+MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE})
+MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE})
+MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE})
+MESSAGE(STATUS "============= compile version parameter information end  ============= ")
+
+STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})

cmake/gnuregex_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# gnuregex
+ExternalProject_Add(gnuregex
+        URL https://launchpad.net/gnuregex/trunk/2.9/+download/libgnurx-src-2.9.zip
+        DOWNLOAD_NAME libgnurx-src.zip
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/gnuregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/iconv_CMakeLists.txt.in

@@ -0,0 +1,12 @@
+
+# iconv
+ExternalProject_Add(iconv
+        GIT_REPOSITORY https://github.com/win-iconv/win-iconv.git
+        GIT_TAG v0.0.8
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/iconv"
+        BINARY_DIR ""
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

cmake/pthread_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# pthread
+ExternalProject_Add(pthread
+        GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
+        GIT_TAG v3.0.3.1
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/pthread-win32"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

contrib/CMakeLists.txt

@@ -9,6 +9,16 @@ endfunction(cat IN_FILE OUT_FILE)
 set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
 configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
+# pthread
+if(${BUILD_PTHREAD})
+    cat("${CMAKE_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# gnu regex
+if(${BUILD_GNUREGEX})
+    cat("${CMAKE_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
 # googletest
 if(${BUILD_TEST})
     cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -117,6 +127,11 @@ target_include_directories(
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
 add_subdirectory(zlib)
+target_include_directories(
+    zlibstatic
+    PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
+    PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
+)
 target_include_directories(
     zlib
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@@ -193,6 +208,10 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
+    if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        MESSAGE("Windows need set no-sign-compare")
+        add_compile_options(-Wno-sign-compare)
+    endif ()
     add_subdirectory(libuv)
 endif(${BUILD_WITH_UV})
 
@@ -223,6 +242,7 @@ if(${BUILD_WITH_SQLITE})
     )
 endif(${BUILD_WITH_SQLITE})
 
+# pthread
 
 
 # ================================================================================================

contrib/test/craft/raftMain.c

@@ -10,7 +10,7 @@
 #include <raft.h>
 #include <raft/uv.h>
 #include "raftServer.h"
-#include "common.h"
+#include "tcommon.h"
 
 const char *exe_name;
 
@@ -377,7 +377,7 @@ void printConf(SRaftServerConfig *pConf) {
 
 
 int main(int argc, char **argv) { 
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 	int32_t ret;
 
 	exe_name = argv[0];

contrib/test/craft/raftServer.c

@@ -1,5 +1,5 @@
 #include <stdlib.h>
-#include "common.h"
+#include "tcommon.h"
 #include "raftServer.h"
 
 char *keys;

contrib/test/craft/raftServer.h

@@ -11,7 +11,7 @@ extern "C" {
 #include <string.h>
 #include "raft.h"
 #include "raft/uv.h"
-#include "common.h"
+#include "tcommon.h"
 
 
 // simulate a db store, just for test

contrib/test/craft/simulate_vnode.c

@@ -132,7 +132,7 @@ static void proposeValue(struct raft *r) {
     buf.base = raft_malloc(buf.len);
 
     // mock ts value
-    int vid = rand() % VNODE_COUNT;
+    int vid = taosRand() % VNODE_COUNT;
     snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL));
 
 	printf("propose value: %s \n", (char*)buf.base);
@@ -174,7 +174,7 @@ void usage() {
 }
 
 int main(int argc, char **argv) {
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 
 	exe_name = argv[0];
 	if (argc < 2) {

example/CMakeLists.txt

@@ -1,16 +1,30 @@
-aux_source_directory(src TMQ_DEMO_SRC)
+add_executable(tmq "")
+add_executable(tstream "")
 
-add_executable(tmq ${TMQ_DEMO_SRC})
+target_sources(tmq
-target_link_libraries(
+    PRIVATE
-    tmq
+    "src/tmq.c"
-    PUBLIC taos
-    #PUBLIC util
-    #PUBLIC common
-    #PUBLIC os
 )
-target_include_directories(
+
-    tmq
+target_sources(tstream
+    PRIVATE
+    "src/tstream.c"
+)
+target_link_libraries(tmq
+    taos
+)
+
+target_link_libraries(tstream
+    taos
+)
+
+target_include_directories(tmq
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_include_directories(tstream
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
 SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
+SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)

example/src/tmq.c

@@ -44,7 +44,7 @@ int32_t init_env() {
 
   pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
+    printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
@@ -66,7 +66,7 @@ int32_t init_env() {
 }
 
 int32_t create_topic() {
-  printf("create topic");
+  printf("create topic\n");
   TAOS_RES* pRes;
   TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
@@ -91,6 +91,10 @@ int32_t create_topic() {
   return 0;
 }
 
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+  printf("commit %d\n", resp);
+}
+
 tmq_t* build_consumer() {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -103,6 +107,7 @@ tmq_t* build_consumer() {
 
   tmq_conf_t* conf = tmq_conf_new();
   tmq_conf_set(conf, "group.id", "tg2");
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
   tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
   return tmq;
 }
@@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
-  static const int MIN_COMMIT_COUNT = 1000;
+  static const int MIN_COMMIT_COUNT = 1;
 
   int            msg_count = 0;
   tmq_resp_err_t err;
@@ -155,7 +160,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   }
 
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
@@ -214,6 +219,6 @@ int main(int argc, char* argv[]) {
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   /*perf_loop(tmq, topic_list);*/
-  basic_consume_loop(tmq, topic_list);
+  /*basic_consume_loop(tmq, topic_list);*/
-  /*sync_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
 }

example/src/tstream.c

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include "taos.h"
+
+int32_t init_env() {
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  if (pConn == NULL) {
+    return -1;
+  }
+
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  return 0;
+}
+
+int32_t create_stream() {
+  printf("create stream\n");
+  TAOS_RES* pRes;
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  if (pConn == NULL) {
+    return -1;
+  }
+
+  pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
+  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
+  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  taos_close(pConn);
+  return 0;
+}
+
+int main(int argc, char* argv[]) {
+  int code;
+  if (argc > 1) {
+    printf("env init\n");
+    code = init_env();
+  }
+  create_stream();
+#if 0
+  tmq_t*      tmq = build_consumer();
+  tmq_list_t* topic_list = build_topic_list();
+  /*perf_loop(tmq, topic_list);*/
+  /*basic_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
+#endif
+}

examples/c/schemaless.c

@@ -19,7 +19,7 @@ void shuffle(char**lines, size_t n)
     size_t i;
     for (i = 0; i < n - 1; i++)
     {
-      size_t j = i + rand() / (RAND_MAX / (n - i) + 1);
+      size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1);
       char* t = lines[j];
       lines[j] = lines[i];
       lines[i] = t;

include/client/taos.h

@@ -39,18 +39,19 @@ typedef void **TAOS_ROW;
 #define TSDB_DATA_TYPE_BIGINT     5   // 8 bytes
 #define TSDB_DATA_TYPE_FLOAT      6   // 4 bytes
 #define TSDB_DATA_TYPE_DOUBLE     7   // 8 bytes
-#define TSDB_DATA_TYPE_BINARY    8   // string, alias for varchar
+#define TSDB_DATA_TYPE_VARCHAR    8   // string, alias for varchar
 #define TSDB_DATA_TYPE_TIMESTAMP  9   // 8 bytes
 #define TSDB_DATA_TYPE_NCHAR      10  // unicode string
 #define TSDB_DATA_TYPE_UTINYINT   11  // 1 byte
 #define TSDB_DATA_TYPE_USMALLINT  12  // 2 bytes
 #define TSDB_DATA_TYPE_UINT       13  // 4 bytes
 #define TSDB_DATA_TYPE_UBIGINT    14  // 8 bytes
-#define TSDB_DATA_TYPE_VARCHAR   15  // string
+#define TSDB_DATA_TYPE_JSON       15  // json string
 #define TSDB_DATA_TYPE_VARBINARY  16  // binary
-#define TSDB_DATA_TYPE_JSON      17  // json
+#define TSDB_DATA_TYPE_DECIMAL    17  // decimal
-#define TSDB_DATA_TYPE_DECIMAL   18  // decimal
+#define TSDB_DATA_TYPE_BLOB       18  // binary
-#define TSDB_DATA_TYPE_BLOB      19  // binary
+#define TSDB_DATA_TYPE_MEDIUMBLOB 19
+#define TSDB_DATA_TYPE_BINARY     TSDB_DATA_TYPE_VARCHAR  // string
 
 typedef enum {
   TSDB_OPTION_LOCALE,
@@ -84,11 +85,7 @@ typedef struct taosField {
   int32_t bytes;
 } TAOS_FIELD;
 
-#ifdef _TD_GO_DLL_
-#define DLL_EXPORT __declspec(dllexport)
-#else
 #define DLL_EXPORT
-#endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
@@ -198,8 +195,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
 /* --------------------------TMQ INTERFACE------------------------------- */
 
 enum tmq_resp_err_t {
+  TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
-  TMQ_RESP_ERR__FAIL = 1,
 };
 
 typedef enum tmq_resp_err_t tmq_resp_err_t;
@@ -216,18 +213,17 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
+DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 
-DLL_EXPORT TAOS_RES   *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
 DLL_EXPORT tmq_t      *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+DLL_EXPORT tmq_t      *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 DLL_EXPORT void        tmq_message_destroy(tmq_message_t *tmq_message);
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
-#if 0
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
-DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics);
+DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-#endif
 DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 #if 0
@@ -238,6 +234,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 #endif
+DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {
@@ -249,14 +246,30 @@ enum tmq_conf_res_t {
 typedef enum tmq_conf_res_t tmq_conf_res_t;
 
 DLL_EXPORT tmq_conf_t    *tmq_conf_new();
-DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
+DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
 
 // temporary used function for demo only
 void    tmqShowMsg(tmq_message_t *tmq_message);
 int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 
+/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
+
+DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
+DLL_EXPORT char    *tmq_get_topic_name(tmq_message_t *message);
+DLL_EXPORT char    *tmq_get_topic_schema(tmq_t *tmq, const char *topic);
+
+/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
+DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
+
+DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
+
+/* -------------------------------- OTHER -------------------------------- */
+typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
+
+DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
+
 #ifdef __cplusplus
 }
 #endif

include/common/common.h

@@ -1,283 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_COMMON_H
-#define TDENGINE_COMMON_H
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "taosdef.h"
-#include "tarray.h"
-#include "tmsg.h"
-#include "tvariant.h"
-// typedef struct STimeWindow {
-//   TSKEY skey;
-//   TSKEY ekey;
-// } STimeWindow;
-
-// typedef struct {
-//   int32_t dataLen;
-//   char    name[TSDB_TABLE_FNAME_LEN];
-//   char   *data;
-// } STagData;
-
-// typedef struct SSchema {
-//   uint8_t type;
-//   char    name[TSDB_COL_NAME_LEN];
-//   int16_t colId;
-//   int16_t bytes;
-// } SSchema;
-
-#define TMQ_REQ_TYPE_COMMIT_ONLY 0
-#define TMQ_REQ_TYPE_CONSUME_ONLY 1
-#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2
-
-typedef struct {
-  uint32_t  numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
-} STableGroupInfo;
-
-typedef struct SColumnDataAgg {
-  int16_t colId;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-} SColumnDataAgg;
-
-typedef struct SDataBlockInfo {
-  STimeWindow window;
-  int32_t     rows;
-  int32_t     numOfCols;
-  int64_t     uid;
-} SDataBlockInfo;
-
-typedef struct SConstantItem {
-  SColumnInfo info;
-  int32_t     startRow;  // run-length-encoding to save the space for multiple rows
-  int32_t     endRow;
-  SVariant    value;
-} SConstantItem;
-
-// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
-typedef struct SSDataBlock {
-  SColumnDataAgg *pBlockAgg;
-  SArray         *pDataBlock;    // SArray<SColumnInfoData>
-  SArray         *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
-  SDataBlockInfo  info;
-} SSDataBlock;
-
-typedef struct SVarColAttr {
-  int32_t *offset;    // start position for each entry in the list
-  uint32_t length;    // used buffer size that contain the valid data
-  uint32_t allocLen;  // allocated buffer size
-} SVarColAttr;
-
-// pBlockAgg->numOfNull == info.rows, all data are null
-// pBlockAgg->numOfNull == 0, no data are null.
-typedef struct SColumnInfoData {
-  SColumnInfo info;   // TODO filter info needs to be removed
-  bool        hasNull;// if current column data has null value.
-  char       *pData;  // the corresponding block data in memory
-  union {
-    char       *nullbitmap;  // bitmap, one bit for each item in the list
-    SVarColAttr varmeta;
-  };
-} SColumnInfoData;
-
-static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
-  int64_t tbUid = pBlock->info.uid;
-  int32_t numOfCols = pBlock->info.numOfCols;
-  int32_t rows = pBlock->info.rows;
-  int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
-
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, tbUid);
-  tlen += taosEncodeFixedI32(buf, numOfCols);
-  tlen += taosEncodeFixedI32(buf, rows);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
-    tlen += taosEncodeFixedI16(buf, pColData->info.colId);
-    tlen += taosEncodeFixedI16(buf, pColData->info.type);
-    tlen += taosEncodeFixedI16(buf, pColData->info.bytes);
-    int32_t colSz = rows * pColData->info.bytes;
-    tlen += taosEncodeBinary(buf, pColData->pData, colSz);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
-  int32_t sz;
-
-  buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
-  buf = taosDecodeFixedI32(buf, &pBlock->info.numOfCols);
-  buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData));
-  for (int32_t i = 0; i < sz; i++) {
-    SColumnInfoData data = {0};
-    buf = taosDecodeFixedI16(buf, &data.info.colId);
-    buf = taosDecodeFixedI16(buf, &data.info.type);
-    buf = taosDecodeFixedI16(buf, &data.info.bytes);
-    int32_t colSz = pBlock->info.rows * data.info.bytes;
-    buf = taosDecodeBinary(buf, (void**)&data.pData, colSz);
-    taosArrayPush(pBlock->pDataBlock, &data);
-  }
-  return (void*)buf;
-}
-
-static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) {
-  int32_t tlen = 0;
-  int32_t sz = 0;
-  tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->committedOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
-  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
-  tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return tlen;
-  tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas);
-  if (pRsp->pBlockData) {
-    sz = taosArrayGetSize(pRsp->pBlockData);
-  }
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-    tlen += tEncodeDataBlock(buf, pBlock);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
-  int32_t sz;
-  buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->committedOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
-  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
-  buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return buf;
-  pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper));
-  if (pRsp->schemas == NULL) return NULL;
-  buf = tDecodeSSchemaWrapper(buf, pRsp->schemas);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock block = {0};
-    tDecodeDataBlock(buf, &block);
-    taosArrayPush(pRsp->pBlockData, &block);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
-  if (pBlock == NULL) {
-    return;
-  }
-
-  // int32_t numOfOutput = pBlock->info.numOfCols;
-  int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
-  for (int32_t i = 0; i < sz; ++i) {
-    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
-    tfree(pColInfoData->pData);
-  }
-
-  taosArrayDestroy(pBlock->pDataBlock);
-  tfree(pBlock->pBlockAgg);
-  // tfree(pBlock);
-}
-
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) {
-  if (pRsp->schemas) {
-    if (pRsp->schemas->nCols) {
-      tfree(pRsp->schemas->pSchema);
-    }
-    free(pRsp->schemas);
-  }
-  taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock);
-  pRsp->pBlockData = NULL;
-  // for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) {
-  // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-  // tDeleteSSDataBlock(pDataBlock);
-  //}
-}
-
-//======================================================================================================================
-// the following structure shared by parser and executor
-typedef struct SColumn {
-  uint64_t    uid;
-  char        name[TSDB_COL_NAME_LEN];
-  int8_t      flag;  // column type: normal column, tag, or user-input column (integer/float/string)
-  SColumnInfo info;
-} SColumn;
-
-typedef struct SLimit {
-  int64_t limit;
-  int64_t offset;
-} SLimit;
-
-typedef struct SOrder {
-  uint32_t order;
-  SColumn  col;
-} SOrder;
-
-typedef struct SGroupbyExpr {
-  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
-  bool    groupbyTag;  // group by tag or column
-} SGroupbyExpr;
-
-// the structure for sql function in select clause
-typedef struct SSqlExpr {
-  char    token[TSDB_COL_NAME_LEN];  // original token
-  SSchema resSchema;
-
-  int32_t   numOfCols;
-  SColumn*  pColumns;       // data columns that are required by query
-  int32_t   interBytes;     // inter result buffer size
-  int16_t   numOfParams;    // argument value of each function
-  SVariant  param[3];       // parameters are not more than 3
-} SSqlExpr;
-
-typedef struct SExprInfo {
-  struct SSqlExpr   base;
-  struct tExprNode* pExpr;
-} SExprInfo;
-
-typedef struct SStateWindow {
-  SColumn col;
-} SStateWindow;
-
-typedef struct SSessionWindow {
-  int64_t gap;  // gap between two session window(in microseconds)
-  SColumn col;
-} SSessionWindow;
-
-#define QUERY_ASC_FORWARD_STEP 1
-#define QUERY_DESC_FORWARD_STEP -1
-
-#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_COMMON_H

include/common/taosdef.h

@@ -16,44 +16,29 @@
 #ifndef _TD_COMMON_TAOS_DEF_H_
 #define _TD_COMMON_TAOS_DEF_H_
 
+#include "taos.h"
+#include "tdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "taos.h"
-#include "tdef.h"
-
 typedef int64_t tb_uid_t;
 
 #define TSWINDOW_INITIALIZER       ((STimeWindow){INT64_MIN, INT64_MAX})
 #define TSWINDOW_DESC_INITIALIZER  ((STimeWindow){INT64_MAX, INT64_MIN})
 #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
 
-typedef enum {
-  TAOS_QTYPE_RPC = 1,
-  TAOS_QTYPE_FWD = 2,
-  TAOS_QTYPE_WAL = 3,
-  TAOS_QTYPE_CQ = 4,
-  TAOS_QTYPE_QUERY = 5
-} EQType;
-
 typedef enum {
   TSDB_SUPER_TABLE  = 1,    // super table
   TSDB_CHILD_TABLE  = 2,    // table created from super table
   TSDB_NORMAL_TABLE = 3,    // ordinary table
   TSDB_STREAM_TABLE = 4,    // table created from stream computing
   TSDB_TEMP_TABLE   = 5,    // temp table created by nest query
-  TSDB_TABLE_MAX = 6
+  TSDB_SYSTEM_TABLE = 6,
+  TSDB_TABLE_MAX    = 7
 } ETableType;
 
-typedef enum {
-  TSDB_MOD_MNODE = 1,
-  TSDB_MOD_HTTP = 2,
-  TSDB_MOD_MONITOR = 3,
-  TSDB_MOD_MQTT = 4,
-  TSDB_MOD_MAX = 5
-} EModuleType;
-
 typedef enum {
   TSDB_CHECK_ITEM_NETWORK,
   TSDB_CHECK_ITEM_MEM,
@@ -66,10 +51,35 @@ typedef enum {
   TSDB_CHECK_ITEM_MAX
 } ECheckItemType;
 
-typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig;
+typedef enum {
+  TD_ROW_DISCARD_UPDATE = 0,
+  TD_ROW_OVERWRITE_UPDATE = 1,
+  TD_ROW_PARTIAL_UPDATE = 2,
+} TDUpdateConfig;
+
+typedef enum {
+  TSDB_STATIS_OK = 0,    // statis part exist and load successfully
+  TSDB_STATIS_NONE = 1,  // statis part not exist
+} ETsdbStatisStatus;
+
+typedef enum {
+  TSDB_SMA_STAT_UNKNOWN = -1,  // unknown
+  TSDB_SMA_STAT_OK = 0,        // ready to provide service
+  TSDB_SMA_STAT_EXPIRED = 1,   // not ready or expired
+  TSDB_SMA_STAT_DROPPED = 2,   // sma dropped
+} ETsdbSmaStat;                // bit operation
+
+
+typedef enum {
+  TSDB_SMA_TYPE_BLOCK = 0,       // Block-wise SMA
+  TSDB_SMA_TYPE_TIME_RANGE = 1,  // Time-range-wise SMA
+  TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
+} ETsdbSmaType;
 
 extern char *qtypeStr[];
 
+#define TSDB_PORT_HTTP      11
+
 #ifdef __cplusplus
 }
 #endif

include/common/tcommon.h

@@ -0,0 +1,254 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_DEF_H_
+#define _TD_COMMON_DEF_H_
+
+#include "taosdef.h"
+#include "tarray.h"
+#include "tmsg.h"
+#include "tvariant.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+  TMQ_CONF__RESET_OFFSET__LATEST = -1,
+  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
+  TMQ_CONF__RESET_OFFSET__NONE = -3,
+};
+
+enum {
+  TMQ_MSG_TYPE__DUMMY = 0,
+  TMQ_MSG_TYPE__POLL_RSP,
+  TMQ_MSG_TYPE__EP_RSP,
+};
+
+typedef struct {
+  uint32_t  numOfTables;
+  SArray*   pGroupList;
+  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
+} STableGroupInfo;
+
+typedef struct SColumnDataAgg {
+  int16_t colId;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+} SColumnDataAgg;
+
+typedef struct SDataBlockInfo {
+  STimeWindow window;
+  int32_t     rows;
+  int32_t     rowSize;
+  int16_t     numOfCols;
+  int16_t     hasVarCol;
+  union {
+    int64_t uid;
+    int64_t blockId;
+  };
+  int64_t groupId;  // no need to serialize
+} SDataBlockInfo;
+
+typedef struct SSDataBlock {
+  SColumnDataAgg* pBlockAgg;
+  SArray*         pDataBlock;  // SArray<SColumnInfoData>
+  SDataBlockInfo  info;
+} SSDataBlock;
+
+typedef struct SVarColAttr {
+  int32_t* offset;    // start position for each entry in the list
+  uint32_t length;    // used buffer size that contain the valid data
+  uint32_t allocLen;  // allocated buffer size
+} SVarColAttr;
+
+// pBlockAgg->numOfNull == info.rows, all data are null
+// pBlockAgg->numOfNull == 0, no data are null.
+typedef struct SColumnInfoData {
+  SColumnInfo info;     // TODO filter info needs to be removed
+  bool        hasNull;  // if current column data has null value.
+  char*       pData;    // the corresponding block data in memory
+  union {
+    char*       nullbitmap;  // bitmap, one bit for each item in the list
+    SVarColAttr varmeta;
+  };
+} SColumnInfoData;
+
+void*   blockDataDestroy(SSDataBlock* pBlock);
+int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
+void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
+
+int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
+void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
+
+static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
+  // WARNING: do not use info.numOfCols,
+  // sometimes info.numOfCols != array size
+  int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfOutput; ++i) {
+    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
+    if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+      taosMemoryFreeClear(pColInfoData->varmeta.offset);
+    } else {
+      taosMemoryFreeClear(pColInfoData->nullbitmap);
+    }
+
+    taosMemoryFreeClear(pColInfoData->pData);
+  }
+
+  taosArrayDestroy(pBlock->pDataBlock);
+  taosMemoryFreeClear(pBlock->pBlockAgg);
+}
+
+static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
+
+static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
+  int32_t tlen = 0;
+  int32_t sz = 0;
+  // tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
+  if (pRsp->numOfTopics == 0) return tlen;
+  tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema);
+  if (pRsp->pBlockData) {
+    sz = taosArrayGetSize(pRsp->pBlockData);
+  }
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
+    tlen += tEncodeDataBlock(buf, pBlock);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
+  int32_t sz;
+  // buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
+  if (pRsp->numOfTopics == 0) return buf;
+  pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
+  if (pRsp->schema == NULL) return NULL;
+  buf = taosDecodeSSchemaWrapper(buf, pRsp->schema);
+  buf = taosDecodeFixedI32(buf, &sz);
+  pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock block = {0};
+    tDecodeDataBlock(buf, &block);
+    taosArrayPush(pRsp->pBlockData, &block);
+  }
+  return buf;
+}
+
+static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
+  if (pRsp->schema) {
+    if (pRsp->schema->nCols) {
+      taosMemoryFreeClear(pRsp->schema->pSchema);
+    }
+    taosMemoryFree(pRsp->schema);
+  }
+  taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
+  pRsp->pBlockData = NULL;
+}
+
+//======================================================================================================================
+// the following structure shared by parser and executor
+typedef struct SColumn {
+  union {
+    uint64_t uid;
+    int64_t  dataBlockId;
+  };
+
+  int16_t colId;
+  int16_t slotId;
+
+  char    name[TSDB_COL_NAME_LEN];
+  int8_t  flag;  // column type: normal column, tag, or user-input column (integer/float/string)
+  int16_t type;
+  int32_t bytes;
+  uint8_t precision;
+  uint8_t scale;
+} SColumn;
+
+typedef struct SLimit {
+  int64_t limit;
+  int64_t offset;
+} SLimit;
+
+typedef struct SOrder {
+  uint32_t order;
+  SColumn  col;
+} SOrder;
+
+typedef struct SGroupbyExpr {
+  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
+  bool    groupbyTag;  // group by tag or column
+} SGroupbyExpr;
+
+typedef struct SFunctParam {
+  int32_t  type;
+  SColumn* pCol;
+  SVariant param;
+} SFunctParam;
+
+// the structure for sql function in select clause
+typedef struct SResSchame {
+  int8_t  type;
+  int32_t colId;
+  int32_t bytes;
+  int32_t precision;
+  int32_t scale;
+  char    name[TSDB_COL_NAME_LEN];
+} SResSchema;
+
+// TODO move away to executor.h
+typedef struct SExprBasicInfo {
+  SResSchema   resSchema;
+  int16_t      numOfParams;  // argument value of each function
+  SFunctParam* pParam;
+} SExprBasicInfo;
+
+typedef struct SExprInfo {
+  struct SExprBasicInfo base;
+  struct tExprNode*     pExpr;
+} SExprInfo;
+
+typedef struct SStateWindow {
+  SColumn col;
+} SStateWindow;
+
+typedef struct SSessionWindow {
+  int64_t gap;  // gap between two session window(in microseconds)
+  SColumn col;
+} SSessionWindow;
+
+#define QUERY_ASC_FORWARD_STEP  1
+#define QUERY_DESC_FORWARD_STEP -1
+
+#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_DEF_H_*/

include/common/tdatablock.h

@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_EP_H_
+#define _TD_COMMON_EP_H_
+
+#include "tcommon.h"
+#include "tmsg.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SCorEpSet {
+  int32_t version;
+  SEpSet  epSet;
+} SCorEpSet;
+
+typedef struct SBlockOrderInfo {
+  int32_t          order;
+  int32_t          colIndex;
+  SColumnInfoData* pColData;
+} SBlockOrderInfo;
+
+int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
+void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
+bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
+
+void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
+SEpSet getEpSet_s(SCorEpSet* pEpSet);
+
+#define NBIT                     (3u)
+#define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
+#define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])
+#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
+
+#define colDataSetNull_f(bm_, r_)                    \
+  do {                                               \
+    BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
+  } while (0)
+
+static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
+  if (!pColumnInfoData->hasNull) {
+    return false;
+  }
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    return pColumnInfoData->varmeta.offset[row] == -1;
+  } else {
+    if (pColumnInfoData->nullbitmap == NULL) {
+      return false;
+    }
+
+    return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
+  }
+}
+
+static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row,
+                                       SColumnDataAgg* pColAgg) {
+  if (!pColumnInfoData->hasNull) {
+    return false;
+  }
+
+  if (pColAgg != NULL) {
+    if (pColAgg->numOfNull == totalRows) {
+      ASSERT(pColumnInfoData->nullbitmap == NULL);
+      return true;
+    } else if (pColAgg->numOfNull == 0) {
+      ASSERT(pColumnInfoData->nullbitmap == NULL);
+      return false;
+    }
+  }
+
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    return pColumnInfoData->varmeta.offset[row] == -1;
+  } else {
+    if (pColumnInfoData->nullbitmap == NULL) {
+      return false;
+    }
+
+    return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
+  }
+}
+
+#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
+
+// SColumnInfoData, rowNumber
+#define colDataGetData(p1_, r_)                                                        \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
+                                        : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
+
+int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
+                        uint32_t numOfRow2);
+int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
+int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
+
+int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
+void    colDataTrim(SColumnInfoData* pColumnInfoData);
+
+size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
+size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
+
+int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
+int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
+                           int32_t pageSize);
+int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
+int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
+
+SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
+
+size_t blockDataGetSize(const SSDataBlock* pBlock);
+size_t blockDataGetRowSize(SSDataBlock* pBlock);
+double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
+size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
+
+SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
+
+int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
+int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
+
+int32_t      blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
+int32_t      blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+void         blockDataCleanup(SSDataBlock* pDataBlock);
+SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
+size_t       blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
+void*        blockDataDestroy(SSDataBlock* pBlock);
+
+void blockDebugShowData(const SArray* dataBlocks);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_EP_H_*/

include/common/tdataformat.h

@@ -12,6 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
 #ifndef _TD_COMMON_DATA_FORMAT_H_
 #define _TD_COMMON_DATA_FORMAT_H_
 
@@ -58,12 +59,15 @@ extern "C" {
   } while (0);
 
 // ----------------- TSDB COLUMN DEFINITION
+#pragma pack(push, 1)
 typedef struct {
-  int8_t   type;    // Column type
   col_id_t colId;        // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
-  int16_t  bytes;   // column bytes (restore to int16_t in case of misuse)
+  int32_t  type : 8;     // column type
-  uint16_t offset;  // point offset in STpRow after the header part.
+  int32_t  bytes : 24;   // column bytes (restore to int32_t in case of misuse)
+  int32_t  sma : 8;      // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
+  int32_t  offset : 24;  // point offset in STpRow after the header part.
 } STColumn;
+#pragma pack(pop)
 
 #define colType(col)   ((col)->type)
 #define colColId(col)  ((col)->colId)
@@ -77,9 +81,9 @@ typedef struct {
 
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
-  int version;    // version
+  int32_t version;    // version
-  int numOfCols;  // Number of columns appended
+  int32_t numOfCols;  // Number of columns appended
-  int tlen;       // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
+  int32_t tlen;   // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
                   // (bytes))
   uint16_t flen;  // First part length in a STpRow after the header part
   uint16_t vlen;  // pure value part length, excluded the overhead (bytes only)
@@ -92,13 +96,13 @@ typedef struct {
 #define schemaFLen(s)     ((s)->flen)
 #define schemaVLen(s)     ((s)->vlen)
 #define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s) tfree((s))
+#define tdFreeSchema(s)   taosMemoryFreeClear((s))
 
 STSchema *tdDupSchema(const STSchema *pSchema);
-int       tdEncodeSchema(void **buf, STSchema *pSchema);
+int32_t   tdEncodeSchema(void **buf, STSchema *pSchema);
 void     *tdDecodeSchema(void *buf, STSchema **pRSchema);
 
-static FORCE_INLINE int comparColId(const void *key1, const void *key2) {
+static FORCE_INLINE int32_t comparColId(const void *key1, const void *key2) {
   if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
     return 1;
   } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
@@ -116,12 +120,12 @@ static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
 
 // ----------------- SCHEMA BUILDER DEFINITION
 typedef struct {
-  int       tCols;
+  int32_t   tCols;
-  int       nCols;
+  int32_t   nCols;
-  int       tlen;
+  int32_t   tlen;
   uint16_t  flen;
   uint16_t  vlen;
-  int       version;
+  int32_t   version;
   STColumn *columns;
 } STSchemaBuilder;
 
@@ -132,10 +136,10 @@ typedef struct {
 #define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
 #define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
 
-int       tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
+int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
-int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition
@@ -194,7 +198,7 @@ static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
   return tdGetTKEY(lkey);
 }
 
-static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
+static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
   TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
   TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
 
@@ -243,7 +247,7 @@ void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
 // offset here not include dataRow header length
-static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
+static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
                                            int32_t offset) {
   assert(value != NULL);
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
@@ -268,7 +272,7 @@ static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool
 }
 
 // offset here not include dataRow header length
-static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
+static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
   return tdAppendDataColVal(row, value, true, type, offset);
 }
 
@@ -281,25 +285,25 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o
   }
 }
 
-static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int idx) {
+static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) {
   return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset);
 }
 
-static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int idx) {
+static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
   int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
   int8_t  type = pSchema->columns[idx].type;
 
   return tdGetRowDataOfCol(row, type, offset);
 }
 
-static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) {
+static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
   int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
   int8_t  type = pSchema->columns[idx].type;
 
   return isNull(tdGetRowDataOfCol(row, type, offset), type);
 }
 
-static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) {
+static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
   int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
   int8_t  type = pSchema->columns[idx].type;
   int16_t bytes = pSchema->columns[idx].bytes;
@@ -307,8 +311,8 @@ static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSche
   setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
 }
 
-static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src,
+static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src,
-                                                STSchema *pSrcSchema, int srcIdx) {
+                                                STSchema *pSrcSchema, int32_t srcIdx) {
   int8_t type = pDstSchema->columns[dstIdx].type;
   assert(type == pSrcSchema->columns[srcIdx].type);
   void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
@@ -364,10 +368,10 @@ typedef struct SDataCol {
   uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
   uint8_t         reserve : 7;
   int16_t         colId;      // column ID
-  int             bytes;      // column data bytes defined
+  int32_t         bytes;      // column data bytes defined
-  int             offset;     // data offset in a SDataRow (including the header size)
+  int32_t         offset;     // data offset in a SDataRow (including the header size)
-  int             spaceSize;  // Total space size for this column
+  int32_t         spaceSize;  // Total space size for this column
-  int             len;        // column data length
+  int32_t         len;        // column data length
   VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
   void           *pData;      // Actual data pointer
   void           *pBitmap;    // Bitmap pointer
@@ -378,17 +382,17 @@ typedef struct SDataCol {
 #define isAllRowsNone(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
-int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
+int32_t tdAllocMemForCol(SDataCol *pCol, int32_t maxPoints);
 
-void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
+void    dataColInit(SDataCol *pDataCol, STColumn *pCol, int32_t maxPoints);
-int  dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
+int32_t dataColAppendVal(SDataCol *pCol, const void *value, int32_t numOfRows, int32_t maxPoints);
-void *dataColSetOffset(SDataCol *pCol, int nEle);
+void   *dataColSetOffset(SDataCol *pCol, int32_t nEle);
 
-bool isNEleNull(SDataCol *pCol, int nEle);
+bool isNEleNull(SDataCol *pCol, int32_t nEle);
 
 #if 0
 // Get the data pointer from a column-wised data
-static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
+static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) {
   if (isAllRowsNull(pCol)) {
     return getNullValue(pCol->type);
   }
@@ -399,7 +403,7 @@ static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
   }
 }
 
-static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
+static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
   assert(rows > 0);
 
   if (IS_VAR_DATA_TYPE(pDataCol->type)) {
@@ -412,9 +416,9 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
 typedef struct {
   col_id_t  maxCols;    // max number of columns
   col_id_t  numOfCols;  // Total number of cols
-  int       maxPoints;  // max number of points
+  int32_t   maxPoints;  // max number of points
-  int       numOfRows;
+  int32_t   numOfRows;
-  int       sversion;  // TODO: set sversion
+  int32_t   sversion;  // TODO: set sversion
   SDataCol *cols;
 } SDataCols;
 
@@ -429,7 +433,7 @@ static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) {
   }
 }
 
-static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int row) {
+static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int32_t row) {
   assert(row < pCols->numOfRows);
   return dataColsKeyAt(pCols, row);
 }
@@ -458,12 +462,12 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
   }
 }
 
-SDataCols *tdNewDataCols(int maxCols, int maxRows);
+SDataCols *tdNewDataCols(int32_t maxCols, int32_t maxRows);
 void       tdResetDataCols(SDataCols *pCols);
-int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
+int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int        tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull);
+int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|
@@ -492,7 +496,7 @@ typedef struct {
 #define kvRowCpy(dst, r)       memcpy((dst), (r), kvRowLen(r))
 #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
 #define kvRowColIdxAt(r, i)    (kvRowColIdx(r) + (i))
-#define kvRowFree(r) tfree(r)
+#define kvRowFree(r)           taosMemoryFreeClear(r)
 #define kvRowEnd(r)            POINTER_SHIFT(r, kvRowLen(r))
 #define kvRowValLen(r)         (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r))
 #define kvRowTKey(r)           (*(TKEY *)(kvRowValues(r)))
@@ -501,12 +505,12 @@ typedef struct {
 #define kvRowDeleted(r)        TKEY_IS_DELETED(kvRowTKey(r))
 
 SKVRow  tdKVRowDup(SKVRow row);
-int    tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
+int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
-int    tdEncodeKVRow(void **buf, SKVRow row);
+int32_t tdEncodeKVRow(void **buf, SKVRow row);
 void   *tdDecodeKVRow(void *buf, SKVRow *row);
 void    tdSortKVRowByColIdx(SKVRow row);
 
-static FORCE_INLINE int comparTagId(const void *key1, const void *key2) {
+static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) {
   if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
     return 1;
   } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
@@ -528,7 +532,7 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
 
 #if 0
 // offset here not include kvRow header length
-static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
+static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
                                          int32_t offset) {
   assert(value != NULL);
   int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
@@ -584,15 +588,15 @@ typedef struct {
   void    *buf;
 } SKVRowBuilder;
 
-int    tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
+int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) {
+static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
-    SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
+    SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
     if (pColIdx == NULL) return -1;
     pBuilder->pColIdx = pColIdx;
   }
@@ -602,12 +606,12 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId,
 
   pBuilder->nCols++;
 
-  int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
+  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
   if (tlen > pBuilder->alloc - pBuilder->size) {
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;
     }
-    void *buf = realloc(pBuilder->buf, pBuilder->alloc);
+    void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc);
     if (buf == NULL) return -1;
     pBuilder->buf = buf;
   }
@@ -736,7 +740,7 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_
   }
 }
 
-static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
+static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
                                              int8_t type, int32_t offset) {
   if (isDataRow(row)) {
     tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);

include/common/tep.h

@@ -1,109 +0,0 @@
-#ifndef TDENGINE_TEP_H
-#define TDENGINE_TEP_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "os.h"
-#include "tmsg.h"
-#include "common.h"
-
-typedef struct SCorEpSet {
-  int32_t version;
-  SEpSet  epSet;
-} SCorEpSet;
-
-typedef struct SBlockOrderInfo {
-  int32_t          order;
-  int32_t          colIndex;
-  SColumnInfoData *pColData;
-//  int32_t          type;
-//  int32_t          bytes;
-//  bool             hasNull;
-} SBlockOrderInfo;
-
-int  taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
-void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
-
-bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
-
-void   updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
-SEpSet getEpSet_s(SCorEpSet *pEpSet);
-
-#define NBIT                     (3u)
-#define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
-#define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])
-#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
-
-#define colDataSetNull_f(bm_, r_)                    \
-  do {                                               \
-    BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
-  } while (0)
-
-static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) {
-  if (!pColumnInfoData->hasNull) {
-    return false;
-  }
-
-  if (pColAgg != NULL) {
-    if (pColAgg->numOfNull == totalRows) {
-      ASSERT(pColumnInfoData->nullbitmap == NULL);
-      return true;
-    } else if (pColAgg->numOfNull == 0) {
-      ASSERT(pColumnInfoData->nullbitmap == NULL);
-      return false;
-    }
-  }
-
-  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
-  } else {
-    if (pColumnInfoData->nullbitmap == NULL) {
-      return false;
-    }
-
-    return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
-  }
-}
-
-#define colDataGet(p1_, r_)                                                          \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
-                                        : (p1_)->pData + ((r_) * (p1_)->info.bytes));
-
-int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
-int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
-
-int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
-void colDataTrim(SColumnInfoData* pColumnInfoData);
-
-size_t colDataGetNumOfCols(const SSDataBlock* pBlock);
-size_t colDataGetNumOfRows(const SSDataBlock* pBlock);
-
-int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
-int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize);
-SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
-
-int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
-int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
-
-size_t blockDataGetSize(const SSDataBlock* pBlock);
-size_t blockDataGetRowSize(const SSDataBlock* pBlock);
-double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
-
-size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize);
-
-int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
-int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
-
-int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
-void    blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol);
-void   *blockDataDestroy(SSDataBlock *pBlock);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TEP_H

include/common/tglobal.h

@@ -16,57 +16,61 @@
 #ifndef _TD_COMMON_GLOBAL_H_
 #define _TD_COMMON_GLOBAL_H_
 
+#include "tarray.h"
+#include "tdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "tdef.h"
-#include "tcfg.h"
-
 // cluster
 extern char     tsFirst[];
 extern char     tsSecond[];
 extern char     tsLocalFqdn[];
 extern char     tsLocalEp[];
 extern uint16_t tsServerPort;
+extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
-extern int8_t   tsEnableTelemetryReporting;
+extern bool     tsEnableTelemetryReporting;
-extern int32_t  tsNumOfSupportVnodes;
 
 // common
-extern int      tsRpcTimer;
+extern int32_t tsRpcTimer;
-extern int      tsRpcMaxTime;
+extern int32_t tsRpcMaxTime;
-extern int      tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
+extern bool    tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
 extern int32_t tsMaxConnections;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
-extern uint32_t tsMaxTmrCtrl;
+extern int32_t tsMaxTmrCtrl;
 extern float   tsNumOfThreadsPerCore;
 extern int32_t tsNumOfCommitThreads;
 extern float   tsRatioOfQueryCores;
-extern int8_t   tsDaylight;
-extern int8_t   tsEnableCoreFile;
 extern int32_t tsCompressMsgSize;
 extern int32_t tsCompressColData;
 extern int32_t tsMaxNumOfDistinctResults;
-extern char     tsTempDir[];
+extern int32_t tsCompatibleModel;
-extern int      tsCompatibleModel;  // 2.0 compatible model
+extern bool    tsEnableSlaveQuery;
-extern int8_t   tsEnableSlaveQuery;
+extern bool    tsPrintAuth;
-extern int8_t   tsEnableAdjustMaster;
-extern int8_t   tsPrintAuth;
 extern int64_t tsTickPerDay[3];
+extern int32_t tsMultiProcess;
+
+// monitor
+extern bool     tsEnableMonitor;
+extern int32_t  tsMonitorInterval;
+extern char     tsMonitorFqdn[];
+extern uint16_t tsMonitorPort;
+extern int32_t  tsMonitorMaxLogs;
+extern bool     tsMonitorComp;
 
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
 extern int64_t tsQueryBufferSizeBytes;   // maximum allowed usage buffer size in byte for each data node
-extern int32_t tsRetrieveBlockingModel;  // retrieve threads will be blocked
+extern bool    tsRetrieveBlockingModel;  // retrieve threads will be blocked
-extern int8_t  tsKeepOriginalColumnName;
+extern bool    tsKeepOriginalColumnName;
-extern int8_t  tsDeadLockKillQuery;
+extern bool    tsDeadLockKillQuery;
 
 // client
 extern int32_t tsMaxWildCardsLen;
 extern int32_t tsMaxRegexStringLen;
-extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
@@ -77,24 +81,10 @@ extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the
 extern int32_t tsProjectExecInterval;
 extern int64_t tsMaxRetentWindow;
 
-// system info
-extern float    tsTotalLogDirGB;
-extern float    tsTotalTmpDirGB;
-extern float    tsTotalDataDirGB;
-extern float    tsAvailLogDirGB;
-extern float    tsAvailTmpDirectorySpace;
-extern float    tsAvailDataDirGB;
-extern float    tsUsedDataDirGB;
-extern float    tsMinimalLogDirGB;
-extern float    tsReservedTmpDirectorySpace;
-extern float    tsMinimalDataDirGB;
-extern uint32_t tsVersion;
-
 // build info
 extern char version[];
 extern char compatible_version[];
 extern char gitinfo[];
-extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
 // lossy
@@ -105,18 +95,19 @@ extern uint32_t tsMaxRange;
 extern uint32_t tsCurRange;
 extern char     tsCompressor[];
 
+// tfs
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-void    taosInitGlobalCfg();
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
-int32_t taosCheckAndPrintCfg();
+                      const char *apolloUrl, SArray *pArgs, bool tsc);
-int32_t taosCfgDynamicOptions(char *msg);
+int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc);
-bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+void    taosCleanupCfg();
-void    taosAddDataDir(int index, char *v1, int level, int primary);
+void    taosCfgDynamicOptions(const char *option, const char *value);
-void    taosReadDataDirCfg(char *v1, char *v2, char *v3);
+
-void    taosPrintDataDirCfg();
+struct SConfig *taosGetCfg();
 
 #ifdef __cplusplus
 }

include/common/tmsgcb.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_MSG_CB_H_
+#define _TD_COMMON_MSG_CB_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SRpcMsg      SRpcMsg;
+typedef struct SEpSet       SEpSet;
+typedef struct SMgmtWrapper SMgmtWrapper;
+typedef enum {
+  QUERY_QUEUE,
+  FETCH_QUEUE,
+  READ_QUEUE,
+  WRITE_QUEUE,
+  APPLY_QUEUE,
+  SYNC_QUEUE,
+  MERGE_QUEUE,
+  QUEUE_MAX,
+} EQueueType;
+
+typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
+typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
+typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq);
+typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
+typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp);
+
+typedef struct {
+  SMgmtWrapper*  pWrapper;
+  PutToQueueFp   queueFps[QUEUE_MAX];
+  GetQueueSizeFp qsizeFp;
+  SendReqFp      sendReqFp;
+  SendMnodeReqFp sendMnodeReqFp;
+  SendRspFp      sendRspFp;
+} SMsgCb;
+
+int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
+int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
+int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq);
+int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
+void    tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_MSG_CB_H_*/

include/common/tmsgdef.h

@@ -47,7 +47,7 @@ enum {
 #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
 #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
 
-int tMsgDict[] = {
+int32_t tMsgDict[] = {
 
 #elif defined(TD_MSG_SEG_CODE_)
 
@@ -127,26 +127,34 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_STB_META, "mnode-stb-meta", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "mnode-systable-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -179,19 +187,36 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "vnode-query-heartbeat", NULL, NULL)
 
   TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_PIPE_EXEC, "vnode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_MERGE_EXEC, "vnode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
 
+  TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
   // Requests handled by QNODE
   TD_NEW_MSG_SEG(TDMT_QND_MSG)
 
   // Requests handled by SNODE
   TD_NEW_MSG_SEG(TDMT_SND_MSG)
+  TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
+  TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_SND_TASK_PIPE_EXEC, "snode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_SND_TASK_MERGE_EXEC, "snode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+
+  // Requests handled by SCHEDULER
+  TD_NEW_MSG_SEG(TDMT_SCH_MSG)
+  TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL)
   
 #if defined(TD_MSG_NUMBER_)
   TDMT_MAX
 #endif
 };
-// clang-format on

include/common/tname.h

@@ -13,17 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TNAME_H
+#ifndef _TD_COMMON_NAME_H_
-#define TDENGINE_TNAME_H
+#define _TD_COMMON_NAME_H_
+
+#include "tdef.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-
-#include "tdef.h"
-#include "tmsg.h"
-
 #define TSDB_DB_NAME_T    1
 #define TSDB_TABLE_NAME_T 2
 
@@ -62,11 +60,11 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type);
 
 int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
-SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name);
+bool tNameDBNameEqual(SName* left, SName* right);
+
 
 #ifdef __cplusplus
 }
 #endif
 
-
+#endif  /*_TD_COMMON_NAME_H_*/
-#endif  // TDENGINE_TNAME_H

include/common/trequest.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TREQUEST_H_
+#ifndef _TD_COMMON_REQUEST_H_
-#define _TD_TREQUEST_H_
+#define _TD_COMMON_REQUEST_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,7 +36,7 @@ void            tdClearRBIter(SReqBatchIter *pIter);
 
 /* ------------------------ TYPES DEFINITION ------------------------ */
 struct SReqBatchIter {
-  int        iReq;
+  int32_t    iReq;
   SReqBatch *pReqBatch;
 };
 
@@ -44,4 +44,4 @@ struct SReqBatchIter {
 }
 #endif
 
-#endif /*_TD_TREQUEST_H_*/
+#endif /*_TD_COMMON_REQUEST_H_*/

include/common/trow.h

@@ -18,11 +18,11 @@
 
 #include "os.h"
 #include "talgo.h"
+#include "taosdef.h"
 #include "taoserror.h"
 #include "tbuffer.h"
-#include "tdef.h"
-#include "taosdef.h"
 #include "tdataformat.h"
+#include "tdef.h"
 #include "tschema.h"
 #include "ttypes.h"
 #include "tutil.h"
@@ -103,6 +103,7 @@ typedef struct {
 
 typedef struct {
   // TODO
+  int tmp;  // TODO: to avoid compile error
 } STpRow;   // tuple
 
 #pragma pack(push, 1)
@@ -118,6 +119,8 @@ typedef struct {
 } SKvRow;
 
 typedef struct {
+  /// timestamp
+  TSKEY ts;
   union {
     /// union field for encode and decode
     uint32_t info;
@@ -138,8 +141,6 @@ typedef struct {
   uint32_t len;
   /// row version
   uint64_t ver;
-  /// timestamp
-  TSKEY ts;
   /// the inline data, maybe a tuple or a k-v tuple
   char data[];
 } STSRow;
@@ -165,6 +166,7 @@ typedef struct {
 #define TD_ROW_HEAD_LEN  (sizeof(STSRow))
 #define TD_ROW_NCOLS_LEN (sizeof(col_id_t))
 
+#define TD_ROW_INFO(r)     ((r)->info)
 #define TD_ROW_TYPE(r)     ((r)->type)
 #define TD_ROW_DELETE(r)   ((r)->del)
 #define TD_ROW_ENDIAN(r)   ((r)->endian)
@@ -173,12 +175,13 @@ typedef struct {
 #define TD_ROW_DATA(r)     ((r)->data)
 #define TD_ROW_LEN(r)      ((r)->len)
 #define TD_ROW_KEY(r)      ((r)->ts)
-#define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16)
+#define TD_ROW_KEY_ADDR(r) (r)
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
-// (int)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
+// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
 #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
 
+#define TD_ROW_SET_INFO(r, i)  (TD_ROW_INFO(r) = (i))
 #define TD_ROW_SET_TYPE(r, t)  (TD_ROW_TYPE(r) = (t))
 #define TD_ROW_SET_DELETE(r)   (TD_ROW_DELETE(r) = 1)
 #define TD_ROW_SET_SVER(r, v)  (TD_ROW_SVER(r) = (v))
@@ -217,7 +220,7 @@ static FORCE_INLINE void    tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row,
 static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType);
 int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType);
 static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
-int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints);
+int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset);
 static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
@@ -355,7 +358,6 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
  *
  */
 
-
 /**
  * @brief
  *
@@ -432,7 +434,6 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t
       pBuilder->rowType = TD_ROW_TP;
     }
 
-
   } else {
     pBuilder->rowType = TD_ROW_TP;
   }
@@ -474,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
     return terrno;
   }
 
+  TD_ROW_SET_INFO(pBuilder->pBuf, 0);
   TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
 
   uint32_t len = 0;
@@ -669,8 +671,9 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
  * @param colIdx sorted column index, start from 0
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t colId, int8_t colType, TDRowValT valType,
+static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType,
-                                                const void *val, bool isCopyVarData, int32_t offset, int16_t colIdx) {
+                                                TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset,
+                                                col_id_t colIdx) {
   STSRow *pRow = pBuilder->pBuf;
   if (!val) {
 #ifdef TD_SUPPORT_BITMAP
@@ -698,9 +701,9 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t c
   }
   // TODO:  We can avoid the type judegement by FP, but would prevent the inline scheme.
   if (TD_IS_TP_ROW(pRow)) {
-    tdAppendColValToTpRow(pBuilder, valType, val, true, colType, colIdx, offset);
+    tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset);
   } else {
-    tdAppendColValToKvRow(pBuilder, valType, val, true, colType, colIdx, offset, colId);
+    tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -731,7 +734,7 @@ static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, v
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE int compareKvRowColId(const void *key1, const void *key2) {
+static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) {
   if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
     return 1;
   } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
@@ -793,8 +796,8 @@ static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema)
   pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId;
 }
 
-static int tdCompareColId(const void *arg1, const void *arg2) {
+static int32_t tdCompareColId(const void *arg1, const void *arg2) {
-  int       colId = *(int *)arg1;
+  int32_t   colId = *(int32_t *)arg1;
   STColumn *pCol = (STColumn *)arg2;
 
   if (colId < pCol->colId) {
@@ -959,7 +962,7 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
 STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2);
 
 // Get the data pointer from a column-wised data
-static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int row) {
+static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row) {
   if (isAllRowsNone(pCol)) {
     pVal->valType = TD_VTYPE_NULL;
 #ifdef TD_SUPPORT_READ2
@@ -969,10 +972,14 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in
 #endif
     return TSDB_CODE_SUCCESS;
   }
-  if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
+
+  if (TD_COL_ROWS_NORM(pCol)) {
+    pVal->valType = TD_VTYPE_NORM;
+  } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
     return terrno;
   }
-  if (TD_COL_ROWS_NORM(pCol) || tdValTypeIsNorm(pVal->valType)) {
+
+  if (tdValTypeIsNorm(pVal->valType)) {
     if (IS_VAR_DATA_TYPE(pCol->type)) {
       pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
     } else {
@@ -1000,7 +1007,8 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t
   return true;
 }
 
-static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) {
+static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx,
+                                        SCellVal *pVal) {
   if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
     tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
     return true;
@@ -1010,7 +1018,7 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o
   return true;
 }
 
-static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
+static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
   ASSERT(rows > 0);
   int32_t result = 0;
 
@@ -1082,12 +1090,12 @@ typedef struct {
 void    trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver);
 void    trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver);
 void    trbSetRowTS(STSRowBuilder *pRB, TSKEY ts);
-int  trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid);
+int32_t trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid);
 
 // STSRowReader
 #define tRowReaderInit(schema, row) \
   { .schema = (schema), .row = (row) }
-int tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size);
+int32_t tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size);
 
 // STSRowBatchIter
 #define tRowBatchIterInit(pRB) \

include/common/tschema.h

@@ -70,7 +70,7 @@ typedef struct {
   { .size = (capacity), .pSchema = (target) }
 void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
 void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
-int  tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
+int32_t  tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
 
 #endif
 

include/common/ttime.h

@@ -16,15 +16,26 @@
 #ifndef _TD_COMMON_TIME_H_
 #define _TD_COMMON_TIME_H_
 
+#include "taosdef.h"
+#include "tmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "taosdef.h"
-#include "tmsg.h"
-
 #define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y')
 
+#define TIME_UNIT_NANOSECOND  'b'
+#define TIME_UNIT_MICROSECOND 'u'
+#define TIME_UNIT_MILLISECOND 'a'
+#define TIME_UNIT_SECOND      's'
+#define TIME_UNIT_MINUTE      'm'
+#define TIME_UNIT_HOUR        'h'
+#define TIME_UNIT_DAY         'd'
+#define TIME_UNIT_WEEK        'w'
+#define TIME_UNIT_MONTH       'n'
+#define TIME_UNIT_YEAR        'y'
+
 /*
  * @return timestamp decided by global conf variable, tsTimePrecision
  * if precision == TSDB_TIME_PRECISION_MICRO, it returns timestamp in microsecond.
@@ -52,6 +63,8 @@ void    deltaToUtcInitOnce();
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
 
+void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
+
 #ifdef __cplusplus
 }
 #endif

include/common/ttokendef.h

@@ -13,284 +13,190 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TTOKENDEF_H
+#ifndef _TD_COMMON_TOKEN_H_
-#define TDENGINE_TTOKENDEF_H
+#define _TD_COMMON_TOKEN_H_
 
-#define TK_ID                               1
+#define TK_OR                               1
-#define TK_BOOL                             2
+#define TK_AND                              2
-#define TK_INTEGER                          3
+#define TK_UNION                            3
-#define TK_FLOAT                            4
+#define TK_ALL                              4
-#define TK_STRING                           5
+#define TK_MINUS                            5
-#define TK_TIMESTAMP                        6
+#define TK_EXCEPT                           6
-#define TK_OR                               7
+#define TK_INTERSECT                        7
-#define TK_AND                              8
+#define TK_NK_BITAND                        8
-#define TK_NOT                              9
+#define TK_NK_BITOR                         9
-#define TK_EQ                              10
+#define TK_NK_LSHIFT                       10
-#define TK_NE                              11
+#define TK_NK_RSHIFT                       11
-#define TK_ISNULL                          12
+#define TK_NK_PLUS                         12
-#define TK_NOTNULL                         13
+#define TK_NK_MINUS                        13
-#define TK_IS                              14
+#define TK_NK_STAR                         14
-#define TK_LIKE                            15
+#define TK_NK_SLASH                        15
-#define TK_MATCH                           16
+#define TK_NK_REM                          16
-#define TK_NMATCH                          17
+#define TK_NK_CONCAT                       17
-#define TK_GLOB                            18
+#define TK_CREATE                          18
-#define TK_BETWEEN                         19
+#define TK_ACCOUNT                         19
-#define TK_IN                              20
+#define TK_NK_ID                           20
-#define TK_GT                              21
+#define TK_PASS                            21
-#define TK_GE                              22
+#define TK_NK_STRING                       22
-#define TK_LT                              23
+#define TK_ALTER                           23
-#define TK_LE                              24
+#define TK_PPS                             24
-#define TK_BITAND                          25
+#define TK_TSERIES                         25
-#define TK_BITOR                           26
+#define TK_STORAGE                         26
-#define TK_LSHIFT                          27
+#define TK_STREAMS                         27
-#define TK_RSHIFT                          28
+#define TK_QTIME                           28
-#define TK_PLUS                            29
+#define TK_DBS                             29
-#define TK_MINUS                           30
+#define TK_USERS                           30
-#define TK_DIVIDE                          31
+#define TK_CONNS                           31
-#define TK_TIMES                           32
+#define TK_STATE                           32
-#define TK_STAR                            33
+#define TK_USER                            33
-#define TK_SLASH                           34
+#define TK_PRIVILEGE                       34
-#define TK_REM                             35
+#define TK_DROP                            35
-#define TK_CONCAT                          36
+#define TK_DNODE                           36
-#define TK_UMINUS                          37
+#define TK_PORT                            37
-#define TK_UPLUS                           38
+#define TK_NK_INTEGER                      38
-#define TK_BITNOT                          39
+#define TK_DNODES                          39
-#define TK_SHOW                            40
+#define TK_NK_IPTOKEN                      40
-#define TK_DATABASES                       41
+#define TK_LOCAL                           41
-#define TK_TOPICS                          42
+#define TK_QNODE                           42
-#define TK_FUNCTIONS                       43
+#define TK_ON                              43
-#define TK_MNODES                          44
+#define TK_DATABASE                        44
-#define TK_DNODES                          45
+#define TK_USE                             45
-#define TK_ACCOUNTS                        46
+#define TK_IF                              46
-#define TK_USERS                           47
+#define TK_NOT                             47
-#define TK_MODULES                         48
+#define TK_EXISTS                          48
-#define TK_QUERIES                         49
+#define TK_BLOCKS                          49
-#define TK_CONNECTIONS                     50
+#define TK_CACHE                           50
-#define TK_STREAMS                         51
+#define TK_CACHELAST                       51
-#define TK_VARIABLES                       52
+#define TK_COMP                            52
-#define TK_SCORES                          53
+#define TK_DAYS                            53
-#define TK_GRANTS                          54
+#define TK_FSYNC                           54
-#define TK_VNODES                          55
+#define TK_MAXROWS                         55
-#define TK_DOT                             56
+#define TK_MINROWS                         56
-#define TK_CREATE                          57
+#define TK_KEEP                            57
-#define TK_TABLE                           58
+#define TK_PRECISION                       58
-#define TK_STABLE                          59
+#define TK_QUORUM                          59
-#define TK_DATABASE                        60
+#define TK_REPLICA                         60
-#define TK_TABLES                          61
+#define TK_TTL                             61
-#define TK_STABLES                         62
+#define TK_WAL                             62
 #define TK_VGROUPS                         63
-#define TK_DROP                            64
+#define TK_SINGLE_STABLE                   64
-#define TK_TOPIC                           65
+#define TK_STREAM_MODE                     65
-#define TK_FUNCTION                        66
+#define TK_RETENTIONS                      66
-#define TK_DNODE                           67
+#define TK_FILE_FACTOR                     67
-#define TK_USER                            68
+#define TK_NK_FLOAT                        68
-#define TK_ACCOUNT                         69
+#define TK_TABLE                           69
-#define TK_USE                             70
+#define TK_NK_LP                           70
-#define TK_DESCRIBE                        71
+#define TK_NK_RP                           71
-#define TK_DESC                            72
+#define TK_STABLE                          72
-#define TK_ALTER                           73
+#define TK_ADD                             73
-#define TK_PASS                            74
+#define TK_COLUMN                          74
-#define TK_PRIVILEGE                       75
+#define TK_MODIFY                          75
-#define TK_LOCAL                           76
+#define TK_RENAME                          76
-#define TK_COMPACT                         77
+#define TK_TAG                             77
-#define TK_LP                              78
+#define TK_SET                             78
-#define TK_RP                              79
+#define TK_NK_EQ                           79
-#define TK_IF                              80
+#define TK_USING                           80
-#define TK_EXISTS                          81
+#define TK_TAGS                            81
-#define TK_PORT                            82
+#define TK_NK_DOT                          82
-#define TK_IPTOKEN                         83
+#define TK_NK_COMMA                        83
-#define TK_AS                              84
+#define TK_COMMENT                         84
-#define TK_OUTPUTTYPE                      85
+#define TK_BOOL                            85
-#define TK_AGGREGATE                       86
+#define TK_TINYINT                         86
-#define TK_BUFSIZE                         87
+#define TK_SMALLINT                        87
-#define TK_PPS                             88
+#define TK_INT                             88
-#define TK_TSERIES                         89
+#define TK_INTEGER                         89
-#define TK_DBS                             90
+#define TK_BIGINT                          90
-#define TK_STORAGE                         91
+#define TK_FLOAT                           91
-#define TK_QTIME                           92
+#define TK_DOUBLE                          92
-#define TK_CONNS                           93
+#define TK_BINARY                          93
-#define TK_STATE                           94
+#define TK_TIMESTAMP                       94
-#define TK_COMMA                           95
+#define TK_NCHAR                           95
-#define TK_KEEP                            96
+#define TK_UNSIGNED                        96
-#define TK_CACHE                           97
+#define TK_JSON                            97
-#define TK_REPLICA                         98
+#define TK_VARCHAR                         98
-#define TK_QUORUM                          99
+#define TK_MEDIUMBLOB                      99
-#define TK_DAYS                           100
+#define TK_BLOB                           100
-#define TK_MINROWS                        101
+#define TK_VARBINARY                      101
-#define TK_MAXROWS                        102
+#define TK_DECIMAL                        102
-#define TK_BLOCKS                         103
+#define TK_SMA                            103
-#define TK_CTIME                          104
+#define TK_ROLLUP                         104
-#define TK_WAL                            105
+#define TK_SHOW                           105
-#define TK_FSYNC                          106
+#define TK_DATABASES                      106
-#define TK_COMP                           107
+#define TK_TABLES                         107
-#define TK_PRECISION                      108
+#define TK_STABLES                        108
-#define TK_UPDATE                         109
+#define TK_MNODES                         109
-#define TK_CACHELAST                      110
+#define TK_MODULES                        110
-#define TK_STREAM                         111
+#define TK_QNODES                         111
-#define TK_MODE                           112
+#define TK_FUNCTIONS                      112
-#define TK_UNSIGNED                       113
+#define TK_INDEXES                        113
-#define TK_TAGS                           114
+#define TK_FROM                           114
-#define TK_USING                          115
+#define TK_LIKE                           115
-#define TK_NULL                           116
+#define TK_INDEX                          116
-#define TK_NOW                            117
+#define TK_FULLTEXT                       117
-#define TK_SELECT                         118
+#define TK_FUNCTION                       118
-#define TK_UNION                          119
+#define TK_INTERVAL                       119
-#define TK_ALL                            120
+#define TK_TOPIC                          120
-#define TK_DISTINCT                       121
+#define TK_AS                             121
-#define TK_FROM                           122
+#define TK_NK_BOOL                        122
-#define TK_VARIABLE                       123
+#define TK_NK_VARIABLE                    123
-#define TK_INTERVAL                       124
+#define TK_BETWEEN                        124
-#define TK_EVERY                          125
+#define TK_IS                             125
-#define TK_SESSION                        126
+#define TK_NULL                           126
-#define TK_STATE_WINDOW                   127
+#define TK_NK_LT                          127
-#define TK_FILL                           128
+#define TK_NK_GT                          128
-#define TK_SLIDING                        129
+#define TK_NK_LE                          129
-#define TK_ORDER                          130
+#define TK_NK_GE                          130
-#define TK_BY                             131
+#define TK_NK_NE                          131
-#define TK_ASC                            132
+#define TK_MATCH                          132
-#define TK_GROUP                          133
+#define TK_NMATCH                         133
-#define TK_HAVING                         134
+#define TK_IN                             134
-#define TK_LIMIT                          135
+#define TK_JOIN                           135
-#define TK_OFFSET                         136
+#define TK_INNER                          136
-#define TK_SLIMIT                         137
+#define TK_SELECT                         137
-#define TK_SOFFSET                        138
+#define TK_DISTINCT                       138
 #define TK_WHERE                          139
-#define TK_RESET                          140
+#define TK_PARTITION                      140
-#define TK_QUERY                          141
+#define TK_BY                             141
-#define TK_SYNCDB                         142
+#define TK_SESSION                        142
-#define TK_ADD                            143
+#define TK_STATE_WINDOW                   143
-#define TK_COLUMN                         144
+#define TK_SLIDING                        144
-#define TK_MODIFY                         145
+#define TK_FILL                           145
-#define TK_TAG                            146
+#define TK_VALUE                          146
-#define TK_CHANGE                         147
+#define TK_NONE                           147
-#define TK_SET                            148
+#define TK_PREV                           148
-#define TK_KILL                           149
+#define TK_LINEAR                         149
-#define TK_CONNECTION                     150
+#define TK_NEXT                           150
-#define TK_COLON                          151
+#define TK_GROUP                          151
-#define TK_ABORT                          152
+#define TK_HAVING                         152
-#define TK_AFTER                          153
+#define TK_ORDER                          153
-#define TK_ATTACH                         154
+#define TK_SLIMIT                         154
-#define TK_BEFORE                         155
+#define TK_SOFFSET                        155
-#define TK_BEGIN                          156
+#define TK_LIMIT                          156
-#define TK_CASCADE                        157
+#define TK_OFFSET                         157
-#define TK_CLUSTER                        158
+#define TK_ASC                            158
-#define TK_CONFLICT                       159
+#define TK_DESC                           159
-#define TK_COPY                           160
+#define TK_NULLS                          160
-#define TK_DEFERRED                       161
+#define TK_FIRST                          161
-#define TK_DELIMITERS                     162
+#define TK_LAST                           162
-#define TK_DETACH                         163
-#define TK_EACH                           164
-#define TK_END                            165
-#define TK_EXPLAIN                        166
-#define TK_FAIL                           167
-#define TK_FOR                            168
-#define TK_IGNORE                         169
-#define TK_IMMEDIATE                      170
-#define TK_INITIALLY                      171
-#define TK_INSTEAD                        172
-#define TK_KEY                            173
-#define TK_OF                             174
-#define TK_RAISE                          175
-#define TK_REPLACE                        176
-#define TK_RESTRICT                       177
-#define TK_ROW                            178
-#define TK_STATEMENT                      179
-#define TK_TRIGGER                        180
-#define TK_VIEW                           181
-#define TK_SEMI                           182
-#define TK_NONE                           183
-#define TK_PREV                           184
-#define TK_LINEAR                         185
-#define TK_IMPORT                         186
-#define TK_TBNAME                         187
-#define TK_JOIN                           188
-#define TK_INSERT                         189
-#define TK_INTO                           190
-#define TK_VALUES                         191
 
-#define NEW_TK_OR                               1
+#define TK_NK_SPACE                       300
-#define NEW_TK_AND                              2
+#define TK_NK_COMMENT                     301
-#define NEW_TK_UNION                            3
+#define TK_NK_ILLEGAL                     302
-#define NEW_TK_ALL                              4
+#define TK_NK_HEX                         303   // hex number  0x123
-#define NEW_TK_MINUS                            5
+#define TK_NK_OCT                         304   // oct number
-#define NEW_TK_EXCEPT                           6
+#define TK_NK_BIN                         305   // bin format data 0b111
-#define NEW_TK_INTERSECT                        7
+#define TK_NK_FILE                        306
-#define NEW_TK_NK_PLUS                          8
+#define TK_NK_QUESTION                    307   // denoting the placeholder of "?",when invoking statement bind query
-#define NEW_TK_NK_MINUS                         9
-#define NEW_TK_NK_STAR                         10
-#define NEW_TK_NK_SLASH                        11
-#define NEW_TK_NK_REM                          12
-#define NEW_TK_SHOW                            13
-#define NEW_TK_DATABASES                       14
-#define NEW_TK_NK_INTEGER                      15
-#define NEW_TK_NK_FLOAT                        16
-#define NEW_TK_NK_STRING                       17
-#define NEW_TK_NK_BOOL                         18
-#define NEW_TK_TIMESTAMP                       19
-#define NEW_TK_NK_VARIABLE                     20
-#define NEW_TK_NK_COMMA                        21
-#define NEW_TK_NK_ID                           22
-#define NEW_TK_NK_LP                           23
-#define NEW_TK_NK_RP                           24
-#define NEW_TK_NK_DOT                          25
-#define NEW_TK_BETWEEN                         26
-#define NEW_TK_NOT                             27
-#define NEW_TK_IS                              28
-#define NEW_TK_NULL                            29
-#define NEW_TK_NK_LT                           30
-#define NEW_TK_NK_GT                           31
-#define NEW_TK_NK_LE                           32
-#define NEW_TK_NK_GE                           33
-#define NEW_TK_NK_NE                           34
-#define NEW_TK_NK_EQ                           35
-#define NEW_TK_LIKE                            36
-#define NEW_TK_MATCH                           37
-#define NEW_TK_NMATCH                          38
-#define NEW_TK_IN                              39
-#define NEW_TK_FROM                            40
-#define NEW_TK_AS                              41
-#define NEW_TK_JOIN                            42
-#define NEW_TK_ON                              43
-#define NEW_TK_INNER                           44
-#define NEW_TK_SELECT                          45
-#define NEW_TK_DISTINCT                        46
-#define NEW_TK_WHERE                           47
-#define NEW_TK_PARTITION                       48
-#define NEW_TK_BY                              49
-#define NEW_TK_SESSION                         50
-#define NEW_TK_STATE_WINDOW                    51
-#define NEW_TK_INTERVAL                        52
-#define NEW_TK_SLIDING                         53
-#define NEW_TK_FILL                            54
-#define NEW_TK_VALUE                           55
-#define NEW_TK_NONE                            56
-#define NEW_TK_PREV                            57
-#define NEW_TK_LINEAR                          58
-#define NEW_TK_NEXT                            59
-#define NEW_TK_GROUP                           60
-#define NEW_TK_HAVING                          61
-#define NEW_TK_ORDER                           62
-#define NEW_TK_SLIMIT                          63
-#define NEW_TK_SOFFSET                         64
-#define NEW_TK_LIMIT                           65
-#define NEW_TK_OFFSET                          66
-#define NEW_TK_ASC                             67
-#define NEW_TK_DESC                            68
-#define NEW_TK_NULLS                           69
-#define NEW_TK_FIRST                           70
-#define NEW_TK_LAST                            71
 
-#define TK_SPACE                          300
+#define TK_NK_COLON                       500
-#define TK_COMMENT                        301
+#define TK_NK_BITNOT                      501
-#define TK_ILLEGAL                        302
+#define TK_INSERT                         502
-#define TK_HEX                            303   // hex number  0x123
+#define TK_INTO                           503
-#define TK_OCT                            304   // oct number
+#define TK_NOW                            504
-#define TK_BIN                            305   // bin format data 0b111
+#define TK_VALUES                         507
-#define TK_FILE                           306
+#define TK_IMPORT                         507
-#define TK_QUESTION                       307   // denoting the placeholder of "?",when invoking statement bind query
+#define TK_NK_SEMI                        508
-
-#define TK_NIL                            65535
-
-#endif
 
+#define TK_NK_NIL                         65535
 
+#endif /*_TD_COMMON_TOKEN_H_*/

include/common/ttszip.h

@@ -13,17 +13,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TTSZIP_H
+#ifndef _TD_COMMON_TTSZIP_H_
-#define TDENGINE_TTSZIP_H
+#define _TD_COMMON_TTSZIP_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
 
 #include "os.h"
 #include "tdef.h"
 #include "tvariant.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define MEM_BUF_SIZE           (1 << 20)
 #define TS_COMP_FILE_MAGIC     0x87F5EC4C
 #define TS_COMP_FILE_GROUP_MAX 512
@@ -73,7 +73,7 @@ typedef struct STSGroupBlockInfoEx {
 } STSGroupBlockInfoEx;
 
 typedef struct STSBuf {
-  FILE*    f;
+  TdFilePtr pFile;
   char      path[PATH_MAX];
   uint32_t  fileSize;
 
@@ -144,4 +144,4 @@ bool tsBufIsValidElem(STSElem* pElem);
 }
 #endif
 
-#endif  // TDENGINE_TTSZIP_H
+#endif  /*_TD_COMMON_TTSZIP_H_*/

include/common/ttypes.h

@@ -1,5 +1,20 @@
-#ifndef TDENGINE_TTYPE_H
+/*
-#define TDENGINE_TTYPE_H
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_TTYPE_H_
+#define _TD_COMMON_TTYPE_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -12,13 +27,9 @@ extern "C" {
 typedef int32_t  VarDataOffsetT;
 typedef uint32_t TDRowLenT;
 typedef uint8_t  TDRowValT;
-typedef uint16_t col_id_t;
+typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
-
+typedef int32_t  col_bytes_t;
-typedef struct tstr {
-  VarDataLenT len;
-  char        data[];
-} tstr;
 
 #pragma pack(push, 1)
 typedef struct {
@@ -36,12 +47,11 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
 
-
 // this data type is internally used only in 'in' query to hold the values
 #define TSDB_DATA_TYPE_POINTER_ARRAY (1000)
 #define TSDB_DATA_TYPE_VALUE_ARRAY   (1001)
@@ -91,6 +101,8 @@ typedef struct {
   do {                                         \
     switch (_type) {                           \
       case TSDB_DATA_TYPE_BOOL:                \
+        *(bool *)(_v) = (bool)(_data);         \
+        break;                                 \
       case TSDB_DATA_TYPE_TINYINT:             \
         *(int8_t *)(_v) = (int8_t)(_data);     \
         break;                                 \
@@ -104,6 +116,7 @@ typedef struct {
         *(uint16_t *)(_v) = (uint16_t)(_data); \
         break;                                 \
       case TSDB_DATA_TYPE_BIGINT:              \
+      case TSDB_DATA_TYPE_TIMESTAMP:           \
         *(int64_t *)(_v) = (int64_t)(_data);   \
         break;                                 \
       case TSDB_DATA_TYPE_UBIGINT:             \
@@ -143,9 +156,11 @@ typedef struct {
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
-#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
+#define IS_CONVERT_AS_SIGNED(_t) \
+  (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
 #define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL))
 
+// TODO remove this function
 static FORCE_INLINE bool isNull(const void *val, int32_t type) {
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -188,12 +203,12 @@ typedef struct tDataTypeDescriptor {
   char   *name;
   int64_t minValue;
   int64_t maxValue;
-  int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+  int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output,
-                  char algorithm, char *const buffer, int bufferSize);
+                      int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
-  int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
+  int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
-                    int outputSize, char algorithm, char *const buffer, int bufferSize);
+                        int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
-  void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum,
+  void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
-                        int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
+                     int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypes[15];
@@ -211,7 +226,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
 void *getDataMin(int32_t type);
 void *getDataMax(int32_t type);
 
-
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
 #define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
@@ -219,4 +233,4 @@ void* getDataMax(int32_t type);
 }
 #endif
 
-#endif  // TDENGINE_TTYPE_H
+#endif  /*_TD_COMMON_TTYPE_H_*/

include/common/tvariant.h

@@ -13,15 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TVARIANT_H
+#ifndef _TD_COMMON_VARIANT_H_
-#define TDENGINE_TVARIANT_H
+#define _TD_COMMON_VARIANT_H_
+
+#include "tarray.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "tarray.h"
-
 // variant, each number/string/field_id has a corresponding struct during parsing sql
 typedef struct SVariant {
   uint32_t nType;
@@ -31,7 +31,7 @@ typedef struct SVariant {
     uint64_t u;
     double   d;
     char    *pz;
-    wchar_t *wpz;
+    TdUcs4  *ucs4;
     SArray  *arr;  // only for 'in' query to hold value list, not value for a field
   };
 } SVariant;
@@ -64,4 +64,4 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type);
 }
 #endif
 
-#endif  // TDENGINE_TVARIANT_H
+#endif  /*_TD_COMMON_VARIANT_H_*/

include/dnode/bnode/bnode.h

@@ -16,29 +16,20 @@
 #ifndef _TD_BNODE_H_
 #define _TD_BNODE_H_
 
+#include "tmsgcb.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
 typedef struct SBnode SBnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
 } SBnodeLoad;
 
 typedef struct {
-  int32_t           sver;
+  SMsgCb msgCb;
-  int32_t           dnodeId;
-  int64_t           clusterId;
-  SDnode           *pDnode;
-  SendReqToDnodeFp  sendReqToDnodeFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
-  SendRedirectRspFp sendRedirectRspFp;
 } SBnodeOpt;
 
 /* ------------------------ SBnode ------------------------ */
@@ -76,13 +67,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad);
  */
 int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs);
 
-/**
- * @brief Drop a bnode.
- *
- * @param path Path of the bnode.
- */
-void bndDestroy(const char *path);
-
 #ifdef __cplusplus
 }
 #endif

include/dnode/mgmt/dnode.h

@@ -25,55 +25,40 @@ extern "C" {
 /* ------------------------ TYPES EXPOSED ---------------- */
 typedef struct SDnode SDnode;
 
-/* ------------------------ Environment ------------------ */
-typedef struct {
-  int32_t sver;
-  int32_t numOfCores;
-  int16_t numOfCommitThreads;
-  int8_t  enableTelem;
-  char    timezone[TSDB_TIMEZONE_LEN];
-  char    locale[TSDB_LOCALE_LEN];
-  char    charset[TSDB_LOCALE_LEN];
-  char    buildinfo[64];
-  char    gitinfo[48];
-} SDnodeEnvCfg;
-
 /**
  * @brief Initialize the environment
  *
- * @param pOption Option of the environment
  * @return int32_t 0 for success and -1 for failure
  */
-int32_t dndInit(const SDnodeEnvCfg *pCfg);
+int32_t dndInit();
 
 /**
- * @brief clear the environment
+ * @brief Clear the environment
- *
  */
 void dndCleanup();
 
 /* ------------------------ SDnode ----------------------- */
 typedef struct {
   int32_t   numOfSupportVnodes;
-  int32_t  statusInterval;
-  float    numOfThreadsPerCore;
-  float    ratioOfQueryCores;
-  int32_t  maxShellConns;
-  int32_t  shellActivityTimer;
   uint16_t  serverPort;
-  char     dataDir[TSDB_FILENAME_LEN];
+  char      dataDir[PATH_MAX];
   char      localEp[TSDB_EP_LEN];
   char      localFqdn[TSDB_FQDN_LEN];
   char      firstEp[TSDB_EP_LEN];
-} SDnodeObjCfg;
+  char      secondEp[TSDB_EP_LEN];
+  SDiskCfg *pDisks;
+  int32_t   numOfDisks;
+} SDnodeOpt;
+
+typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent;
 
 /**
  * @brief Initialize and start the dnode.
  *
- * @param pCfg Config of the dnode.
+ * @param pOption Option of the dnode.
  * @return SDnode* The dnode object.
  */
-SDnode *dndCreate(SDnodeObjCfg *pCfg);
+SDnode *dndCreate(const SDnodeOpt *pOption);
 
 /**
  * @brief Stop and cleanup the dnode.
@@ -82,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg);
  */
 void dndClose(SDnode *pDnode);
 
+/**
+ * @brief Run dnode until specific event is receive.
+ *
+ * @param pDnode The dnode object to run.
+ */
+int32_t dndRun(SDnode *pDnode);
+
+/**
+ * @brief Handle event in the dnode.
+ *
+ * @param pDnode The dnode object to close.
+ * @param event The event to handle.
+ */
+void dndHandleEvent(SDnode *pDnode, EDndEvent event);
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/mnode/mnode.h

@@ -16,45 +16,17 @@
 #ifndef _TD_MND_H_
 #define _TD_MND_H_
 
+#include "monitor.h"
+#include "tmsg.h"
+#include "tmsgcb.h"
+#include "trpc.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode    SDnode;
 typedef struct SMnode SMnode;
-typedef struct SMnodeMsg SMnodeMsg;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-
-typedef struct SMnodeLoad {
-  int64_t numOfDnode;
-  int64_t numOfMnode;
-  int64_t numOfVgroup;
-  int64_t numOfDatabase;
-  int64_t numOfSuperTable;
-  int64_t numOfChildTable;
-  int64_t numOfNormalTable;
-  int64_t numOfColumn;
-  int64_t totalPoints;
-  int64_t totalStorage;
-  int64_t compStorage;
-} SMnodeLoad;
-
-typedef struct SMnodeCfg {
-  int32_t sver;
-  int8_t  enableTelem;
-  int32_t statusInterval;
-  int32_t shellActivityTimer;
-  char   *timezone;
-  char   *locale;
-  char   *charset;
-  char   *buildinfo;
-  char   *gitinfo;
-} SMnodeCfg;
 
 typedef struct {
   int32_t  dnodeId;
@@ -62,13 +34,7 @@ typedef struct {
   int8_t   replica;
   int8_t   selfIndex;
   SReplica replicas[TSDB_MAX_REPLICA];
-  SMnodeCfg         cfg;
+  SMsgCb   msgCb;
-  SDnode           *pDnode;
-  PutReqToMWriteQFp putReqToMWriteQFp;
-  PutReqToMReadQFp  putReqToMReadQFp;
-  SendReqToDnodeFp  sendReqToDnodeFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
-  SendRedirectRspFp sendRedirectRspFp;
 } SMnodeOpt;
 
 /* ------------------------ SMnode ------------------------ */
@@ -98,20 +64,23 @@ void mndClose(SMnode *pMnode);
 int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption);
 
 /**
- * @brief Drop a mnode.
+ * @brief Start mnode
- *
- * @param path Path of the mnode.
- */
-void mndDestroy(const char *path);
-
-/**
- * @brief Get mnode statistics info.
  *
  * @param pMnode The mnode object.
- * @param pLoad Statistics of the mnode.
+ */
+int32_t mndStart(SMnode *pMnode);
+
+/**
+ * @brief Get mnode monitor info.
+ *
+ * @param pMnode The mnode object.
+ * @param pClusterInfo
+ * @param pVgroupInfo
+ * @param pGrantInfo
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
+int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                          SMonGrantInfo *pGrantInfo);
 
 /**
  * @brief Get user authentication info.
@@ -126,37 +95,13 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
  */
 int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
 
-/**
- * @brief Initialize mnode msg.
- *
- * @param pMnode The mnode object.
- * @param pMsg The request rpc msg.
- * @return int32_t The created mnode msg.
- */
-SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- */
-void mndCleanupMsg(SMnodeMsg *pMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- * @param code The error code.
- */
-void mndSendRsp(SMnodeMsg *pMsg, int32_t code);
-
 /**
  * @brief Process the read, write, sync request.
  *
  * @param pMsg The request msg.
  * @return int32_t 0 for success, -1 for failure.
  */
-void mndProcessMsg(SMnodeMsg *pMsg);
+int32_t mndProcessMsg(SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

include/dnode/mnode/sdb/sdb.h

@@ -113,14 +113,17 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_SUBSCRIBE = 10,
+  SDB_STREAM = 10,
-  SDB_CONSUMER = 11,
+  SDB_OFFSET = 11,
-  SDB_TOPIC = 12,
+  SDB_SUBSCRIBE = 12,
-  SDB_VGROUP = 13,
+  SDB_CONSUMER = 13,
-  SDB_STB = 14,
+  SDB_TOPIC = 14,
-  SDB_DB = 15,
+  SDB_VGROUP = 15,
-  SDB_FUNC = 16,
+  SDB_SMA = 16,
-  SDB_MAX = 17
+  SDB_STB = 17,
+  SDB_DB = 18,
+  SDB_FUNC = 19,
+  SDB_MAX = 20
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/dnode/qnode/qnode.h

@@ -16,16 +16,14 @@
 #ifndef _TD_QNODE_H_
 #define _TD_QNODE_H_
 
+#include "tmsgcb.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
 typedef struct SQnode SQnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
   int64_t numOfStartTask;
@@ -39,13 +37,7 @@ typedef struct {
 } SQnodeLoad;
 
 typedef struct {
-  int32_t           sver;
+  SMsgCb msgCb;
-  int32_t           dnodeId;
-  int64_t           clusterId;
-  SDnode           *pDnode;
-  SendReqToDnodeFp  sendReqToDnodeFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
-  SendRedirectRspFp sendRedirectRspFp;
 } SQnodeOpt;
 
 /* ------------------------ SQnode ------------------------ */
@@ -78,10 +70,9 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad);
  *
  * @param pQnode The qnode object.
  * @param pMsg The request message
- * @param pRsp The response message
- * @return int32_t 0 for success, -1 for failure
  */
-int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
+int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg);
+int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

include/dnode/snode/snode.h

@@ -16,29 +16,23 @@
 #ifndef _TD_SNODE_H_
 #define _TD_SNODE_H_
 
+#include "tmsgcb.h"
+#include "tmsg.h"
+#include "trpc.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
 typedef struct SSnode SSnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
+  int32_t reserved;
 } SSnodeLoad;
 
 typedef struct {
-  int32_t           sver;
+  SMsgCb msgCb;
-  int32_t           dnodeId;
-  int64_t           clusterId;
-  SDnode           *pDnode;
-  SendReqToDnodeFp  sendReqToDnodeFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
-  SendRedirectRspFp sendRedirectRspFp;
 } SSnodeOpt;
 
 /* ------------------------ SSnode ------------------------ */
@@ -73,16 +67,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
  * @param pSnode The snode object.
  * @param pMsg The request message
  * @param pRsp The response message
- * @return int32_t 0 for success, -1 for failure
  */
-int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
+void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
+void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
-/**
- * @brief Drop a snode.
- *
- * @param path Path of the snode.
- */
-void sndDestroy(const char *path);
 
 #ifdef __cplusplus
 }

include/libs/catalog/catalog.h

@@ -24,7 +24,7 @@ extern "C" {
 #include "taosdef.h"
 #include "query.h"
 #include "tname.h"
-#include "common.h"
+#include "tcommon.h"
 #include "tarray.h"
 #include "thash.h"
 #include "tmsg.h"
@@ -51,7 +51,7 @@ typedef struct SMetaData {
   SArray    *pTableMeta;  // STableMeta array
   SArray    *pVgroupInfo; // SVgroupInfo list
   SArray    *pUdfList;    // udf info list
-  SEpSet    *pEpSet;      // qnode epset list
+  SArray    *pEpSetList;  // qnode epset list, SArray<SEpSet>
 } SMetaData;
 
 typedef struct SCatalogCfg {
@@ -74,9 +74,9 @@ typedef struct SDbVgVersion {
   char    dbFName[TSDB_DB_FNAME_LEN];
   int64_t dbId;
   int32_t vgVersion;
+  int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
 } SDbVgVersion;
 
-
 int32_t catalogInit(SCatalogCfg *cfg);
 
 /**
@@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
  */
 void catalogFreeHandle(SCatalog* pCatalog);
 
-int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version);
+int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum);
 
 /**
  * Get a DB's all vgroup info.
@@ -103,16 +103,17 @@ int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* v
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
  * @param pDBName (input, full db name)
- * @param forceUpdate (input, force update db vgroup info from mnode) 
  * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
  * @return error code
  */
-int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
+int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, SArray** pVgroupList);
 
 int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo);
 
 int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
 
+int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName);
+
 int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
 
 /**
@@ -120,7 +121,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId,
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param pTableMeta(output, table meta data, NEED to free it by calller)
  * @return error code
  */
@@ -131,7 +132,7 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSe
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param pTableMeta(output, table meta data, NEED to free it by calller)
  * @return error code
  */
@@ -140,12 +141,22 @@ int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpS
 int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
 
 
+/**
+ * Force refresh DB's local cached vgroup info. 
+ * @param pCtg (input, got with catalogGetHandle)
+ * @param pTrans (input, rpc object)
+ * @param pMgmtEps (input, mnode EPs)
+ * @param dbFName (input, db full name)
+ * @return error code
+ */
+int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName);
+
 /**
  * Force refresh a table's local cached meta data. 
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) 
  * @return error code
  */
@@ -156,7 +167,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param pTableMeta(output, table meta data, NEED to free it by calller) 
  * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) 
  * @return error code
@@ -170,7 +181,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
  * @return error code
  */
@@ -181,7 +192,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, void *pTransporter, const
  * @param pCatalog (input, got with catalogGetHandle)
  * @param pTransporter (input, rpc object)
  * @param pMgmtEps (input, mnode EPs)
- * @param pTableName (input, table name, NOT including db name)
+ * @param pTableName (input, table name)
  * @param vgInfo (output, vgroup info)
  * @return error code
  */

include/libs/executor/dataSinkMgt.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "os.h"
 #include "thash.h"
 #include "executor.h"
+#include "plannodes.h"
 
 #define DS_BUF_LOW   1
 #define DS_BUF_FULL  2
@@ -40,7 +41,6 @@ int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg);
 
 typedef struct SInputData {
   const struct SSDataBlock* pData;
-  SHashObj* pTableRetrieveTsMap;
 } SInputData;
 
 typedef struct SOutputData {
@@ -59,7 +59,7 @@ typedef struct SOutputData {
  * @param pHandle output
  * @return error code
  */
-int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pHandle);
+int32_t dsCreateDataSinker(const SDataSinkNode* pDataSink, DataSinkHandle* pHandle);
 
 /**
  * Put the result set returned by the executor into datasinker.

include/libs/executor/executor.h

@@ -20,7 +20,8 @@
 extern "C" {
 #endif
 
-#include "common.h"
+#include "tcommon.h"
+#include "query.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -30,7 +31,12 @@ struct SSubplan;
 typedef struct SReadHandle {
   void* reader;
   void* meta;
+  void* config;
 } SReadHandle;
+
+#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
+#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
+  
 /**
  * Create the exec task for streaming mode
  * @param pMsg
@@ -40,12 +46,23 @@ typedef struct SReadHandle {
 qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
 
 /**
- *
+ * Set the input data block for the stream scan.
  * @param tinfo
  * @param input
+ * @param type
  * @return
  */
-int32_t     qSetStreamInput(qTaskInfo_t tinfo, const void* input);
+int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
+
+/**
+ * Update the table id list, add or remove.
+ *
+ * @param tinfo
+ * @param id
+ * @param isAdd
+ * @return
+ */
+int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd);
 
 /**
  * Create the exec task object according to task json
@@ -56,7 +73,8 @@ int32_t     qSetStreamInput(qTaskInfo_t tinfo, const void* input);
  * @param qId
  * @return
  */
-int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
+int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
+                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
 
 /**
  * The main task execution function, including query on both table and multiple tables,
@@ -78,25 +96,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
  */
 int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
 
-/**
- *
- * Retrieve the actual results to fill the response message payload.
- * Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
- *
- * @param tinfo  tinfo object
- * @param pRsp    response message
- * @param contLen payload length
- * @return
- */
-//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
-
-/**
- * return the transporter context (RPC)
- * @param tinfo
- * @return
- */
-void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
-
 /**
  * kill the ongoing query and free the query handle and corresponding resources automatically
  * @param tinfo  qhandle
@@ -148,7 +147,8 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
  * @param numOfIndex
  * @return
  */
-//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex);
+// int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex*
+// groupByIndex, int32_t numOfIndex);
 
 /**
  * Update the table id list of a given query.

include/libs/function/function.h

@@ -20,9 +20,30 @@
 extern "C" {
 #endif
 
-#include "common.h"
-#include "tvariant.h"
 #include "tbuffer.h"
+#include "tcommon.h"
+#include "tvariant.h"
+
+struct SqlFunctionCtx;
+struct SResultRowEntryInfo;
+
+typedef struct SFunctionNode SFunctionNode;
+
+typedef struct SFuncExecEnv {
+  int32_t calcMemSize;
+} SFuncExecEnv;
+
+typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
+typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+
+typedef struct SFuncExecFuncs {
+  FExecGetEnv getEnv;
+  FExecInit init;
+  FExecProcess process;
+  FExecFinalize finalize;
+} SFuncExecFuncs;
 
 #define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
 
@@ -111,62 +132,65 @@ struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 
 //for selectivity query, the corresponding tag value is assigned if the data is qualified
-typedef struct SExtTagsInfo {
+typedef struct SSubsidiaryResInfo {
-  int16_t                 tagsLen;      // keep the tags data for top/bottom query result
+  int16_t                 bufLen;      // keep the tags data for top/bottom query result
-  int16_t                 numOfTagCols;
+  int16_t                 numOfCols;
-  struct SqlFunctionCtx **pTagCtxList;
+  struct SqlFunctionCtx **pCtx;
-} SExtTagsInfo;
+} SSubsidiaryResInfo;
 
 typedef struct SResultDataInfo {
+  int16_t precision;
+  int16_t scale;
   int16_t type;
   int16_t bytes;
-  int32_t intermediateBytes;
+  int32_t interBufSize;
 } SResultDataInfo;
 
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
-typedef struct SFunctionFpSet {
+typedef struct SInputColumnInfoData {
-  bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);  // setup the execute environment
+  int32_t           totalRows;      // total rows in current columnar data
-  void (*addInput)(struct SqlFunctionCtx *pCtx);
+  int32_t           startRowIndex;  // handle started row index
-
+  int32_t           numOfRows;      // the number of rows needs to be handled
-  // finalizer must be called after all exec has been executed to generated final result.
+  int32_t           numOfInputCols; // PTS is not included
-  void (*finalize)(struct SqlFunctionCtx *pCtx);
+  bool              colDataAggIsSet;// if agg is set or not
-  void (*combine)(struct SqlFunctionCtx *pCtx);
+  SColumnInfoData  *pPTS;           // primary timestamp column
-} SFunctionFpSet;
+  SColumnInfoData **pData;
-
+  SColumnDataAgg  **pColumnDataAgg;
-extern SFunctionFpSet fpSet[1];
+} SInputColumnInfoData;
 
 // sql function runtime context
 typedef struct SqlFunctionCtx {
-  int32_t      size;      // number of rows
+  SInputColumnInfoData input;
-  void *       pInput;    // input data buffer
-  uint32_t     order;     // asc|desc
-  int16_t      inputType;
-  int16_t      inputBytes;
-
   SResultDataInfo      resDataInfo;
-  bool         hasNull;       // null value exist in current block
+  uint32_t             order;  // data block scanner order: asc|desc
-  bool         requireNull;   // require null in some function
+  ////////////////////////////////////////////////////////////////
+  int32_t          startRow;   // start row index
+  int32_t          size;       // handled processed row number
+  SColumnInfoData* pInput;
+  SColumnDataAgg   agg;
+  int16_t          inputType;    // TODO remove it
+  int16_t          inputBytes;   // TODO remove it
+  bool             hasNull;      // null value exist in current block, TODO remove it
+  bool             requireNull;  // require null in some function, TODO remove it
+  int32_t          columnIndex;  // TODO remove it
+  uint8_t          currentStage;  // record current running step, default: 0
+  bool             isAggSet;
+  /////////////////////////////////////////////////////////////////
   bool             stableQuery;
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
-  uint8_t      currentStage;  // record current running step, default: 0
   int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block
   int32_t          numOfParams;
   SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t         *ptsList;       // corresponding timestamp array list
   void            *ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
   SVariant         tag;
-
-  bool        isAggSet;
-  SColumnDataAgg agg;
   struct  SResultRowEntryInfo *resultInfo;
-  SExtTagsInfo tagInfo;
+  SSubsidiaryResInfo     subsidiaryRes;
   SPoint1          start;
   SPoint1          end;
-
+  SFuncExecFuncs   fpSet;
-  int32_t      columnIndex;
-  SFunctionFpSet* fpSet;
 } SqlFunctionCtx;
 
 enum {
@@ -192,9 +216,10 @@ typedef struct tExprNode {
     struct SVariant    *pVal;   // value node
 
     struct {// function node
-      char              functionName[FUNCTIONS_NAME_MAX_LENGTH];
+      char              functionName[FUNCTIONS_NAME_MAX_LENGTH];  // todo refactor
+      int32_t           functionId;
       int32_t           num;
-
+      SFunctionNode    *pFunctNode;
       // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
       // calculation instead.
       // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
@@ -205,7 +230,6 @@ typedef struct tExprNode {
   };
 } tExprNode;
 
-//TODO create?
 void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
 
@@ -228,6 +252,12 @@ typedef struct SAggFunctionInfo {
 
 typedef struct SScalarParam {
   void            *data;
+  union {
+    SColumnInfoData *columnData;
+    void            *data;
+  } orig;  
+  char            *bitmap;
+  bool             dataInBlock;
   int32_t          num;
   int32_t          type;
   int32_t          bytes;
@@ -265,19 +295,8 @@ typedef struct SMultiFunctionsDesc {
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength,
                           bool isSuperTable);
 
-/**
- * If the given name is a valid built-in sql function, the value of true will be returned.
- * @param name
- * @param len
- * @return
- */
-int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction);
-
 bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId);
 
-bool qIsAggregateFunction(const char* functionName);
-bool qIsSelectivityFunction(const char* functionName);
-
 tExprNode* exprTreeFromBinary(const void* data, size_t size);
 
 void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc);
@@ -286,7 +305,7 @@ tExprNode* exprdup(tExprNode* pTree);
 
 void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);
 void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell);
-int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num);
+int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock);
 bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
 bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);
 
@@ -308,7 +327,7 @@ bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
 
 struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
                               int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
-                              struct SFillColInfo* pFillCol, void* handle);
+                              struct SFillColInfo* pFillCol, const char* id);
 
 void* taosDestroyFillInfo(struct SFillInfo *pFillInfo);
 int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity);

include/libs/function/functionMgt.h

@@ -57,18 +57,19 @@ typedef enum EFunctionType {
 
   // math function
   FUNCTION_TYPE_ABS = 1000,
-  FUNCTION_TYPE_ACOS,
-  FUNCTION_TYPE_ASION,
-  FUNCTION_TYPE_ATAN,
-  FUNCTION_TYPE_CEIL,
-  FUNCTION_TYPE_COS,
-  FUNCTION_TYPE_FLOOR,
   FUNCTION_TYPE_LOG,
   FUNCTION_TYPE_POW,
-  FUNCTION_TYPE_ROUND,
-  FUNCTION_TYPE_SIN,
   FUNCTION_TYPE_SQRT,
+  FUNCTION_TYPE_CEIL,
+  FUNCTION_TYPE_FLOOR,
+  FUNCTION_TYPE_ROUND,
+
+  FUNCTION_TYPE_SIN,
+  FUNCTION_TYPE_COS,
   FUNCTION_TYPE_TAN,
+  FUNCTION_TYPE_ASIN,
+  FUNCTION_TYPE_ACOS,
+  FUNCTION_TYPE_ATAN,
 
   // string function
   FUNCTION_TYPE_CHAR_LENGTH = 1500,
@@ -102,22 +103,6 @@ struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
 
-typedef struct SFuncExecEnv {
-  int32_t calcMemSize;
-} SFuncExecEnv;
-
-typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
-typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
-typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
-
-typedef struct SFuncExecFuncs {
-  FExecGetEnv getEnv;
-  FExecInit init;
-  FExecProcess process;
-  FExecFinalize finalize;
-} SFuncExecFuncs;
-
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -127,6 +112,8 @@ typedef struct SScalarFuncExecFuncs {
 
 int32_t fmFuncMgtInit();
 
+void fmFuncMgtDestroy();
+
 int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc);

include/libs/index/index.h

@@ -29,6 +29,12 @@ typedef struct SIndexOpts           SIndexOpts;
 typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
 typedef struct SArray               SIndexMultiTerm;
 
+typedef struct SIndex               SIndexJson;
+typedef struct SIndexTerm           SIndexJsonTerm;
+typedef struct SIndexOpts           SIndexJsonOpts;
+typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery;
+typedef struct SArray               SIndexJsonMultiTerm;
+
 typedef enum {
   ADD_VALUE,     // add index colume value
   DEL_VALUE,     // delete index column value
@@ -39,24 +45,108 @@ typedef enum {
 } SIndexOperOnColumn;
 
 typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
-typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType;
+typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
+
 /*
- * @param: oper
+ * create multi query
- *
+ * @param oper (input, relation between querys)
  */
 SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper);
+
+/*
+ * destroy multi query
+ * @param pQuery (input, multi-query-object to be destory)
+ */
+
 void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
+/*
+ * add query to multi query
+ * @param pQuery (input, multi-query-object)
+ * @param term (input, single query term)
+ * @param type (input, single query type)
+ * @return error code
+ */
 int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
 /*
- * @param:
+ * open index
- * @param:
+ * @param opt (input, index opt)
+ * @param path (input, index path)
+ * @param index (output, index object)
+ * @return error code
  */
 int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
+/*
+ * close index
+ * @param index (input, index to be closed)
+ * @return error code
+ */
 void indexClose(SIndex* index);
+
+/*
+ * insert terms into index
+ * @param index (input, index object)
+ * @param term (input, terms inserted into index)
+ * @param uid  (input, uid of terms)
+ * @return error code
+ */
 int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
+/*
+ * delete terms that meet query condition
+ * @param index (input, index object)
+ * @param query (input, condition query to deleted)
+ * @return error code
+ */
+
 int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
+/*
+ * search index
+ * @param index (input, index object)
+ * @param query (input, multi query condition)
+ * @param result(output, query result)
+ * @return error code
+ */
 int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
+/*
+ * rebuild index
+ * @param index (input, index object)
+ * @parma opt   (input, rebuild index opts)
+ * @return error code
+ */
 int indexRebuild(SIndex* index, SIndexOpts* opt);
+
+/*
+ * open index
+ * @param opt (input,index json opt)
+ * @param path (input, index json path)
+ * @param index (output, index json object)
+ * @return error code
+ */
+int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index);
+/*
+ * close index
+ * @param index (input, index to be closed)
+ * @return void
+ */
+
+void tIndexJsonClose(SIndexJson* index);
+
+/*
+ * insert terms into index
+ * @param index (input, index object)
+ * @param term (input, terms inserted into index)
+ * @param uid  (input, uid of terms)
+ * @return error code
+ */
+int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
+/*
+ * search index
+ * @param index (input, index object)
+ * @param query (input, multi query condition)
+ * @param result(output, query result)
+ * @return error code
+ */
+
+int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result);
 /*
  * @param
  * @param

include/libs/monitor/monitor.h

@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MONITOR_H_
+#define _TD_MONITOR_H_
+
+#include "tarray.h"
+#include "tdef.h"
+#include "tlog.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MON_STATUS_LEN 8
+#define MON_ROLE_LEN   9
+#define MON_VER_LEN    12
+#define MON_LOG_LEN    1024
+
+typedef struct {
+  int32_t dnode_id;
+  char    dnode_ep[TSDB_EP_LEN];
+  int64_t cluster_id;
+  int32_t protocol;
+} SMonBasicInfo;
+
+typedef struct {
+  int32_t dnode_id;
+  char    dnode_ep[TSDB_EP_LEN];
+  char    status[MON_STATUS_LEN];
+} SMonDnodeDesc;
+
+typedef struct {
+  int32_t mnode_id;
+  char    mnode_ep[TSDB_EP_LEN];
+  char    role[MON_ROLE_LEN];
+} SMonMnodeDesc;
+
+typedef struct {
+  char    first_ep[TSDB_EP_LEN];
+  int32_t first_ep_dnode_id;
+  char    version[MON_VER_LEN];
+  float   master_uptime;     // day
+  int32_t monitor_interval;  // sec
+  int32_t vgroups_total;
+  int32_t vgroups_alive;
+  int32_t vnodes_total;
+  int32_t vnodes_alive;
+  int32_t connections_total;
+  SArray *dnodes;  // array of SMonDnodeDesc
+  SArray *mnodes;  // array of SMonMnodeDesc
+} SMonClusterInfo;
+
+typedef struct {
+  int32_t dnode_id;
+  char    vnode_role[MON_ROLE_LEN];
+} SMonVnodeDesc;
+
+typedef struct {
+  int32_t       vgroup_id;
+  char          database_name[TSDB_DB_NAME_LEN];
+  int32_t       tables_num;
+  char          status[MON_STATUS_LEN];
+  SMonVnodeDesc vnodes[TSDB_MAX_REPLICA];
+} SMonVgroupDesc;
+
+typedef struct {
+  SArray *vgroups;  // array of SMonVgroupDesc
+} SMonVgroupInfo;
+
+typedef struct {
+  int32_t expire_time;
+  int32_t timeseries_used;
+  int32_t timeseries_total;
+} SMonGrantInfo;
+
+typedef struct {
+  float   uptime;  // day
+  double  cpu_engine;
+  double  cpu_system;
+  float   cpu_cores;
+  int64_t mem_engine;   // KB
+  int64_t mem_system;   // KB
+  int64_t mem_total;    // KB
+  int64_t disk_engine;  // Byte
+  int64_t disk_used;    // Byte
+  int64_t disk_total;   // Byte
+  int64_t net_in;       // bytes
+  int64_t net_out;      // bytes
+  int64_t io_read;      // bytes
+  int64_t io_write;     // bytes
+  int64_t io_read_disk;   // bytes
+  int64_t io_write_disk;  // bytes
+  int64_t req_select;
+  int64_t req_insert;
+  int64_t req_insert_success;
+  int64_t req_insert_batch;
+  int64_t req_insert_batch_success;
+  int32_t errors;
+  int32_t vnodes_num;
+  int32_t masters;
+  int8_t  has_mnode;
+} SMonDnodeInfo;
+
+typedef struct {
+  char      name[TSDB_FILENAME_LEN];
+  int8_t    level;
+  SDiskSize size;
+} SMonDiskDesc;
+
+typedef struct {
+  SArray      *datadirs;  // array of SMonDiskDesc
+  SMonDiskDesc logdir;
+  SMonDiskDesc tempdir;
+} SMonDiskInfo;
+
+typedef struct SMonInfo SMonInfo;
+
+typedef struct {
+  const char *server;
+  uint16_t    port;
+  int32_t     maxLogs;
+  bool        comp;
+} SMonCfg;
+
+int32_t monInit(const SMonCfg *pCfg);
+void    monCleanup();
+void    monRecordLog(int64_t ts, ELogLevel level, const char *content);
+
+SMonInfo *monCreateMonitorInfo();
+void      monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
+void      monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo);
+void      monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo);
+void      monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo);
+void      monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo);
+void      monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo);
+void      monSendReport(SMonInfo *pMonitor);
+void      monCleanupMonitorInfo(SMonInfo *pMonitor);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MONITOR_H_*/

include/libs/nodes/cmdnodes.h

@@ -0,0 +1,250 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_CMD_NODES_H_
+#define _TD_CMD_NODES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "querynodes.h"
+
+typedef struct SDatabaseOptions {
+  ENodeType type;
+  int32_t numOfBlocks;
+  int32_t cacheBlockSize;
+  int8_t cachelast;
+  int32_t compressionLevel;
+  int32_t daysPerFile;
+  int32_t fsyncPeriod;
+  int32_t maxRowsPerBlock;
+  int32_t minRowsPerBlock;
+  int32_t keep;
+  int32_t precision;
+  int32_t quorum;
+  int32_t replica;
+  int32_t ttl;
+  int32_t walLevel;
+  int32_t numOfVgroups;
+  int8_t singleStable;
+  int8_t streamMode;
+} SDatabaseOptions;
+
+typedef struct SCreateDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  bool ignoreExists;
+  SDatabaseOptions* pOptions;
+} SCreateDatabaseStmt;
+
+typedef struct SUseDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+} SUseDatabaseStmt;
+
+typedef struct SDropDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  bool ignoreNotExists;
+} SDropDatabaseStmt;
+
+typedef struct SAlterDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  SDatabaseOptions* pOptions;
+} SAlterDatabaseStmt;
+
+typedef struct STableOptions {
+  ENodeType type;
+  int32_t keep;
+  int32_t ttl;
+  char comments[TSDB_STB_COMMENT_LEN];
+  SNodeList* pSma;
+} STableOptions;
+
+typedef struct SColumnDefNode {
+  ENodeType type;
+  char colName[TSDB_COL_NAME_LEN];
+  SDataType dataType;
+  char comments[TSDB_STB_COMMENT_LEN];
+} SColumnDefNode;
+
+typedef struct SCreateTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreExists;
+  SNodeList* pCols;
+  SNodeList* pTags;
+  STableOptions* pOptions;
+} SCreateTableStmt;
+
+typedef struct SCreateSubTableClause {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  char useDbName[TSDB_DB_NAME_LEN];
+  char useTableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreExists;
+  SNodeList* pSpecificTags;
+  SNodeList* pValsOfTags;
+} SCreateSubTableClause;
+
+typedef struct SCreateMultiTableStmt {
+  ENodeType type;
+  SNodeList* pSubTables;
+} SCreateMultiTableStmt;
+
+typedef struct SDropTableClause {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropTableClause;
+
+typedef struct SDropTableStmt {
+  ENodeType type;
+  SNodeList* pTables;
+} SDropTableStmt;
+
+typedef struct SDropSuperTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropSuperTableStmt;
+
+typedef struct SAlterTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  int8_t alterType;
+  char colName[TSDB_COL_NAME_LEN];
+  char newColName[TSDB_COL_NAME_LEN];
+  STableOptions* pOptions;
+  SDataType dataType;
+  SValueNode* pVal;
+} SAlterTableStmt;
+
+typedef struct SCreateUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+  char password[TSDB_USET_PASSWORD_LEN];
+} SCreateUserStmt;
+
+typedef struct SAlterUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+  char password[TSDB_USET_PASSWORD_LEN];
+  int8_t alterType;
+} SAlterUserStmt;
+
+typedef struct SDropUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+} SDropUserStmt;
+
+typedef struct SCreateDnodeStmt {
+  ENodeType type;
+  char fqdn[TSDB_FQDN_LEN];
+  int32_t port;
+} SCreateDnodeStmt;
+
+typedef struct SDropDnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+  char fqdn[TSDB_FQDN_LEN];
+  int32_t port;
+} SDropDnodeStmt;
+
+typedef struct SAlterDnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+  char config[TSDB_DNODE_CONFIG_LEN];
+  char value[TSDB_DNODE_VALUE_LEN];
+} SAlterDnodeStmt;
+
+typedef struct SShowStmt {
+  ENodeType type;
+  SNode* pDbName;        // SValueNode
+  SNode* pTbNamePattern; // SValueNode
+} SShowStmt;
+
+typedef enum EIndexType {
+  INDEX_TYPE_SMA = 1,
+  INDEX_TYPE_FULLTEXT
+} EIndexType;
+
+typedef struct SIndexOptions {
+  ENodeType type;
+  SNodeList* pFuncs;
+  SNode* pInterval;
+  SNode* pOffset;
+  SNode* pSliding;
+} SIndexOptions;
+
+typedef struct SCreateIndexStmt {
+  ENodeType type;
+  EIndexType indexType;
+  bool ignoreExists;
+  char indexName[TSDB_INDEX_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  SNodeList* pCols;
+  SIndexOptions* pOptions;
+} SCreateIndexStmt;
+
+typedef struct SDropIndexStmt {
+  ENodeType type;
+  bool ignoreNotExists;
+  char indexName[TSDB_INDEX_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SDropIndexStmt;
+
+typedef struct SCreateQnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+} SCreateQnodeStmt;
+
+typedef struct SDropQnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+} SDropQnodeStmt;
+
+typedef struct SCreateTopicStmt {
+  ENodeType type;
+  char topicName[TSDB_TABLE_NAME_LEN];
+  char subscribeDbName[TSDB_DB_NAME_LEN];
+  bool ignoreExists;
+  SNode* pQuery;
+} SCreateTopicStmt;
+
+typedef struct SDropTopicStmt {
+  ENodeType type;
+  char topicName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropTopicStmt;
+
+typedef struct SAlterLocalStmt {
+  ENodeType type;
+  char config[TSDB_DNODE_CONFIG_LEN];
+  char value[TSDB_DNODE_VALUE_LEN];
+} SAlterLocalStmt;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_CMD_NODES_H_*/

include/libs/nodes/nodes.h

@@ -61,22 +61,86 @@ typedef enum ENodeType {
   QUERY_NODE_INTERVAL_WINDOW,
   QUERY_NODE_NODE_LIST,
   QUERY_NODE_FILL,
-  QUERY_NODE_COLUMN_REF,
+  QUERY_NODE_RAW_EXPR, // Only be used in parser module.
   QUERY_NODE_TARGET,
-
+  QUERY_NODE_DATABLOCK_DESC,
-  // Only be used in parser module.
+  QUERY_NODE_SLOT_DESC,
-  QUERY_NODE_RAW_EXPR,
+  QUERY_NODE_COLUMN_DEF,
+  QUERY_NODE_DOWNSTREAM_SOURCE,
+  QUERY_NODE_DATABASE_OPTIONS,
+  QUERY_NODE_TABLE_OPTIONS,
+  QUERY_NODE_INDEX_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
   QUERY_NODE_SELECT_STMT,
-  QUERY_NODE_SHOW_STMT,
+  QUERY_NODE_VNODE_MODIF_STMT,
+  QUERY_NODE_CREATE_DATABASE_STMT,
+  QUERY_NODE_DROP_DATABASE_STMT,
+  QUERY_NODE_ALTER_DATABASE_STMT,
+  QUERY_NODE_CREATE_TABLE_STMT,
+  QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
+  QUERY_NODE_CREATE_MULTI_TABLE_STMT,
+  QUERY_NODE_DROP_TABLE_CLAUSE,
+  QUERY_NODE_DROP_TABLE_STMT,
+  QUERY_NODE_DROP_SUPER_TABLE_STMT,
+  QUERY_NODE_ALTER_TABLE_STMT,
+  QUERY_NODE_CREATE_USER_STMT,
+  QUERY_NODE_ALTER_USER_STMT,
+  QUERY_NODE_DROP_USER_STMT,
+  QUERY_NODE_USE_DATABASE_STMT,
+  QUERY_NODE_CREATE_DNODE_STMT,
+  QUERY_NODE_DROP_DNODE_STMT,
+  QUERY_NODE_ALTER_DNODE_STMT,
+  QUERY_NODE_CREATE_INDEX_STMT,
+  QUERY_NODE_DROP_INDEX_STMT,
+  QUERY_NODE_CREATE_QNODE_STMT,
+  QUERY_NODE_DROP_QNODE_STMT,
+  QUERY_NODE_CREATE_TOPIC_STMT,
+  QUERY_NODE_DROP_TOPIC_STMT,
+  QUERY_NODE_ALTER_LOCAL_STMT,
+  QUERY_NODE_SHOW_DATABASES_STMT,
+  QUERY_NODE_SHOW_TABLES_STMT,
+  QUERY_NODE_SHOW_STABLES_STMT,
+  QUERY_NODE_SHOW_USERS_STMT,
+  QUERY_NODE_SHOW_DNODES_STMT,
+  QUERY_NODE_SHOW_VGROUPS_STMT,
+  QUERY_NODE_SHOW_MNODES_STMT,
+  QUERY_NODE_SHOW_MODULES_STMT,
+  QUERY_NODE_SHOW_QNODES_STMT,
+  QUERY_NODE_SHOW_FUNCTIONS_STMT,
+  QUERY_NODE_SHOW_INDEXES_STMT,
+  QUERY_NODE_SHOW_STREAMS_STMT,
 
+  // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
   QUERY_NODE_LOGIC_PLAN_JOIN,
-  QUERY_NODE_LOGIC_PLAN_FILTER,
   QUERY_NODE_LOGIC_PLAN_AGG,
-  QUERY_NODE_LOGIC_PLAN_PROJECT
+  QUERY_NODE_LOGIC_PLAN_PROJECT,
+  QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
+  QUERY_NODE_LOGIC_PLAN_EXCHANGE,
+  QUERY_NODE_LOGIC_PLAN_WINDOW,
+  QUERY_NODE_LOGIC_PLAN_SORT,
+  QUERY_NODE_LOGIC_SUBPLAN,
+  QUERY_NODE_LOGIC_PLAN,
+
+  // physical plan node
+  QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_PROJECT,
+  QUERY_NODE_PHYSICAL_PLAN_JOIN,
+  QUERY_NODE_PHYSICAL_PLAN_AGG,
+  QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
+  QUERY_NODE_PHYSICAL_PLAN_SORT,
+  QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
+  QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
+  QUERY_NODE_PHYSICAL_PLAN_INSERT,
+  QUERY_NODE_PHYSICAL_SUBPLAN,
+  QUERY_NODE_PHYSICAL_PLAN
 } ENodeType;
 
 /**
@@ -99,15 +163,22 @@ typedef struct SNodeList {
   SListCell* pTail;
 } SNodeList;
 
-SNode* nodesMakeNode(ENodeType type);
+#define SNodeptr void*  
-void nodesDestroyNode(SNode* pNode);
+
+SNodeptr nodesMakeNode(ENodeType type);
+void nodesDestroyNode(SNodeptr pNode);
 
 SNodeList* nodesMakeList();
-int32_t nodesListAppend(SNodeList* pList, SNode* pNode);
+int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
+int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
+int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
+int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
-SNode* nodesListGetNode(SNodeList* pList, int32_t index);
+SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);
+// Only clear the linked list structure, without releasing the elements inside
+void nodesClearList(SNodeList* pList);
 
 typedef enum EDealRes {
   DEAL_RES_CONTINUE = 1,
@@ -116,9 +187,9 @@ typedef enum EDealRes {
 } EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
-void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext);
+void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext);
 void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext);
-void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext);
+void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
 void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
 
 typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext);
@@ -127,14 +198,18 @@ void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
 void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 
-bool nodesEqualNode(const SNode* a, const SNode* b);
+bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
 
-SNode* nodesCloneNode(const SNode* pNode);
+SNodeptr nodesCloneNode(const SNodeptr pNode);
 SNodeList* nodesCloneList(const SNodeList* pList);
 
-int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen);
+const char* nodesNodeName(ENodeType type);
+int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 
+int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
+int32_t nodesStringToList(const char* pStr, SNodeList** pList);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/plannodes.h

@@ -21,20 +21,34 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "query.h"
+#include "tname.h"
 
 typedef struct SLogicNode {
   ENodeType type;
-  int32_t id;
   SNodeList* pTargets; // SColumnNode
   SNode* pConditions;
   SNodeList* pChildren;
   struct SLogicNode* pParent;
 } SLogicNode;
 
+typedef enum EScanType {
+  SCAN_TYPE_TAG,
+  SCAN_TYPE_TABLE,
+  SCAN_TYPE_SYSTEM_TABLE,
+  SCAN_TYPE_STREAM
+} EScanType;
+
 typedef struct SScanLogicNode {
   SLogicNode node;
   SNodeList* pScanCols;
   struct STableMeta* pMeta;
+  SVgroupsInfo* pVgroupList;
+  EScanType scanType;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
+  SName tableName;
+  bool showRewrite;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -43,10 +57,6 @@ typedef struct SJoinLogicNode {
   SNode* pOnConditions;
 } SJoinLogicNode;
 
-typedef struct SFilterLogicNode {
-  SLogicNode node;
-} SFilterLogicNode;
-
 typedef struct SAggLogicNode {
   SLogicNode node;
   SNodeList* pGroupKeys;
@@ -56,8 +66,230 @@ typedef struct SAggLogicNode {
 typedef struct SProjectLogicNode {
   SLogicNode node;
   SNodeList* pProjections;
+  char stmtName[TSDB_TABLE_NAME_LEN]; 
 } SProjectLogicNode;
 
+typedef struct SVnodeModifLogicNode {
+  SLogicNode node;
+  int32_t msgType;
+  SArray* pDataBlocks;
+  SVgDataBlocks* pVgDataBlocks;
+} SVnodeModifLogicNode;
+
+typedef struct SExchangeLogicNode {
+  SLogicNode node;
+  int32_t srcGroupId;
+} SExchangeLogicNode;
+
+typedef enum EWindowType {
+  WINDOW_TYPE_INTERVAL = 1,
+  WINDOW_TYPE_SESSION,
+  WINDOW_TYPE_STATE
+} EWindowType;
+
+typedef struct SWindowLogicNode {
+  SLogicNode node;
+  EWindowType winType;
+  SNodeList* pFuncs;
+  int64_t interval;
+  int64_t offset;
+  int64_t sliding;
+  int8_t  intervalUnit;
+  int8_t  slidingUnit;
+  SFillNode* pFill;
+  int64_t sessionGap;
+  SNode* pTspk;
+} SWindowLogicNode;
+
+typedef struct SSortLogicNode {
+  SLogicNode node;
+  SNodeList* pSortKeys;
+} SSortLogicNode;
+
+typedef enum ESubplanType {
+  SUBPLAN_TYPE_MERGE = 1,
+  SUBPLAN_TYPE_PARTIAL,
+  SUBPLAN_TYPE_SCAN,
+  SUBPLAN_TYPE_MODIFY
+} ESubplanType;
+
+typedef struct SSubplanId {
+  uint64_t queryId;
+  int32_t groupId;
+  int32_t subplanId;
+} SSubplanId;
+
+typedef struct SLogicSubplan {
+  ENodeType type;
+  SSubplanId id;
+  SNodeList* pChildren;
+  SNodeList* pParents;
+  SLogicNode* pNode;
+  ESubplanType subplanType;
+  SVgroupsInfo* pVgroupList;
+  int32_t level;
+  int32_t splitFlag;
+} SLogicSubplan;
+
+typedef struct SQueryLogicPlan {
+  ENodeType type;
+  int32_t totalLevel;
+  SNodeList* pTopSubplans;
+} SQueryLogicPlan;
+
+typedef struct SSlotDescNode {
+  ENodeType type;
+  int16_t slotId;
+  SDataType dataType;
+  bool reserve;
+  bool output;
+  bool tag;
+} SSlotDescNode;
+
+typedef struct SDataBlockDescNode {
+  ENodeType type;
+  int16_t dataBlockId;
+  SNodeList* pSlots;
+  int32_t resultRowSize;
+  int16_t precision;
+} SDataBlockDescNode;
+
+typedef struct SPhysiNode {
+  ENodeType type;
+  SDataBlockDescNode* pOutputDataBlockDesc;
+  SNode* pConditions;
+  SNodeList* pChildren;
+  struct SPhysiNode* pParent;
+} SPhysiNode;
+
+typedef struct SScanPhysiNode {
+  SPhysiNode node;
+  SNodeList* pScanCols;
+  uint64_t uid;           // unique id of the table
+  int8_t tableType;
+  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
+  int32_t count;         // repeat count
+  int32_t reverse;       // reverse scan count
+  SName tableName;
+} SScanPhysiNode;
+
+typedef SScanPhysiNode STagScanPhysiNode;
+typedef SScanPhysiNode SStreamScanPhysiNode;
+
+typedef struct SSystemTableScanPhysiNode {
+  SScanPhysiNode scan;
+  SEpSet mgmtEpSet;
+  bool showRewrite;
+  int32_t accountId;
+} SSystemTableScanPhysiNode;
+
+typedef struct STableScanPhysiNode {
+  SScanPhysiNode scan;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
+  SNode* pScanConditions;
+} STableScanPhysiNode;
+
+typedef STableScanPhysiNode STableSeqScanPhysiNode;
+
+typedef struct SProjectPhysiNode {
+  SPhysiNode node;
+  SNodeList* pProjections;
+} SProjectPhysiNode;
+
+typedef struct SJoinPhysiNode {
+  SPhysiNode node;
+  EJoinType joinType;
+  SNode* pOnConditions; // in or out tuple ?
+  SNodeList* pTargets;
+} SJoinPhysiNode;
+
+typedef struct SAggPhysiNode {
+  SPhysiNode node;
+  SNodeList* pExprs;   // these are expression list of group_by_clause and parameter expression of aggregate function
+  SNodeList* pGroupKeys;
+  SNodeList* pAggFuncs;
+} SAggPhysiNode;
+
+typedef struct SDownstreamSourceNode {
+  ENodeType type;
+  SQueryNodeAddr addr;
+  uint64_t taskId;
+  uint64_t schedId;
+} SDownstreamSourceNode;
+
+typedef struct SExchangePhysiNode {
+  SPhysiNode node;
+  int32_t    srcGroupId;  // group id of datasource suplans
+  SNodeList* pSrcEndPoints;  // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
+} SExchangePhysiNode;
+
+typedef struct SWinodwPhysiNode {
+  SPhysiNode node;
+  SNodeList* pExprs;   // these are expression list of parameter expression of function
+  SNodeList* pFuncs;
+} SWinodwPhysiNode;
+
+typedef struct SIntervalPhysiNode {
+  SWinodwPhysiNode window;
+  SNode* pTspk; // timestamp primary key
+  int64_t    interval;
+  int64_t    offset;
+  int64_t    sliding;
+  int8_t     intervalUnit;
+  int8_t     slidingUnit;
+  SFillNode* pFill;
+} SIntervalPhysiNode;
+
+typedef struct SSessionWinodwPhysiNode {
+  SWinodwPhysiNode window;
+  int64_t    gap;
+} SSessionWinodwPhysiNode;
+
+typedef struct SSortPhysiNode {
+  SPhysiNode node;
+  SNodeList* pExprs;   // these are expression list of order_by_clause and parameter expression of aggregate function
+  SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
+} SSortPhysiNode;
+
+typedef struct SDataSinkNode {
+  ENodeType type;
+  SDataBlockDescNode* pInputDataBlockDesc;
+} SDataSinkNode;
+
+typedef struct SDataDispatcherNode {
+  SDataSinkNode sink;
+} SDataDispatcherNode;
+
+typedef struct SDataInserterNode {
+  SDataSinkNode sink;
+  int32_t   numOfTables;
+  uint32_t  size;
+  char     *pData;
+} SDataInserterNode;
+
+typedef struct SSubplan {
+  ENodeType type;
+  SSubplanId id;           // unique id of the subplan
+  ESubplanType subplanType;
+  int32_t msgType;      // message type for subplan, used to denote the send message type to vnode.
+  int32_t level;        // the execution level of current subplan, starting from 0 in a top-down manner.
+  char dbFName[TSDB_DB_FNAME_LEN];
+  SQueryNodeAddr execNode;    // for the scan/modify subplan, the optional execution node
+  SQueryNodeStat execNodeStat; // only for scan subplan
+  SNodeList* pChildren;    // the datasource subplan,from which to fetch the result
+  SNodeList* pParents;     // the data destination subplan, get data from current subplan
+  SPhysiNode* pNode;        // physical plan of current subplan
+  SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
+} SSubplan;
+
+typedef struct SQueryPlan {
+  ENodeType type;
+  uint64_t queryId;
+  int32_t numOfSubplans;
+  SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
+} SQueryPlan;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/querynodes.h

@@ -21,6 +21,11 @@ extern "C" {
 #endif
 
 #include "nodes.h"
+#include "tmsg.h"
+
+#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
+#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
 
 typedef struct SRawExprNode {
   ENodeType nodeType;
@@ -51,27 +56,29 @@ typedef enum EColumnType {
 typedef struct SColumnNode {
   SExprNode node; // QUERY_NODE_COLUMN
   uint64_t tableId;
-  int16_t colId;
+  col_id_t colId;
   EColumnType colType; // column or tag
   char dbName[TSDB_DB_NAME_LEN];
   char tableName[TSDB_TABLE_NAME_LEN];
   char tableAlias[TSDB_TABLE_NAME_LEN];
   char colName[TSDB_COL_NAME_LEN];
   SNode* pProjectRef;
+  int16_t dataBlockId;
+  int16_t slotId;
 } SColumnNode;
 
-typedef struct SColumnRefNode {
+typedef struct STargetNode {
   ENodeType type;
-  SDataType dataType;
+  int16_t dataBlockId;
-  int16_t tupleId;
   int16_t slotId;
-  int16_t columnId;
+  SNode* pExpr;
-} SColumnRefNode;
+} STargetNode;
 
 typedef struct SValueNode {
   SExprNode node; // QUERY_NODE_VALUE
   char* literal;
   bool isDuration;
+  bool translate;
   union {
     bool b;
     int64_t i;
@@ -79,47 +86,9 @@ typedef struct SValueNode {
     double d;
     char* p;
   } datum;
+  char unit;
 } SValueNode;
 
-typedef enum EOperatorType {
-  // arithmetic operator
-  OP_TYPE_ADD = 1,
-  OP_TYPE_SUB,
-  OP_TYPE_MULTI,
-  OP_TYPE_DIV,
-  OP_TYPE_MOD,
-
-  // bit operator
-  OP_TYPE_BIT_AND,
-  OP_TYPE_BIT_OR,
-
-  // comparison operator
-  OP_TYPE_GREATER_THAN,
-  OP_TYPE_GREATER_EQUAL,
-  OP_TYPE_LOWER_THAN,
-  OP_TYPE_LOWER_EQUAL,
-  OP_TYPE_EQUAL,
-  OP_TYPE_NOT_EQUAL,
-  OP_TYPE_IN,
-  OP_TYPE_NOT_IN,
-  OP_TYPE_LIKE,
-  OP_TYPE_NOT_LIKE,
-  OP_TYPE_MATCH,
-  OP_TYPE_NMATCH,
-  OP_TYPE_IS_NULL,
-  OP_TYPE_IS_NOT_NULL,
-  OP_TYPE_IS_TRUE,
-  OP_TYPE_IS_FALSE,
-  OP_TYPE_IS_UNKNOWN,
-  OP_TYPE_IS_NOT_TRUE,
-  OP_TYPE_IS_NOT_FALSE,
-  OP_TYPE_IS_NOT_UNKNOWN,
-
-  // json operator
-  OP_TYPE_JSON_GET_VALUE,
-  OP_TYPE_JSON_CONTAINS
-} EOperatorType;
-
 typedef struct SOperatorNode {
   SExprNode node; // QUERY_NODE_OPERATOR
   EOperatorType opType;
@@ -127,11 +96,6 @@ typedef struct SOperatorNode {
   SNode* pRight;
 } SOperatorNode;
 
-typedef enum ELogicConditionType {
-  LOGIC_COND_TYPE_AND,
-  LOGIC_COND_TYPE_OR,
-  LOGIC_COND_TYPE_NOT,
-} ELogicConditionType;
 
 typedef struct SLogicConditionNode {
   SExprNode node; // QUERY_NODE_LOGIC_CONDITION
@@ -141,6 +105,7 @@ typedef struct SLogicConditionNode {
 
 typedef struct SNodeListNode {
   ENodeType type; // QUERY_NODE_NODE_LIST
+  SDataType dataType;
   SNodeList* pNodeList;
 } SNodeListNode;
 
@@ -164,6 +129,8 @@ struct STableMeta;
 typedef struct SRealTableNode {
   STableNode table; // QUERY_NODE_REAL_TABLE
   struct STableMeta* pMeta;
+  SVgroupsInfo* pVgroupList;
+  char useDbName[TSDB_DB_NAME_LEN];
 } SRealTableNode;
 
 typedef struct STempTableNode {
@@ -224,12 +191,13 @@ typedef struct SStateWindowNode {
 
 typedef struct SSessionWindowNode {
   ENodeType type; // QUERY_NODE_SESSION_WINDOW
-  int64_t gap;             // gap between two session window(in microseconds)
+  SNode* pCol; // timestamp primary key
-  SNode* pCol;
+  SNode* pGap; // gap between two session window(in microseconds)
 } SSessionWindowNode;
 
 typedef struct SIntervalWindowNode {
   ENodeType type; // QUERY_NODE_INTERVAL_WINDOW
+  SNode* pCol; // timestamp primary key
   SNode* pInterval; // SValueNode
   SNode* pOffset;   // SValueNode
   SNode* pSliding;  // SValueNode
@@ -264,6 +232,7 @@ typedef struct SSelectStmt {
   SNodeList* pOrderByList; // SOrderByExprNode
   SNode* pLimit;
   SNode* pSlimit;
+  char stmtName[TSDB_TABLE_NAME_LEN];
 } SSelectStmt;
 
 typedef enum ESetOperatorType {
@@ -290,6 +259,29 @@ typedef enum ESqlClause {
   SQL_CLAUSE_ORDER_BY
 } ESqlClause;
 
+
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+typedef struct SVgDataBlocks {
+  SVgroupInfo vg;
+  int32_t     numOfTables;  // number of tables in current submit block
+  uint32_t    size;
+  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
+} SVgDataBlocks;
+
+typedef struct SVnodeModifOpStmt {
+  ENodeType   nodeType;
+  ENodeType   sqlNodeType;
+  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
+  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  const char* sql;                 // current sql statement position
+} SVnodeModifOpStmt;
+
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
@@ -306,6 +298,7 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
+
 void* nodesGetValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus

include/libs/parser/parsenodes.h

@@ -1,163 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_PARSENODES_H_
-#define _TD_PARSENODES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "catalog.h"
-#include "common.h"
-#include "function.h"
-#include "tmsgtype.h"
-#include "tname.h"
-#include "tvariant.h"
-
-/**
- * The first field of a node of any type is guaranteed to be the int16_t.
- * Hence the type of any node can be gotten by casting it to SQueryNode. 
- */
-typedef struct SQueryNode {
-  int16_t type;
-} SQueryNode;
-
-#define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type)
-
-typedef struct SFieldInfo {
-  int16_t     numOfOutput;   // number of column in result
-  SField     *final;
-  SArray     *internalField; // SArray<SInternalField>
-} SFieldInfo;
-
-typedef struct SCond {
-  uint64_t uid;
-  int32_t  len;  // length of tag query condition data
-  char *   cond;
-} SCond;
-
-typedef struct SJoinNode {
-  uint64_t uid;
-  int16_t  tagColId;
-  SArray*  tsJoin;
-  SArray*  tagJoin;
-} SJoinNode;
-
-typedef struct SJoinInfo {
-  bool       hasJoin;
-  SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM];
-} SJoinInfo;
-
-typedef struct STagCond {
-  int16_t   relType;     // relation between tbname list and query condition, including : TK_AND or TK_OR
-  SCond     tbnameCond;  // tbname query condition, only support tbname query condition on one table
-  SJoinInfo joinInfo;    // join condition, only support two tables join currently
-  SArray   *pCond;       // for different table, the query condition must be seperated
-} STagCond;
-
-typedef struct STableMetaInfo {
-  STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
-  SVgroupsInfo  *vgroupList;
-  SName         name;
-  char          aliasName[TSDB_TABLE_NAME_LEN];    // alias name of table specified in query sql
-  SArray       *tagColList;                        // SArray<SColumn*>, involved tag columns
-} STableMetaInfo;
-
-typedef struct SColumnIndex {
-  int16_t tableIndex;
-  int16_t columnIndex;
-  int16_t type;               // normal column/tag/ user input constant column
-} SColumnIndex;
-
-// select statement
-typedef struct SQueryStmtInfo {
-  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
-  uint32_t         type;          // query/insert type
-  STimeWindow      window;        // the whole query time window
-  SInterval        interval;      // tumble time window
-  SSessionWindow   sessionWindow; // session time window
-  SStateWindow     stateWindow;   // state window query
-  SGroupbyExpr     groupbyExpr;   // groupby tags info
-  SArray *         colList;       // SArray<SColumn*>
-  SFieldInfo       fieldsInfo;
-  SArray**         exprList;      // SArray<SExprInfo*>
-  SLimit           limit;
-  SLimit           slimit;
-  STagCond         tagCond;
-  SArray *         colCond;
-  SArray *         order;
-  int16_t          numOfTables;
-  int16_t          curTableIdx;
-  STableMetaInfo **pTableMetaInfo;
-  struct STSBuf   *tsBuf;
-
-  int16_t          fillType;      // final result fill type
-  int64_t *        fillVal;       // default value for fill
-  int32_t          numOfFillVal;  // fill value size
-
-  char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
-  int64_t          clauseLimit;   // limit for current sub clause
-
-  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
-  int64_t          vgroupLimit;    // table limit in case of super table projection query + global order + limit
-
-  int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
-  int32_t          bufLen;
-  char*            buf;
-  SArray          *pUdfInfo;
-
-  struct SQueryStmtInfo *sibling;     // sibling
-  SMultiFunctionsDesc    info;
-  SArray            *pDownstream;   // SArray<struct SQueryStmtInfo>
-  int32_t            havingFieldNum;
-  int32_t            exprListLevelIndex;
-} SQueryStmtInfo;
-
-typedef enum {
-  PAYLOAD_TYPE_KV = 0,
-  PAYLOAD_TYPE_RAW = 1,
-} EPayloadType;
-
-typedef struct SVgDataBlocks {
-  SVgroupInfo vg;
-  int32_t     numOfTables;  // number of tables in current submit block
-  uint32_t    size;
-  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
-} SVgDataBlocks;
-
-typedef struct SVnodeModifOpStmtInfo {
-  int16_t     nodeType;
-  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
-  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
-  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
-  const char* sql;                 // current sql statement position
-} SVnodeModifOpStmtInfo;
-
-typedef struct SDclStmtInfo {
-  int16_t     nodeType;
-  int16_t     msgType;
-  SEpSet      epSet;
-  char*       pMsg;
-  int32_t     msgLen;
-  void*       pExtension;  // todo remove it soon
-} SDclStmtInfo;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_PARSENODES_H_*/

include/libs/parser/parser.h

@@ -20,79 +20,48 @@
 extern "C" {
 #endif
 
-#include "parsenodes.h"
+#include "querynodes.h"
 
 typedef struct SParseContext {
   uint64_t         requestId;
   int32_t          acctId;
   const char      *db;
+  bool             topicQuery;
   void            *pTransporter;
   SEpSet           mgmtEpSet;
   const char      *pSql;           // sql string
   size_t           sqlLen;         // length of the sql string
   char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
   int32_t          msgLen;         // max length of the msg
-
   struct SCatalog *pCatalog;
 } SParseContext;
 
-/**
+typedef struct SCmdMsgInfo {
- * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST.
+  int16_t msgType;
- * @param pSql     sql string
+  SEpSet epSet;
- * @param length   length of the sql string
+  void* pMsg;
- * @param id       operator id, generated by uuid generator
+  int32_t msgLen;
- * @param msg      extended error message if exists.
+  void* pExtension;  // todo remove it soon
- * @return         error code
+} SCmdMsgInfo;
- */
-int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQueryNode);
 
-/**
+typedef struct SQuery {
- * Return true if it is a ddl/dcl sql statement
+  bool directRpc;
- * @param pQuery
+  bool haveResultSet;
- * @return
+  SNode* pRoot;
- */
+  int32_t numOfResCols;
-bool qIsDdlQuery(const SQueryNode* pQueryNode);
+  SSchema* pResSchema;
+  SCmdMsgInfo* pCmdMsg;
+  int32_t msgType;
+  SArray* pDbList;
+  SArray* pTableList;
+  bool showRewrite;
+} SQuery;
 
-/**
+int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
- * Destroy logic query plan
- * @param pQueryNode
- */
-void qDestroyQuery(SQueryNode* pQueryNode);
 
-/**
+void qDestroyQuery(SQuery* pQueryNode);
- * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that
- * involved in the subscribe procedure.
- * @param pSql
- * @param length
- * @param pConvertSql
- * @return
- */
-int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql);
 
-void assignExprInfo(SExprInfo* dst, const SExprInfo* src);
+int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
-void columnListCopy(SArray* dst, const SArray* src, uint64_t uid);
-void columnListDestroy(SArray* pColumnList);
-
-void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel);
-void dropOneLevelExprInfo(SArray* pExprInfo);
-
-typedef struct SSourceParam {
-  SArray    *pExprNodeList; //Array<struct tExprNode*>
-  SArray    *pColumnList;   //Array<struct SColumn>
-  int32_t    num;
-} SSourceParam;
-
-SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize);
-int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
-int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
-int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo);
-
-STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex);
-SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex);
-
-int32_t getNewResColId();
-void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn);
-SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema);
 
 #ifdef __cplusplus
 }

include/libs/planner/planner.h

@@ -20,192 +20,35 @@
 extern "C" {
 #endif
 
-#include "query.h"
+#include "plannodes.h"
-#include "tmsg.h"
-#include "tarray.h"
-#include "trpc.h"
 
-#define QUERY_TYPE_MERGE       1
+typedef struct SPlanContext {
-#define QUERY_TYPE_PARTIAL     2
-#define QUERY_TYPE_SCAN        3
-#define QUERY_TYPE_MODIFY      4
-
-enum OPERATOR_TYPE_E {
-  OP_Unknown,
-#define INCLUDE_AS_ENUM
-#include "plannerOp.h"
-#undef INCLUDE_AS_ENUM
-  OP_TotalNum
-};
-
-enum DATASINK_TYPE_E {
-  DSINK_Unknown,
-  DSINK_Dispatch,
-  DSINK_Insert,
-  DSINK_TotalNum
-};
-
-struct SEpSet;
-struct SQueryStmtInfo;
-
-typedef SSchema SSlotSchema;
-
-typedef struct SDataBlockSchema {
-  SSlotSchema *pSchema;
-  int32_t      numOfCols;    // number of columns
-  int32_t      resultRowSize;
-  int16_t      precision;
-} SDataBlockSchema;
-
-typedef struct SQueryNodeBasicInfo {
-  int32_t     type;          // operator type
-  const char *name;          // operator name
-} SQueryNodeBasicInfo;
-
-typedef struct SDataSink {
-  SQueryNodeBasicInfo info;
-  SDataBlockSchema schema;
-} SDataSink;
-
-typedef struct SDataDispatcher {
-  SDataSink sink;
-} SDataDispatcher;
-
-typedef struct SDataInserter {
-  SDataSink sink;
-  int32_t   numOfTables;
-  uint32_t  size;
-  char     *pData;
-} SDataInserter;
-
-typedef struct SPhyNode {
-  SQueryNodeBasicInfo info;
-  SArray             *pTargets;      // target list to be computed or scanned at this node
-  SArray             *pConditions;   // implicitly-ANDed qual conditions
-  SDataBlockSchema    targetSchema;
-  // children plan to generated result for current node to process
-  // in case of join, multiple plan nodes exist.
-  SArray             *pChildren;
-  struct SPhyNode    *pParent;
-} SPhyNode;
-
-typedef struct SScanPhyNode {
-  SPhyNode    node;
-  uint64_t    uid;           // unique id of the table
-  int8_t      tableType;
-  int32_t     order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
-  int32_t     count;         // repeat count
-  int32_t     reverse;       // reverse scan count
-} SScanPhyNode;
-
-typedef SScanPhyNode SSystemTableScanPhyNode;
-typedef SScanPhyNode STagScanPhyNode;
-
-typedef struct STableScanPhyNode {
-  SScanPhyNode scan;
-  uint8_t      scanFlag;         // denotes reversed scan of data or not
-  STimeWindow  window;
-  SArray      *pTagsConditions; // implicitly-ANDed tag qual conditions
-} STableScanPhyNode;
-
-typedef STableScanPhyNode STableSeqScanPhyNode;
-
-typedef struct SProjectPhyNode {
-  SPhyNode node;
-} SProjectPhyNode;
-
-typedef struct SDownstreamSource {
-  SQueryNodeAddr addr;
-  uint64_t       taskId;
-  uint64_t       schedId;
-} SDownstreamSource;
-
-typedef struct SExchangePhyNode {
-  SPhyNode    node;
-  uint64_t    srcTemplateId;  // template id of datasource suplans
-  SArray     *pSrcEndPoints;  // SArray<SDownstreamSource>, scheduler fill by calling qSetSuplanExecutionNode
-} SExchangePhyNode;
-
-typedef enum EAggAlgo {
-	AGG_ALGO_PLAIN = 1, // simple agg across all input rows
-	AGG_ALGO_SORTED,    // grouped agg, input must be sorted
-	AGG_ALGO_HASHED     // grouped agg, use internal hashtable
-} EAggAlgo;
-
-typedef enum EAggSplit {
-	AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result
-	AGG_SPLIT_FINAL    // second level agg, must calculate the final result
-} EAggSplit;
-
-typedef struct SAggPhyNode {
-  SPhyNode    node;
-  EAggAlgo    aggAlgo;  // algorithm used by agg operator 
-  EAggSplit   aggSplit; // distributed splitting mode
-  SArray     *pExprs;   // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function
-  SArray     *pGroupByList; // SColIndex list, but these must be column node
-} SAggPhyNode;
-
-typedef struct SSubplanId {
   uint64_t queryId;
-  uint64_t templateId;
+  int32_t acctId;
-  uint64_t subplanId;
+  SEpSet mgmtEpSet;
-} SSubplanId;
+  SNode* pAstRoot;
+  bool topicQuery;
+  bool streamQuery;
+  bool showRewrite;
+} SPlanContext;
 
-typedef struct SSubplan {
+// Create the physical plan for the query, according to the AST.
-  SSubplanId id;           // unique id of the subplan
+int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList);
-  int32_t    type;         // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN|QUERY_TYPE_MODIFY
-  int32_t    msgType;      // message type for subplan, used to denote the send message type to vnode.
-  int32_t    level;        // the execution level of current subplan, starting from 0 in a top-down manner.
-  SQueryNodeAddr     execNode;    // for the scan/modify subplan, the optional execution node
-  SArray    *pChildren;    // the datasource subplan,from which to fetch the result
-  SArray    *pParents;     // the data destination subplan, get data from current subplan
-  SPhyNode  *pNode;        // physical plan of current subplan
-  SDataSink *pDataSink;    // data of the subplan flow into the datasink
-} SSubplan;
-
-typedef struct SQueryDag {
-  uint64_t queryId;
-  int32_t  numOfSubplans;
-  SArray  *pSubplans; // SArray*<SArray*<SSubplan*>>. The execution level of subplan, starting from 0.
-} SQueryDag;
-
-struct SQueryNode;
-
- /**
-  * Create the physical plan for the query, according to the AST.
-  * @param pQueryInfo
-  * @param pDag
-  * @param requestId
-  * @return
-  */
- int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId);
 
 // Set datasource of this subplan, multiple calls may be made to a subplan.
-// @subplan subplan to be schedule
+// @pSubplan subplan to be schedule
-// @templateId templateId of a group of datasource subplans of this @subplan
+// @groupId id of a group of datasource subplans of this @pSubplan
-// @ep one execution location of this group of datasource subplans 
+// @pSource one execution location of this group of datasource subplans 
-void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource);
+int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str);
+// Convert to subplan to string for the scheduler to send to the executor
+int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
+int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);
 
-/**
+char* qQueryPlanToString(const SQueryPlan* pPlan);
- * Convert to subplan to string for the scheduler to send to the executor
+SQueryPlan* qStringToQueryPlan(const char* pStr);
- */
-int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len);
 
-int32_t qStringToSubplan(const char* str, SSubplan** subplan);
+void qDestroyQueryPlan(SQueryPlan* pPlan);
-
-void qDestroySubplan(SSubplan* pSubplan);
-
-/**
- * Destroy the physical plan.
- * @param pQueryPhyNode
- * @return
- */
-void qDestroyQueryDag(SQueryDag* pDag);
-
-char* qDagToString(const SQueryDag* pDag);
-SQueryDag* qStringToDag(const char* pStr);
 
 #ifdef __cplusplus
 }

include/libs/planner/plannerOp.h

@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#if defined(INCLUDE_AS_ENUM)  // enum define mode
-  #undef OP_ENUM_MACRO
-  #define OP_ENUM_MACRO(op)    OP_##op,
-#elif defined(INCLUDE_AS_NAME)  // comment define mode
-  #undef OP_ENUM_MACRO
-  #define OP_ENUM_MACRO(op)    #op,
-#else
-  #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME
-#endif
-
-OP_ENUM_MACRO(StreamScan)
-OP_ENUM_MACRO(TableScan)
-OP_ENUM_MACRO(TableSeqScan)
-OP_ENUM_MACRO(TagScan)
-OP_ENUM_MACRO(SystemTableScan)
-OP_ENUM_MACRO(StreamBlockScan)
-OP_ENUM_MACRO(Aggregate)
-OP_ENUM_MACRO(Project)
-// OP_ENUM_MACRO(Groupby)
-OP_ENUM_MACRO(Limit)
-OP_ENUM_MACRO(SLimit)
-OP_ENUM_MACRO(TimeWindow)
-OP_ENUM_MACRO(SessionWindow)
-OP_ENUM_MACRO(StateWindow)
-OP_ENUM_MACRO(Fill)
-OP_ENUM_MACRO(MultiTableAggregate)
-OP_ENUM_MACRO(MultiTableTimeInterval)
-OP_ENUM_MACRO(Filter)
-OP_ENUM_MACRO(Distinct)
-OP_ENUM_MACRO(Join)
-OP_ENUM_MACRO(AllTimeWindow)
-OP_ENUM_MACRO(AllMultiTableTimeInterval)
-OP_ENUM_MACRO(Order)
-OP_ENUM_MACRO(Exchange)
-
-//OP_ENUM_MACRO(TableScan)

include/libs/qcom/query.h

@@ -25,7 +25,7 @@ extern "C" {
 #include "tlog.h"
 #include "tmsg.h"
 
-enum {
+typedef enum {
   JOB_TASK_STATUS_NULL = 0,
   JOB_TASK_STATUS_NOT_START = 1,
   JOB_TASK_STATUS_EXECUTING,
@@ -35,21 +35,24 @@ enum {
   JOB_TASK_STATUS_CANCELLING,
   JOB_TASK_STATUS_CANCELLED,
   JOB_TASK_STATUS_DROPPING,
-  JOB_TASK_STATUS_FREEING,
+} EJobTaskType;
-};
 
-enum {
+typedef enum {
   TASK_TYPE_PERSISTENT = 1,
   TASK_TYPE_TEMP,
-};
+} ETaskType;
 
 typedef struct STableComInfo {
   uint8_t  numOfTags;     // the number of tags in schema
   uint8_t  precision;     // the number of precision
-  int16_t numOfColumns;   // the number of columns
+  col_id_t numOfColumns;  // the number of columns
   int32_t  rowSize;       // row size of the schema
 } STableComInfo;
 
+typedef struct SIndexMeta {
+
+} SIndexMeta;
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -63,7 +66,8 @@ typedef struct SCTableMeta {
 } SCTableMeta;
 
 /*
- * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
+ * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a
+ * SCTableMeta.
  */
 typedef struct STableMeta {
   // BEGIN: KEEP THIS PART SAME WITH SCTableMeta
@@ -73,7 +77,8 @@ typedef struct STableMeta {
   uint64_t suid;
   // END: KEEP THIS PART SAME WITH SCTableMeta
 
-  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
+  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
+  // info
   int16_t       sversion;
   int16_t       tversion;
   STableComInfo tableInfo;
@@ -83,6 +88,7 @@ typedef struct STableMeta {
 typedef struct SDBVgInfo {
   int32_t   vgVersion;
   int8_t    hashMethod;
+  int32_t   numOfTable;  // DB's table num, unit is TSDB_TABLE_NUM_UNIT
   SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
 } SDBVgInfo;
 
@@ -92,13 +98,7 @@ typedef struct SUseDbOutput {
   SDBVgInfo* dbVgroup;
 } SUseDbOutput;
 
-enum {
+enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE };
-  META_TYPE_NULL_TABLE = 1,
-  META_TYPE_CTABLE,
-  META_TYPE_TABLE,
-  META_TYPE_BOTH_TABLE
-};
-
 
 typedef struct STableMetaOutput {
   int32_t     metaType;
@@ -130,9 +130,14 @@ typedef struct SMsgSendInfo {
 
 typedef struct SQueryNodeAddr {
   int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epset;
+  SEpSet  epSet;
 } SQueryNodeAddr;
 
+
+typedef struct SQueryNodeStat {
+  int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
+} SQueryNodeStat;
+
 int32_t initTaskQueue();
 int32_t cleanupTaskQueue();
 
@@ -145,6 +150,8 @@ int32_t cleanupTaskQueue();
  */
 int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
 
+int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx);
+
 /**
  * Asynchronously send message to server, after the response received, the callback will be incured.
  *
@@ -156,30 +163,77 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
  */
 int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
 
+int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp);
+
 void initQueryModuleMsgHandle();
 
 const SSchema* tGetTbnameColumnSchema();
 bool           tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
 int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
+char *jobTaskStatusStr(int32_t status);
+
+SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
 
 extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen);
 extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
 
-
 #define SET_META_TYPE_NULL(t)       (t) = META_TYPE_NULL_TABLE
 #define SET_META_TYPE_CTABLE(t)     (t) = META_TYPE_CTABLE
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define qFatal(...)  do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
-#define qError(...)  do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
-#define qWarn(...)   do { if (qDebugFlag & DEBUG_WARN)  { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }}  while(0)
+#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
-#define qInfo(...)   do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define qDebug(...)  do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
-#define qTrace(...)  do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
-#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
 
+#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
+
+#define REQUEST_MAX_TRY_TIMES 5
+
+#define qFatal(...)                                                     \
+  do {                                                                  \
+    if (qDebugFlag & DEBUG_FATAL) {                                     \
+      taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
+    }                                                                   \
+  } while (0)
+#define qError(...)                                                     \
+  do {                                                                  \
+    if (qDebugFlag & DEBUG_ERROR) {                                     \
+      taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
+    }                                                                   \
+  } while (0)
+#define qWarn(...)                                                    \
+  do {                                                                \
+    if (qDebugFlag & DEBUG_WARN) {                                    \
+      taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
+    }                                                                 \
+  } while (0)
+#define qInfo(...)                                               \
+  do {                                                           \
+    if (qDebugFlag & DEBUG_INFO) {                               \
+      taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
+    }                                                            \
+  } while (0)
+#define qDebug(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_DEBUG) {                               \
+      taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qTrace(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_TRACE) {                               \
+      taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qDebugL(...)                                                     \
+  do {                                                                   \
+    if (qDebugFlag & DEBUG_DEBUG) {                                      \
+      taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                                    \
+  } while (0)
 
 #ifdef __cplusplus
 }

include/libs/qworker/qworker.h

@@ -20,6 +20,7 @@
 extern "C" {
 #endif
 
+#include "tmsgcb.h"
 #include "trpc.h"
 
 
@@ -27,6 +28,7 @@ enum {
   NODE_TYPE_VNODE = 1,
   NODE_TYPE_QNODE,
   NODE_TYPE_SNODE,
+  NODE_TYPE_MNODE,
 };
 
 
@@ -48,11 +50,7 @@ typedef struct {
   uint64_t numOfErrors;
 } SQWorkerStat;
 
-typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
+int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb);
-typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *);
-
-int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
-                    putReqToQueryQFp fp1, sendReqToDnodeFp fp2);
 
 int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
@@ -72,6 +70,8 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
 int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
+int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+
 int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
 int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);

include/libs/scalar/filter.h

@@ -19,13 +19,32 @@
 extern "C" {
 #endif
 
+#include "tcommon.h"
+#include "nodes.h"
+
 typedef struct SFilterInfo SFilterInfo;
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+
+enum {
+  FLT_OPTION_NO_REWRITE = 1,
+  FLT_OPTION_TIMESTAMP = 2,
+  FLT_OPTION_NEED_UNIQE = 4,
+};
 
 typedef struct SFilterColumnParam{
   int32_t numOfCols;
   SArray* pDataBlock;
 } SFilterColumnParam;
 
+extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
+extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
+extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
+extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param);
+extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
+extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
+extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
+extern void filterFreeInfo(SFilterInfo *info);
+extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
 
 #ifdef __cplusplus
 }

include/libs/scalar/scalar.h

@@ -25,14 +25,38 @@ extern "C" {
 
 typedef struct SFilterInfo SFilterInfo;
 
-
+/*
+pNode will be freed in API;
+*pRes need to freed in caller
+*/
 int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
-int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
+
+/* 
+pDst need to freed in caller 
+*/
+int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst);
+
 int32_t scalarGetOperatorParamNum(EOperatorType type);
+int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
 
 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
 int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);
 
+int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
+int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
+int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }

include/libs/scheduler/scheduler.h

@@ -23,10 +23,9 @@ extern "C" {
 #include "catalog.h"
 #include "planner.h"
 
-struct SSchJob;
-
 typedef struct SSchedulerCfg {
   uint32_t maxJobNum;
+  int32_t  maxNodeTableNum;
 } SSchedulerCfg;
 
 typedef struct SQueryProfileSummary {
@@ -72,7 +71,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
  * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes);
+int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, int64_t* pJob, const char* sql, SQueryResult *pRes);
 
 /**
  * Process the query job, generated according to the query physical plan.
@@ -80,7 +79,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str
  * @param pNodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob);
+int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan* pDag, const char* sql, int64_t *pJob);
 
 /**
  * Fetch query result from the remote query executor
@@ -88,7 +87,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDa
  * @param data
  * @return
  */
-int32_t schedulerFetchRows(struct SSchJob *pJob, void **data);
+int32_t schedulerFetchRows(int64_t job, void **data);
 
 
 /**
@@ -102,7 +101,7 @@ int32_t schedulerFetchRows(struct SSchJob *pJob, void **data);
  * Free the query job
  * @param pJob
  */
-void schedulerFreeJob(void *pJob);
+void schedulerFreeJob(int64_t job);
 
 void schedulerDestroy(void);
 
@@ -112,7 +111,7 @@ void schedulerDestroy(void);
  * @param pTasks SArray**<STaskInfo>
  * @return
  */
-int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks);
+int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks);
 
 /**
  * make one task info's multiple copies

include/libs/stream/tstream.h

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdatablock.h"
+#include "tmsg.h"
+#include "tmsgcb.h"
+#include "trpc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef _TSTREAM_H_
+#define _TSTREAM_H_
+
+enum {
+  STREAM_TASK_STATUS__RUNNING = 1,
+  STREAM_TASK_STATUS__STOP,
+};
+
+#if 0
+// pipe  -> fetch/pipe queue
+// merge -> merge      queue
+// write -> write      queue
+enum {
+  TASK_DISPATCH_MSG__SND_PIPE = 1,
+  TASK_DISPATCH_MSG__SND_MERGE,
+  TASK_DISPATCH_MSG__VND_PIPE,
+  TASK_DISPATCH_MSG__VND_MERGE,
+  TASK_DISPATCH_MSG__VND_WRITE,
+};
+#endif
+
+typedef struct {
+  int32_t nodeId;  // 0 for snode
+  SEpSet  epSet;
+} SStreamTaskEp;
+
+typedef struct {
+  void* inputHandle;
+  void* executor;
+} SStreamRunner;
+
+typedef struct {
+  int8_t parallelizable;
+  char*  qmsg;
+  // followings are not applicable to encoder and decoder
+  int8_t         numOfRunners;
+  SStreamRunner* runners;
+} STaskExec;
+
+typedef struct {
+  int8_t reserved;
+} STaskDispatcherInplace;
+
+typedef struct {
+  int32_t nodeId;
+  SEpSet  epSet;
+} STaskDispatcherFixedEp;
+
+typedef struct {
+  int8_t  hashMethod;
+  SArray* info;
+} STaskDispatcherShuffle;
+
+typedef struct {
+  int8_t reserved;
+  // not applicable to encoder and decoder
+  SHashObj* pHash;  // groupId to tbuid
+} STaskSinkTb;
+
+typedef struct {
+  int8_t reserved;
+} STaskSinkSma;
+
+typedef struct {
+  int8_t reserved;
+} STaskSinkFetch;
+
+typedef struct {
+  int8_t reserved;
+} STaskSinkShow;
+
+enum {
+  TASK_SOURCE__SCAN = 1,
+  TASK_SOURCE__PIPE,
+  TASK_SOURCE__MERGE,
+};
+
+enum {
+  TASK_EXEC__NONE = 1,
+  TASK_EXEC__PIPE,
+  TASK_EXEC__MERGE,
+};
+
+enum {
+  TASK_DISPATCH__NONE = 1,
+  TASK_DISPATCH__INPLACE,
+  TASK_DISPATCH__FIXED,
+  TASK_DISPATCH__SHUFFLE,
+};
+
+enum {
+  TASK_SINK__NONE = 1,
+  TASK_SINK__TABLE,
+  TASK_SINK__SMA,
+  TASK_SINK__FETCH,
+  TASK_SINK__SHOW,
+};
+
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int8_t  status;
+
+  int8_t  sourceType;
+  int8_t  execType;
+  int8_t  sinkType;
+  int8_t  dispatchType;
+  int16_t dispatchMsgType;
+  int32_t downstreamTaskId;
+
+  int32_t nodeId;
+  SEpSet  epSet;
+
+  // source preprocess
+
+  // exec
+  STaskExec exec;
+
+  // local sink
+  union {
+    STaskSinkTb    tbSink;
+    STaskSinkSma   smaSink;
+    STaskSinkFetch fetchSink;
+    STaskSinkShow  showSink;
+  };
+
+  // dispatch
+  union {
+    STaskDispatcherInplace inplaceDispatcher;
+    STaskDispatcherFixedEp fixedEpDispatcher;
+    STaskDispatcherShuffle shuffleDispatcher;
+  };
+
+  // state storage
+
+} SStreamTask;
+
+SStreamTask* tNewSStreamTask(int64_t streamId);
+int32_t      tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask);
+int32_t      tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask);
+void         tFreeSStreamTask(SStreamTask* pTask);
+
+typedef struct {
+  // SMsgHead     head;
+  SStreamTask* task;
+} SStreamTaskDeployReq;
+
+typedef struct {
+  int32_t reserved;
+} SStreamTaskDeployRsp;
+
+typedef struct {
+  // SMsgHead head;
+  int64_t streamId;
+  int32_t taskId;
+  SArray* data;  // SArray<SSDataBlock>
+} SStreamTaskExecReq;
+
+int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq);
+void*   tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq);
+void    tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq);
+
+typedef struct {
+  int32_t reserved;
+} SStreamTaskExecRsp;
+
+typedef struct {
+  // SMsgHead head;
+  int64_t streamId;
+  int64_t version;
+  SArray* res;  // SArray<SSDataBlock>
+} SStreamSinkReq;
+
+int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef _TSTREAM_H_ */

include/libs/sync/sync.h

@@ -21,7 +21,10 @@ extern "C" {
 #endif
 
 #include <stdint.h>
+#include <tdatablock.h>
 #include "taosdef.h"
+#include "trpc.h"
+#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -29,28 +32,29 @@ typedef int64_t  SyncIndex;
 typedef uint64_t SyncTerm;
 
 typedef enum {
-  TAOS_SYNC_STATE_FOLLOWER = 0,
+  TAOS_SYNC_STATE_FOLLOWER = 100,
-  TAOS_SYNC_STATE_CANDIDATE = 1,
+  TAOS_SYNC_STATE_CANDIDATE = 101,
-  TAOS_SYNC_STATE_LEADER = 2,
+  TAOS_SYNC_STATE_LEADER = 102,
+  TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct {
+typedef struct SSyncBuffer {
   void*  data;
   size_t len;
 } SSyncBuffer;
 
-typedef struct {
+typedef struct SNodeInfo {
-  SyncNodeId nodeId;
   uint16_t nodePort;                 // node sync Port
   char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
 } SNodeInfo;
 
-typedef struct {
+typedef struct SSyncCfg {
   int32_t   replicaNum;
+  int32_t   myIndex;
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct {
+typedef struct SNodesRole {
   int32_t    replicaNum;
   SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
   ESyncState role[TSDB_MAX_REPLICA];
@@ -65,17 +69,20 @@ typedef struct SSnapshot {
 typedef struct SSyncFSM {
   void* data;
 
-  // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result
+  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
   // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                     ESyncState state);
 
-  // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result
+  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
   // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                        ESyncState state);
 
   // when log entry is updated by a new one, FpRollBackCb is called
   // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SSyncBuffer* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                       ESyncState state);
 
   // user should implement this function, use "data" to take snapshot into "snapshot"
   int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
@@ -85,25 +92,22 @@ typedef struct SSyncFSM {
 
 } SSyncFSM;
 
+struct SSyncRaftEntry;
+typedef struct SSyncRaftEntry SSyncRaftEntry;
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
   void* data;
 
   // append one log entry
-  int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf);
+  int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
 
-  // get one log entry, user need to free pBuf->data
+  // get one log entry, user need to free pEntry->pCont
-  int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf);
+  SSyncRaftEntry* (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index);
 
-  // update log store commit index with "index"
+  // truncate log with index, entries after the given index (>=index) will be deleted
-  int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
+  int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
-
-  // truncate log with index, entries after the given index (>index) will be deleted
-  int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index);
-
-  // return commit index of log
-  SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
 
   // return index of last entry
   SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore);
@@ -111,6 +115,12 @@ typedef struct SSyncLogStore {
   // return term of last entry
   SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore);
 
+  // update log store commit index with "index"
+  int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
+
+  // return commit index of log
+  SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
+
 } SSyncLogStore;
 
 // raft need to persist two variables in storage: currentTerm, voteFor
@@ -128,12 +138,17 @@ typedef struct SStateMgr {
 
 } SStateMgr;
 
-typedef struct {
+typedef struct SSyncInfo {
   SyncGroupId vgId;
   SSyncCfg    syncCfg;
-  SSyncLogStore logStore;
+  char        path[TSDB_FILENAME_LEN];
-  SStateMgr     stateManager;
+  SWal*       pWal;
-  SSyncFSM      syncFsm;
+  SSyncFSM*   pFsm;
+
+  void* rpcClient;
+  int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
+  void* queue;
+  int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
 
 } SSyncInfo;
 
@@ -146,12 +161,14 @@ void    syncCleanUp();
 int64_t    syncStart(const SSyncInfo* pSyncInfo);
 void       syncStop(int64_t rid);
 int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-
+int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
-int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
-
 ESyncState syncGetMyRole(int64_t rid);
-void       syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
+
+// propose with sequence number, to implement linearizable semantics
+int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
 extern int32_t sDebugFlag;
 

include/libs/tfs/tfs.h

@@ -16,7 +16,8 @@
 #ifndef _TD_TFS_H_
 #define _TD_TFS_H_
 
-#include "tcfg.h"
+#include "tdef.h"
+#include "monitor.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -197,6 +198,16 @@ void tfsBasename(const STfsFile *pFile, char *dest);
  */
 void tfsDirname(const STfsFile *pFile, char *dest);
 
+/**
+ * @brief Get the absolute file name of rname.
+ *
+ * @param pTfs
+ * @param diskId
+ * @param rname relative file name
+ * @param aname absolute file name
+ */
+void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname);
+
 /**
  * @brief Remove file in tfs.
  *
@@ -237,6 +248,14 @@ const STfsFile *tfsReaddir(STfsDir *pDir);
  */
 void tfsClosedir(STfsDir *pDir);
 
+/**
+ * @brief Get disk info of tfs.
+ *
+ * @param pTfs The fs object.
+ * @param pInfo The info object.
+ */
+int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/transport/thttp.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_HTTP_H_
+#define _TD_UTIL_HTTP_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
+
+int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_UTIL_H_*/

include/libs/transport/trpc.h

@@ -29,8 +29,6 @@ extern "C" {
 
 extern int tsRpcHeadSize;
 
-typedef struct SRpcPush SRpcPush;
-
 typedef struct SRpcConnInfo {
   uint32_t clientIp;
   uint16_t clientPort;
@@ -45,16 +43,18 @@ typedef struct SRpcMsg {
   int32_t code;
   void *  handle;         // rpc handle returned to app
   void *  ahandle;        // app handle set by client
-  int     persist;  // keep handle or not, default 0
+  int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
-
+  int     persistHandle;  // persist handle or not
-  SRpcPush *push;
 
 } SRpcMsg;
 
-typedef struct SRpcPush {
+typedef struct {
-  void *arg;
+  char    user[TSDB_USER_LEN];
-  int (*callback)(void *arg, SRpcMsg *rpcMsg);
+  SRpcMsg rpcMsg;
-} SRpcPush;
+  int32_t rspLen;
+  void *  pRsp;
+  void *  pNode;
+} SNodeMsg;
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
@@ -64,7 +64,6 @@ typedef struct SRpcInit {
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int      idleTime;      // milliseconds, 0 means idle timer is disabled
 
-  bool noPool;  //  create conn pool or not
   // the following is for client app ecurity only
   char *user;     // user name
   char  spi;      // security parameter index
@@ -81,6 +80,24 @@ typedef struct SRpcInit {
   void *parent;
 } SRpcInit;
 
+typedef struct {
+  void     *val;
+  int32_t (*clone)(void *src, void **dst);
+  void    (*freeFunc)(const void *arg);
+} SRpcCtxVal;
+
+typedef struct {
+  int32_t   msgType;
+  void     *val;
+  int32_t (*clone)(void *src, void **dst);
+  void    (*freeFunc)(const void *arg);
+} SRpcBrokenlinkVal;
+
+typedef struct {
+  SHashObj *        args;
+  SRpcBrokenlinkVal brokenVal;
+} SRpcCtx;
+
 int32_t rpcInit();
 void    rpcCleanup();
 void *  rpcOpen(const SRpcInit *pRpc);
@@ -89,12 +106,19 @@ void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
 void *  rpcReallocCont(void *ptr, int contLen);
 void    rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
+void    rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
+
 void rpcSendResponse(const SRpcMsg *pMsg);
 void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
 int  rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
 int  rpcReportProgress(void *pConn, char *pCont, int contLen);
 void rpcCancelRequest(int64_t rid);
+void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
+// just release client conn to rpc instance, no close sock
+void rpcReleaseHandle(void *handle, int8_t type);  //
+void rpcRefHandle(void *handle, int8_t type);
+void rpcUnrefHandle(void *handle, int8_t type);
 
 #ifdef __cplusplus
 }

include/libs/wal/wal.h

@@ -24,42 +24,40 @@
 extern "C" {
 #endif
 
-extern int32_t wDebugFlag;
-
 #define wFatal(...)                                              \
   {                                                              \
     if (wDebugFlag & DEBUG_FATAL) {                              \
-      taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
     }                                                            \
   }
 #define wError(...)                                              \
   {                                                              \
     if (wDebugFlag & DEBUG_ERROR) {                              \
-      taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
     }                                                            \
   }
 #define wWarn(...)                                             \
   {                                                            \
     if (wDebugFlag & DEBUG_WARN) {                             \
-      taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
     }                                                          \
   }
 #define wInfo(...)                                        \
   {                                                       \
     if (wDebugFlag & DEBUG_INFO) {                        \
-      taosPrintLog("WAL ", 255, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
     }                                                     \
   }
 #define wDebug(...)                                               \
   {                                                               \
     if (wDebugFlag & DEBUG_DEBUG) {                               \
-      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
     }                                                             \
   }
 #define wTrace(...)                                               \
   {                                                               \
     if (wDebugFlag & DEBUG_TRACE) {                               \
-      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+      taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
     }                                                             \
   }
 
@@ -77,15 +75,30 @@ extern int32_t wDebugFlag;
 #define WAL_CUR_FAILED 1
 
 #pragma pack(push, 1)
-typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType;
+typedef enum {
+  TAOS_WAL_NOLOG = 0,
+  TAOS_WAL_WRITE = 1,
+  TAOS_WAL_FSYNC = 2,
+} EWalType;
+
+// used by sync module
+typedef struct {
+  int8_t   isWeek;
+  uint64_t seqNum;
+  uint64_t term;
+} SSyncLogMeta;
 
 typedef struct SWalReadHead {
   int8_t  headVer;
-  int16_t msgType;
   int8_t  reserved;
+  int16_t msgType;
   int32_t len;
   int64_t ingestTs;  // not implemented
   int64_t version;
+
+  // sync meta
+  SSyncLogMeta syncMeta;
+
   char body[];
 } SWalReadHead;
 
@@ -120,8 +133,8 @@ typedef struct SWal {
   int32_t fsyncSeq;
   // meta
   SWalVer   vers;
-  int64_t writeLogTfd;
+  TdFilePtr pWriteLogTFile;
-  int64_t writeIdxTfd;
+  TdFilePtr pWriteIdxTFile;
   int32_t   writeCur;
   SArray   *fileInfoSet;
   // status
@@ -129,7 +142,7 @@ typedef struct SWal {
   int64_t lastRollSeq;
   // ctl
   int64_t       refId;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
   // path
   char path[WAL_PATH_LEN];
   // reusable write head
@@ -138,8 +151,8 @@ typedef struct SWal {
 
 typedef struct SWalReadHandle {
   SWal     *pWal;
-  int64_t   readLogTfd;
+  TdFilePtr pReadLogTFile;
-  int64_t   readIdxTfd;
+  TdFilePtr pReadIdxTFile;
   int64_t   curFileFirstVer;
   int64_t   curVersion;
   int64_t   capacity;
@@ -160,6 +173,8 @@ int32_t walAlter(SWal *, SWalCfg *pCfg);
 void    walClose(SWal *);
 
 // write
+int64_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
+                             int32_t bodyLen);
 int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
 void    walFsync(SWal *, bool force);
 

include/os/os.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_OS_H
+#ifndef _TD_OS_H_
-#define TDENGINE_OS_H
+#define _TD_OS_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -22,7 +22,35 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
+#include <semaphore.h>
+
+#include <regex.h>
+
+#if !defined(WINDOWS)
+#include <unistd.h>
 #include <dirent.h>
+#include <sched.h>
+#include <wordexp.h>
+#include <libgen.h>
+
+#include <sys/utsname.h>
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <termios.h>
+#include <sys/statvfs.h>
+
+#if defined(DARWIN)
+#else
+#include <sys/prctl.h>
+#include <argp.h>
+#endif
+
+#endif
+
 #include <errno.h>
 #include <fcntl.h>
 #include <float.h>
@@ -30,8 +58,6 @@ extern "C" {
 #include <limits.h>
 #include <locale.h>
 #include <math.h>
-#include <regex.h>
-#include <sched.h>
 #include <setjmp.h>
 #include <signal.h>
 #include <stdarg.h>
@@ -41,28 +67,21 @@ extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/utsname.h>
-#include <unistd.h>
 #include <wchar.h>
 #include <wctype.h>
-#include <wordexp.h>
-#include <libgen.h>
 
-#include <sys/mman.h>
-#include <sys/prctl.h>
 
 #include "osAtomic.h"
 #include "osDef.h"
 #include "osDir.h"
 #include "osEndian.h"
-#include "osEnv.h"
 #include "osFile.h"
+#include "osLocale.h"
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"
 #include "osRand.h"
+#include "osThread.h"
 #include "osSemaphore.h"
 #include "osSignal.h"
 #include "osSleep.h"
@@ -70,14 +89,15 @@ extern "C" {
 #include "osString.h"
 #include "osSysinfo.h"
 #include "osSystem.h"
-#include "osThread.h"
 #include "osTime.h"
 #include "osTimer.h"
+#include "osTimezone.h"
+#include "osEnv.h"
 
-void osInit();
+void osDefaultInit();
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_OS_H_*/

include/os/osAtomic.h

@@ -20,336 +20,95 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+// If the error is in a third-party library, place this header file under the third-party library header file.
-  #define atomic_load_8(ptr) (*(char volatile*)(ptr))
+// When you want to use this feature, you should find or add the same function in the following section.
-  #define atomic_load_16(ptr) (*(short volatile*)(ptr))
+#ifndef ALLOW_FORBID_FUNC
-  #define atomic_load_32(ptr) (*(long volatile*)(ptr))
+  #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
-  #define atomic_load_64(ptr) (*(__int64 volatile*)(ptr))
+  #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
-  #define atomic_load_ptr(ptr) (*(void* volatile*)(ptr))
+  #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
-
+  #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
-  #define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val))
+  #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
-  #define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val))
+  #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
-  #define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val))
+  #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
-  #define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val))
+  #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
-  #define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val))
+  #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
-
+  #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
-  #define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val))
+  #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
-  #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val))
+  #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
-  #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val))
+  #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
-  #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val))
+  #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
-  #ifdef _WIN64 
-    #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) 
-  #else
-    #define atomic_exchange_ptr(ptr, val) _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
 #endif
 
-  #ifdef _TD_GO_DLL_
+int8_t atomic_load_8(int8_t volatile *ptr);
-    #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
+int16_t atomic_load_16(int16_t volatile *ptr);
-  #else
+int32_t atomic_load_32(int32_t volatile *ptr);
-    #define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval))
+int64_t atomic_load_64(int64_t volatile *ptr);
-  #endif
+void* atomic_load_ptr(void *ptr);
-  #define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval))
+void atomic_store_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval))
+void atomic_store_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval))
+void atomic_store_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval))
+void atomic_store_64(int64_t volatile *ptr, int64_t val);
-
+void atomic_store_ptr(void *ptr, void *val);
-  char    interlocked_add_fetch_8(char volatile *ptr, char val);
+int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
-  short   interlocked_add_fetch_16(short volatile *ptr, short val);
+int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
-  long    interlocked_add_fetch_32(long volatile *ptr, long val);
+int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
-  __int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val);
+int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
-
+void* atomic_exchange_ptr(void *ptr, void *val);
-  char interlocked_and_fetch_8(char volatile* ptr, char val);
+int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
-  short interlocked_and_fetch_16(short volatile* ptr, short val);
+int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
-  long interlocked_and_fetch_32(long volatile* ptr, long val);
+int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval);
-  __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val);
+int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval);
-
+void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
-  __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val);
+int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
-
+int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
-  char interlocked_or_fetch_8(char volatile* ptr, char val);
+int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
-  short interlocked_or_fetch_16(short volatile* ptr, short val);
+int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-  long interlocked_or_fetch_32(long volatile* ptr, long val);
+void* atomic_add_fetch_ptr(void *ptr, int32_t val);
-  __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val);
+int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
-
+int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
-  char interlocked_xor_fetch_8(char volatile* ptr, char val);
+int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
-  short interlocked_xor_fetch_16(short volatile* ptr, short val);
+int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-  long interlocked_xor_fetch_32(long volatile* ptr, long val);
+void* atomic_fetch_add_ptr(void *ptr, int32_t val);
-  __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val);
+int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
-
+int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
-  __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val);
+int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
-
+int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-  #define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val))
+void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
-  #define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val))
+int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val))
+int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
+int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
-  #ifdef _TD_GO_DLL_
+int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-    #define atomic_fetch_add_8 __sync_fetch_and_ad
+void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
-    #define atomic_fetch_add_16 __sync_fetch_and_add
+int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
-  #else
+int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
-    #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val))
+int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);
-    #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val))
+int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val);
-  #endif  
+void* atomic_and_fetch_ptr(void *ptr, void *val);
-  #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val))
+int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val))
+int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val))
+int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val))
+int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val);
-  
+void* atomic_fetch_and_ptr(void *ptr, void *val);
-  #define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val))
+int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val))
+int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val))
+int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val))
+int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val);
-
+void* atomic_or_fetch_ptr(void *ptr, void *val);
-  #define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val))
+int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val))
+int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val))
+int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val))
+int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val);
-
+void* atomic_fetch_or_ptr(void *ptr, void *val);
-  #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val))
+int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val))
+int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val))
+int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
+int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val);
-
+void* atomic_xor_fetch_ptr(void *ptr, void *val);
-  #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val))
+int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
-  #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val))
+int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val);
-  #define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val))
+int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
-  #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val))
+int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
-
+void* atomic_fetch_xor_ptr(void *ptr, void *val);
-  #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val))
-  #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val))
-  #define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val))
-  #define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
-
-  #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val))
-  #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val))
-  #define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val))
-  #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val))
-
-  #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val))
-  #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val))
-  #define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val))
-  #define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val))
-
-  #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val))
-  #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val))
-  #define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val))
-  #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val))
-
-  #ifdef _WIN64
-    #define atomic_add_fetch_ptr atomic_add_fetch_64
-    #define atomic_fetch_add_ptr atomic_fetch_add_64
-    #define atomic_sub_fetch_ptr atomic_sub_fetch_64
-    #define atomic_fetch_sub_ptr atomic_fetch_sub_64
-    #define atomic_and_fetch_ptr atomic_and_fetch_64
-    #define atomic_fetch_and_ptr atomic_fetch_and_64
-    #define atomic_or_fetch_ptr  atomic_or_fetch_64
-    #define atomic_fetch_or_ptr  atomic_fetch_or_64
-    #define atomic_xor_fetch_ptr atomic_xor_fetch_64
-    #define atomic_fetch_xor_ptr atomic_fetch_xor_64
-  #else
-    #define atomic_add_fetch_ptr atomic_add_fetch_32
-    #define atomic_fetch_add_ptr atomic_fetch_add_32
-    #define atomic_sub_fetch_ptr atomic_sub_fetch_32
-    #define atomic_fetch_sub_ptr atomic_fetch_sub_32
-    #define atomic_and_fetch_ptr atomic_and_fetch_32
-    #define atomic_fetch_and_ptr atomic_fetch_and_32
-    #define atomic_or_fetch_ptr  atomic_or_fetch_32
-    #define atomic_fetch_or_ptr  atomic_fetch_or_32
-    #define atomic_xor_fetch_ptr atomic_xor_fetch_32
-    #define atomic_fetch_xor_ptr atomic_fetch_xor_32
-  #endif
-#elif defined(_TD_NINGSI_60)
-  /*
-  * type __sync_fetch_and_add (type *ptr, type value);
-  * type __sync_fetch_and_sub (type *ptr, type value);
-  * type __sync_fetch_and_or (type *ptr, type value);
-  * type __sync_fetch_and_and (type *ptr, type value);
-  * type __sync_fetch_and_xor (type *ptr, type value);
-  * type __sync_fetch_and_nand (type *ptr, type value);
-  * type __sync_add_and_fetch (type *ptr, type value);
-  * type __sync_sub_and_fetch (type *ptr, type value);
-  * type __sync_or_and_fetch (type *ptr, type value);
-  * type __sync_and_and_fetch (type *ptr, type value);
-  * type __sync_xor_and_fetch (type *ptr, type value);
-  * type __sync_nand_and_fetch (type *ptr, type value);
-  *
-  * bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...)
-  * type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...)
-  * */
-
-  #define atomic_load_8(ptr)   __sync_fetch_and_add((ptr), 0)
-  #define atomic_load_16(ptr)  __sync_fetch_and_add((ptr), 0)
-  #define atomic_load_32(ptr)  __sync_fetch_and_add((ptr), 0)
-  #define atomic_load_64(ptr)  __sync_fetch_and_add((ptr), 0)
-  #define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0)
-    
-  #define atomic_store_8(ptr, val)   (*(ptr)=(val))
-  #define atomic_store_16(ptr, val)  (*(ptr)=(val))
-  #define atomic_store_32(ptr, val)  (*(ptr)=(val))
-  #define atomic_store_64(ptr, val)  (*(ptr)=(val))
-  #define atomic_store_ptr(ptr, val) (*(ptr)=(val))
-
-  int8_t  atomic_exchange_8_impl(int8_t* ptr, int8_t val );
-  int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val );
-  int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val );
-  int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val );
-  void*   atomic_exchange_ptr_impl( void **ptr, void *val );
-
-  #define atomic_exchange_8(ptr, val)   atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val)
-  #define atomic_exchange_16(ptr, val)  atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val)
-  #define atomic_exchange_32(ptr, val)  atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val)
-  #define atomic_exchange_64(ptr, val)  atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val)
-  #define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val)
-      
-  #define atomic_val_compare_exchange_8   __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_16  __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_32  __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_64  __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
-    
-  #define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val))
-  #define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val))
-  #define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val))
-  #define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val))
-  #define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val))
-    
-  #define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val))
-  #define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val))
-  #define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val))
-  #define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val))
-  #define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val))
-    
-  #define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val))
-  #define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val))
-  #define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val))
-  #define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val))
-  #define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val))
-    
-  #define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val))
-  #define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val))
-  #define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val))
-  #define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val))
-  #define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val))
-    
-  #define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val))
-  #define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val))
-  #define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val))
-  #define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val))
-  #define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val))
-    
-  #define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val))
-  #define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val))
-  #define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val))
-  #define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val))
-  #define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val))
-    
-  #define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val))
-  #define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val))
-  #define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val))
-  #define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val))
-  #define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val))
-    
-  #define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val))
-  #define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val))
-  #define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val))
-  #define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val))
-  #define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val))
-    
-  #define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val))
-  #define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val))
-  #define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val))
-  #define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val))
-  #define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val))
-    
-  #define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val))
-  #define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val))
-  #define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val))
-  #define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val))
-  #define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val))
-
-#else
-  #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-  #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-  #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-  #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-  #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-
-  #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
-  #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
-
-  #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-  #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-  #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#endif
 
 #ifdef __cplusplus
 }

include/os/osDef.h

@@ -49,13 +49,14 @@ extern "C" {
 #endif
 
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#if defined(WINDOWS)
   char *stpcpy (char *dest, const char *src);
   char *stpncpy (char *dest, const char *src, size_t n);
 
   // specific
   typedef int (*__compar_fn_t)(const void *, const void *);
   #define ssize_t int
+  #define _SSIZE_T_
   #define bzero(ptr, size) memset((ptr), 0, (size))
   #define strcasecmp  _stricmp
   #define strncasecmp _strnicmp
@@ -63,16 +64,46 @@ extern "C" {
   #define strtok_r strtok_s
   #define snprintf _snprintf
   #define in_addr_t unsigned long
-  #define socklen_t int
+//  #define socklen_t int
 
-  struct tm *localtime_r(const time_t *timep, struct tm *result);
-  char *     strptime(const char *buf, const char *fmt, struct tm *tm);
   char *     strsep(char **stringp, const char *delim);
   char *     getpass(const char *prefix);
   char *     strndup(const char *s, size_t n);
+  int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
+  // for send function in tsocket.c
+  #define MSG_NOSIGNAL             0
+  #define SO_NO_CHECK              0x1234
+  #define SOL_TCP                  0x1234
+
+  #define SHUT_RDWR                SD_BOTH
+  #define SHUT_RD                  SD_RECEIVE
+  #define SHUT_WR                  SD_SEND
+
+  #define LOCK_EX 1
+  #define LOCK_NB 2
+  #define LOCK_UN 3
+
+  #ifndef PATH_MAX
+  #define PATH_MAX 256
   #endif
 
+  typedef struct {
+    int    we_wordc;
+    char  *we_wordv[1];
+    int    we_offs;
+    char   wordPos[1025];
+  } wordexp_t;
+  int  wordexp(char *words, wordexp_t *pwordexp, int flags);
+  void wordfree(wordexp_t *pwordexp);
+
+  #define openlog(a, b, c)
+  #define closelog()
+  #define LOG_ERR 0
+  #define LOG_INFO 1
+  void syslog(int unused, const char *format, ...);
+#endif // WINDOWS
+  
 #ifndef WINDOWS
   #ifndef O_BINARY
     #define O_BINARY 0
@@ -166,7 +197,7 @@ extern "C" {
   #define PRIzu "zu"  
 #endif
 
-#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64)  || defined(_TD_ARM_32) || defined(_TD_ARM_64)  || defined(_TD_DARWIN_64)
+#if !defined(WINDOWS)
   #if defined(_TD_DARWIN_64)
     // MacOS
     #if !defined(_GNU_SOURCE)
@@ -181,8 +212,7 @@ extern "C" {
   #endif
 #else
   // Windows
-//  #define setThreadName(name)
+  #define setThreadName(name)
-#define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0)
 #endif
 
 #if defined(_WIN32)
@@ -191,6 +221,10 @@ extern "C" {
 #define TD_DIRSEP	"/"
 #endif
 
+#define TD_LOCALE_LEN   64
+#define TD_CHARSET_LEN  64
+#define TD_TIMEZONE_LEN 96
+
 #ifdef __cplusplus
 }
 #endif

include/os/osDir.h

@@ -16,16 +16,40 @@
 #ifndef _TD_OS_DIR_H_
 #define _TD_OS_DIR_H_
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define opendir OPENDIR_FUNC_TAOS_FORBID
+    #define readdir READDIR_FUNC_TAOS_FORBID
+    #define closedir CLOSEDIR_FUNC_TAOS_FORBID
+    #define dirname DIRNAME_FUNC_TAOS_FORBID
+    #undef basename
+    #define basename BASENAME_FUNC_TAOS_FORBID
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct TdDir *TdDirPtr;
+typedef struct TdDirEntry *TdDirEntryPtr;
+
+
 void    taosRemoveDir(const char *dirname);
-int32_t taosDirExist(char *dirname);
+bool    taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
-void    taosRemoveOldFiles(char *dirname, int32_t keepDays);
+void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
-int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen);
+int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);
+bool    taosIsDir(const char *dirname);
+char*   taosDirName(char *dirname);
+char*   taosDirEntryBaseName(char *dirname);
+
+TdDirPtr      taosOpenDir(const char *dirname);
+TdDirEntryPtr taosReadDir(TdDirPtr pDir);
+bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
+char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
+int32_t       taosCloseDir(TdDirPtr pDir);
 
 #ifdef __cplusplus
 }

include/os/osEnv.h

@@ -16,15 +16,40 @@
 #ifndef _TD_OS_ENV_H_
 #define _TD_OS_ENV_H_
 
+#include "osSysinfo.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 extern char            tsOsName[];
+extern char            tsTimezoneStr[];
+extern enum TdTimezone tsTimezone;
+extern char            tsCharset[];
+extern char            tsLocale[];
+extern int8_t          tsDaylight;
+extern bool            tsEnableCoreFile;
+extern int64_t         tsPageSizeKB;
+extern int64_t         tsOpenMax;
+extern int64_t         tsStreamMax;
+extern float           tsNumOfCores;
+extern int64_t         tsTotalMemoryKB;
+
+extern char configDir[];
 extern char tsDataDir[];
 extern char tsLogDir[];
-extern char tsScriptDir[];
+extern char tsTempDir[];
-extern char configDir[];
+
+extern SDiskSpace tsDataSpace;
+extern SDiskSpace tsLogSpace;
+extern SDiskSpace tsTempSpace;
+
+void osDefaultInit();
+void osUpdate();
+void osCleanup();
+bool osLogSpaceAvailable();
+void osSetTimezone(const char *timezone);
+void osSetSystemLocale(const char *inLocale, const char *inCharSet);
 
 #ifdef __cplusplus
 }

include/os/osEok.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_EOK_H
+#define TDENGINE_OS_EOK_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __APPLE__
+
+enum EPOLL_EVENTS
+  {
+    EPOLLIN = 0x001,
+#define EPOLLIN EPOLLIN
+    EPOLLPRI = 0x002,
+#define EPOLLPRI EPOLLPRI
+    EPOLLOUT = 0x004,
+#define EPOLLOUT EPOLLOUT
+    EPOLLRDNORM = 0x040,
+#define EPOLLRDNORM EPOLLRDNORM
+    EPOLLRDBAND = 0x080,
+#define EPOLLRDBAND EPOLLRDBAND
+    EPOLLWRNORM = 0x100,
+#define EPOLLWRNORM EPOLLWRNORM
+    EPOLLWRBAND = 0x200,
+#define EPOLLWRBAND EPOLLWRBAND
+    EPOLLMSG = 0x400,
+#define EPOLLMSG EPOLLMSG
+    EPOLLERR = 0x008,
+#define EPOLLERR EPOLLERR
+    EPOLLHUP = 0x010,
+#define EPOLLHUP EPOLLHUP
+    EPOLLRDHUP = 0x2000,
+#define EPOLLRDHUP EPOLLRDHUP
+    EPOLLEXCLUSIVE = 1u << 28,
+#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE
+    EPOLLWAKEUP = 1u << 29,
+#define EPOLLWAKEUP EPOLLWAKEUP
+    EPOLLONESHOT = 1u << 30,
+#define EPOLLONESHOT EPOLLONESHOT
+    EPOLLET = 1u << 31
+#define EPOLLET EPOLLET
+  };
+
+/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl().  */
+#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface.  */
+#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface.  */
+#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure.  */
+
+
+typedef union epoll_data
+{
+  void *ptr;
+  int fd;
+  uint32_t u32;
+  uint64_t u64;
+} epoll_data_t;
+
+struct epoll_event
+{
+  uint32_t events;  /* Epoll events */
+  epoll_data_t data;  /* User data variable */
+};
+
+int epoll_create(int size);
+int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
+int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout);
+int epoll_close(int epfd);
+
+#endif // __APPLE__
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_
+

include/os/osFile.h

@@ -22,49 +22,81 @@ extern "C" {
 
 #include "osSocket.h"
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+// If the error is in a third-party library, place this header file under the third-party library header file.
-typedef int32_t FileFd;
+// When you want to use this feature, you should find or add the same function in the following sectio
-#else
+#ifndef ALLOW_FORBID_FUNC
-typedef int32_t FileFd;
+    #define open OPEN_FUNC_TAOS_FORBID
+    #define fopen FOPEN_FUNC_TAOS_FORBID
+    #define access ACCESS_FUNC_TAOS_FORBID
+    #define stat STAT_FUNC_TAOS_FORBID
+    #define lstat LSTAT_FUNC_TAOS_FORBID
+    #define fstat FSTAT_FUNC_TAOS_FORBID
+    #define close CLOSE_FUNC_TAOS_FORBID
+    #define fclose FCLOSE_FUNC_TAOS_FORBID
+    #define fsync FSYNC_FUNC_TAOS_FORBID
+    #define getline GETLINE_FUNC_TAOS_FORBID
+    // #define fflush FFLUSH_FUNC_TAOS_FORBID
 #endif
 
-#define FD_INITIALIZER ((int32_t)-1)
-
 #ifndef PATH_MAX
 #define PATH_MAX 256
 #endif
 
-int32_t taosLockFile(FileFd fd);
+typedef struct TdFile *TdFilePtr;
-int32_t taosUnLockFile(FileFd fd);
  
-int32_t taosUmaskFile(FileFd fd);
+#define TD_FILE_CTEATE    0x0001
+#define TD_FILE_WRITE     0x0002
+#define TD_FILE_READ      0x0004
+#define TD_FILE_TRUNC     0x0008
+#define TD_FILE_APPEND    0x0010
+#define TD_FILE_TEXT      0x0020
+#define TD_FILE_AUTO_DEL  0x0040
+#define TD_FILE_EXCL      0x0080
+#define TD_FILE_STREAM    0x0100   // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
+TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
+
+#define TD_FILE_ACCESS_EXIST_OK 0x1
+#define TD_FILE_ACCESS_READ_OK  0x2
+#define TD_FILE_ACCESS_WRITE_OK 0x4
+bool    taosCheckAccessFile(const char *pathname, int mode);
+ 
+int32_t taosLockFile(TdFilePtr pFile);
+int32_t taosUnLockFile(TdFilePtr pFile);
+ 
+int32_t taosUmaskFile(int32_t maskVal);
  
 int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
-int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime);
+int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
+int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
+bool    taosCheckExistFile(const char *pathname);
  
-FileFd taosOpenFileWrite(const char *path);
+int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
-FileFd taosOpenFileCreateWrite(const char *path);
+int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
-FileFd taosOpenFileCreateWriteTrunc(const char *path);
+int32_t taosFsyncFile(TdFilePtr pFile);
-FileFd taosOpenFileCreateWriteAppend(const char *path);
-FileFd taosOpenFileRead(const char *path);
-FileFd taosOpenFileReadWrite(const char *path);
  
-int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence);
+int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
-int32_t taosFtruncateFile(FileFd fd, int64_t length);
+int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
-int32_t taosFsyncFile(FileFd fd);
+int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
+void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
 
-int64_t taosReadFile(FileFd fd, void *buf, int64_t count);
+int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
-int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
 
-void taosCloseFile(FileFd fd);
+int32_t taosEOFFile(TdFilePtr pFile);
+ 
+int64_t taosCloseFile(TdFilePtr *ppFile);
  
 int32_t taosRenameFile(const char *oldName, const char *newName);
 int64_t taosCopyFile(const char *from, const char *to);
+int32_t taosRemoveFile(const char *path);
  
 void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
 
-int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
+int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
-int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
+
+void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
+bool taosValidFile(TdFilePtr pFile);
+
+int32_t taosGetErrorFile(TdFilePtr pFile);
 
 #ifdef __cplusplus
 }

include/os/osLocale.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_LOCALE_H_
+#define _TD_OS_LOCALE_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define setlocale SETLOCALE_FUNC_TAOS_FORBID
+#endif
+
+char *taosCharsetReplace(char *charsetstr);
+void  taosGetSystemLocale(char *outLocale, char *outCharset);
+void  taosSetSystemLocale(const char *inLocale, const char *inCharSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_LOCALE_H_*/

include/os/osMemory.h

@@ -20,12 +20,25 @@
 extern "C" {
 #endif
 
-#define tfree(x)       \
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following sectio
+#ifndef ALLOW_FORBID_FUNC
+    #define malloc MALLOC_FUNC_TAOS_FORBID
+    #define calloc CALLOC_FUNC_TAOS_FORBID
+    #define realloc REALLOC_FUNC_TAOS_FORBID
+    #define free FREE_FUNC_TAOS_FORBID
+#endif
+
+void *taosMemoryMalloc(int32_t size);
+void *taosMemoryCalloc(int32_t num, int32_t size);
+void *taosMemoryRealloc(void *ptr, int32_t size);
+void taosMemoryFree(const void *ptr);
+int32_t taosMemorySize(void *ptr);
+
+#define taosMemoryFreeClear(ptr) \
   do {                           \
-    if (x) {           \
+    taosMemoryFree(ptr);         \
-      free((void *)(x)); \
+    (ptr)=NULL;                  \
-      (x) = 0;           \
-    }                  \
   } while (0)
 
 #ifdef __cplusplus

include/os/osRand.h

@@ -20,7 +20,17 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define rand RAND_FUNC_TAOS_FORBID
+    #define srand SRAND_FUNC_TAOS_FORBID
+    #define rand_r RANDR_FUNC_TAOS_FORBID
+#endif
+
+void taosSeedRand(uint32_t seed);
 uint32_t taosRand(void);
+uint32_t taosRandR(uint32_t *pSeed);
 void     taosRandStr(char* str, int32_t size);
 uint32_t taosSafeRand(void);
 

include/os/osSemaphore.h

@@ -37,25 +37,25 @@ extern "C" {
 #endif
 
 #if defined (_TD_DARWIN_64)
-  #define pthread_rwlock_t pthread_mutex_t
+//  #define TdThreadRwlock TdThreadMutex
-  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+//  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+//  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
-  #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+// #define taosThreadRwlockWrlock(lock) taosThreadMutexLock(lock)
-  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+//  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
-  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+//  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define pthread_spinlock_t pthread_mutex_t
+  #define TdThreadSpinlock TdThreadMutex
-  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
-  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
-  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
 #endif
 
-bool    taosCheckPthreadValid(pthread_t thread);
+bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
-int64_t taosGetPthreadId(pthread_t thread);
+int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(pthread_t* thread);
+void    taosResetPthread(TdThread* thread);
-bool    taosComparePthread(pthread_t first, pthread_t second);
+bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
 int32_t taosGetAppName(char* name, int32_t* len);
 

include/os/osSleep.h

@@ -20,7 +20,18 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define Sleep SLEEP_FUNC_TAOS_FORBID
+    #define sleep SLEEP_FUNC_TAOS_FORBID
+    #define usleep USLEEP_FUNC_TAOS_FORBID
+    #define nanosleep NANOSLEEP_FUNC_TAOS_FORBID
+#endif
+
+void taosSsleep(int32_t s);
 void taosMsleep(int32_t ms);
+void taosUsleep(int32_t us);
 
 #ifdef __cplusplus
 }

include/os/osSocket.h

@@ -16,75 +16,152 @@
 #ifndef _TD_OS_SOCKET_H_
 #define _TD_OS_SOCKET_H_
 
-#ifdef __cplusplus
+// If the error is in a third-party library, place this header file under the third-party library header file.
-extern "C" {
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define socket SOCKET_FUNC_TAOS_FORBID
+    #define bind   BIND_FUNC_TAOS_FORBID
+    #define listen LISTEN_FUNC_TAOS_FORBID
+    #define accept ACCEPT_FUNC_TAOS_FORBID
+    #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+    #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
+    #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
+    #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
+    #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#if defined(WINDOWS)
   #include "winsock2.h"
   #include <WS2tcpip.h>
   #include <winbase.h>
   #include <Winsock2.h>
+#else
+    #include <netinet/in.h>
+    #include <sys/socket.h>
+
+    #if defined(_TD_DARWIN_64)
+        #include <osEok.h>
     #else
         #include <netinet/in.h>
         #include <sys/epoll.h>
     #endif
+#endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(WINDOWS)
+  #define htobe64 htonll
+#endif
+
+#if defined(WINDOWS)
+  #define TAOS_EPOLL_WAIT_TIME 100
+  typedef SOCKET eventfd_t;
+  #define eventfd(a, b) -1
+  typedef SOCKET EpollFd;
+  #define EpollClose(pollFd) epoll_close(pollFd)
+  #ifndef EPOLLWAKEUP
+    #define EPOLLWAKEUP (1u << 29)
+  #endif
+#elif defined(_TD_DARWIN_64)
+  #define TAOS_EPOLL_WAIT_TIME 500
+  typedef int32_t SOCKET;
+  typedef SOCKET EpollFd;
+  #define EpollClose(pollFd) epoll_close(pollFd)
+#else
   #define TAOS_EPOLL_WAIT_TIME 500
   typedef int32_t SOCKET;
   typedef SOCKET EpollFd;
   #define EpollClose(pollFd) taosCloseSocket(pollFd)
-
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-typedef SOCKET SocketFd;
-#else
-typedef int32_t SocketFd;
-#endif
-
-int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
-int32_t taosWriteSocket(SocketFd fd, void *msg, int len);
-int32_t taosReadSocket(SocketFd fd, void *msg, int len);
-int32_t taosCloseSocketNoCheck(SocketFd fd);
-int32_t taosCloseSocket(SocketFd fd);
-void    taosShutDownSocketRD(SOCKET fd);
-void    taosShutDownSocketWR(SOCKET fd);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
-void    taosIgnSIGPIPE();
-void    taosBlockSIGPIPE();
-void    taosSetMaskSIGPIPE();
-int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
-int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
-
-uint32_t    taosInetAddr(char *ipAddr);
-const char *taosInetNtoa(struct in_addr ipInt);
-
-#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 
-  #define htobe64 htonll
-  #if defined(_TD_GO_DLL_)
-    uint64_t htonll(uint64_t val);
-  #endif
 #endif
 
 #if defined(_TD_DARWIN_64)
-  #define htobe64 htonll
+//  #define htobe64 htonll
+
+#	include <libkern/OSByteOrder.h>
+
+#	define htobe16(x) OSSwapHostToBigInt16(x)
+#	define htole16(x) OSSwapHostToLittleInt16(x)
+#	define be16toh(x) OSSwapBigToHostInt16(x)
+#	define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#	define htobe32(x) OSSwapHostToBigInt32(x)
+#	define htole32(x) OSSwapHostToLittleInt32(x)
+#	define be32toh(x) OSSwapBigToHostInt32(x)
+#	define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#	define htobe64(x) OSSwapHostToBigInt64(x)
+#	define htole64(x) OSSwapHostToLittleInt64(x)
+#	define be64toh(x) OSSwapBigToHostInt64(x)
+#	define le64toh(x) OSSwapLittleToHostInt64(x)
+
+#	define __BYTE_ORDER    BYTE_ORDER
+#	define __BIG_ENDIAN    BIG_ENDIAN
+#	define __LITTLE_ENDIAN LITTLE_ENDIAN
+#	define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
+#define TAOS_EPOLL_WAIT_TIME 500
-int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
-int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
 
-SOCKET  taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+typedef int32_t SocketFd;
-SOCKET  taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+typedef SocketFd  EpollFd;
-SOCKET  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int32_t taosKeepTcpAlive(SOCKET sockFd);
 
-int32_t  taosGetFqdn(char *);
+typedef struct TdSocket {
+#if SOCKET_WITH_LOCK
+  TdThreadRwlock rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} *TdSocketPtr, TdSocket;
+
+typedef struct TdSocketServer *TdSocketServerPtr;
+typedef struct TdSocket *TdSocketPtr;
+typedef struct TdEpoll *TdEpollPtr;
+
+int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosCloseSocketNoCheck1(SocketFd fd);
+int32_t taosCloseSocket(TdSocketPtr *ppSocket);
+int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
+int32_t taosShutDownSocketRD(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
+int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
+int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
+int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
+int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
+int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+
+TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+
+int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+
+void     taosBlockSIGPIPE();
 uint32_t taosGetIpv4FromFqdn(const char *);
+int32_t  taosGetFqdn(char *);
 void     tinet_ntoa(char *ipstr, uint32_t ip);
 uint32_t ip2uint(const char *const ip_addr);
+void     taosIgnSIGPIPE();
+void     taosSetMaskSIGPIPE();
+uint32_t taosInetAddr(const char *ipAddr);
+const char *taosInetNtoa(struct in_addr ipInt);
+
+TdEpollPtr taosCreateEpoll(int32_t size);
+int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
+int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
+int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
 
 #ifdef __cplusplus
 }

include/os/osString.h

@@ -20,16 +20,29 @@
 extern "C" {
 #endif
 
+typedef wchar_t TdWchar;
+typedef int32_t TdUcs4;
+
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID
+    #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID
+    #define iconv ICONV_FUNC_TAOS_FORBID
+    #define wcwidth WCWIDTH_FUNC_TAOS_FORBID
+    #define wcswidth WCSWIDTH_FUNC_TAOS_FORBID
+    #define mbtowc MBTOWC_FUNC_TAOS_FORBID
+    #define mbstowcs MBSTOWCS_FUNC_TAOS_FORBID
+    #define wctomb WCTOMB_FUNC_TAOS_FORBID
+    #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
+    #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
+    #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
+#endif
+
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   #define tstrdup(str) _strdup(str)
-  #define tstrndup(str, size) _strndup(str, size)  
-  int32_t tgetline(char **lineptr, size_t *n, FILE *stream);
-  int32_t twcslen(const wchar_t *wcs);
 #else
   #define tstrdup(str) strdup(str)
-  #define tstrndup(str, size) strndup(str, size)
-  #define tgetline(lineptr, n, stream) getline(lineptr, n, stream)
-  #define twcslen wcslen
 #endif
 
 #define tstrncpy(dst, src, size) \
@@ -38,14 +51,21 @@ extern "C" {
     (dst)[(size)-1] = 0;            \
   } while (0)
 
+int32_t taosUcs4len(TdUcs4 *ucs4);
 int64_t taosStr2int64(const char *str);
 
-// USE_LIBICONV
+int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
+bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
-bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
+int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
-int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
+TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
-char *  taosCharsetReplace(char *charsetstr);
+
+int32_t taosWcharWidth(TdWchar wchar);
+int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);
+int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size);
+int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
+int32_t taosWcharToMb(char *pStr, TdWchar wchar);
+int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
 
 #ifdef __cplusplus
 }

include/os/osSysinfo.h

@@ -16,48 +16,49 @@
 #ifndef _TD_OS_SYSINFO_H_
 #define _TD_OS_SYSINFO_H_
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "os.h"
-  
-#define TSDB_LOCALE_LEN   64
-#define TSDB_TIMEZONE_LEN 96
-
-extern int64_t tsPageSize;
-extern int64_t tsOpenMax;
-extern int64_t tsStreamMax;
-extern int32_t tsNumOfCores;
-extern int32_t tsTotalMemoryMB;
-extern char     tsTimezone[];
-extern char     tsLocale[];
-extern char     tsCharset[];            // default encode string
-
 typedef struct {
   int64_t total;
   int64_t used;
   int64_t avail;
 } SDiskSize;
 
-int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
+typedef struct {
-int32_t taosGetCpuCores();
+  int64_t   reserved;
+  SDiskSize size;
+} SDiskSpace;
+
+bool    taosCheckSystemIsSmallEnd();
 void    taosGetSystemInfo();
-bool    taosReadProcIO(int64_t *rchars, int64_t *wchars);
+int32_t taosGetEmail(char *email, int32_t maxLen);
-bool    taosGetProcIO(float *readKB, float *writeKB);
+int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
-bool    taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
+int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
-bool    taosGetBandSpeed(float *bandSpeedKb);
+int32_t taosGetCpuCores(float *numOfCores);
-void    taosGetDisk();
+int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
-bool    taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
+int32_t taosGetTotalMemory(int64_t *totalKB);
-bool    taosGetProcMemory(float *memoryUsedMB);
+int32_t taosGetProcMemory(int64_t *usedKB);
-bool    taosGetSysMemory(float *memoryUsedMB);
+int32_t taosGetSysMemory(int64_t *usedKB);
-void    taosPrintOsInfo();
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
-int     taosSystem(const char *cmd);
+int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
+int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
+
 void    taosKillSystem();
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
-char *  taosGetCmdlineByPID(int pid);
+char   *taosGetCmdlineByPID(int32_t pid);
 void    taosSetCoreDump(bool enable);
 
+#if !defined(LINUX)
+
+#define _UTSNAME_LENGTH 65
+#define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH
+
+#endif // WINDOWS
+
 typedef struct {
   char sysname[_UTSNAME_MACHINE_LENGTH];
   char nodename[_UTSNAME_MACHINE_LENGTH];

include/os/osSystem.h

@@ -20,11 +20,23 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define popen POPEN_FUNC_TAOS_FORBID
+    #define pclose PCLOSE_FUNC_TAOS_FORBID
+    #define tcsetattr TCSETATTR_FUNC_TAOS_FORBID
+    #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
+#endif
+
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
+void setTerminalMode();
+int32_t getOldTerminalMode();
+void resetTerminalMode();
 
 #ifdef __cplusplus
 }

include/os/osThread.h

@@ -16,12 +16,101 @@
 #ifndef _TD_OS_THREAD_H_
 #define _TD_OS_THREAD_H_
 
+#include <pthread.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <pthread.h>
+typedef pthread_t TdThread;
+typedef pthread_spinlock_t TdThreadSpinlock;
+typedef pthread_mutex_t TdThreadMutex;
+typedef pthread_mutexattr_t TdThreadMutexAttr;
+typedef pthread_rwlock_t TdThreadRwlock;
+typedef pthread_attr_t TdThreadAttr;
+typedef pthread_once_t TdThreadOnce;
+typedef pthread_rwlockattr_t TdThreadRwlockAttr;
+typedef pthread_cond_t TdThreadCond;
+typedef pthread_condattr_t TdThreadCondAttr;
 
+#define taosThreadCleanupPush pthread_cleanup_push
+#define taosThreadCleanupPop pthread_cleanup_pop
+
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID
+    #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID
+    #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID
+    #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID
+    #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID
+    #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID
+    #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID
+    #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID
+    #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
+    #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
+    #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID
+    #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
+    #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
+    #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
+    #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID
+    #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID
+    #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID
+    #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
+    #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
+    #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
+    #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
+    #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID
+#endif
+
+int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared);
+int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr);
+int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock);
+int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
+int32_t taosThreadSpinLock(TdThreadSpinlock *lock);
+int32_t taosThreadMutexLock(TdThreadMutex *mutex);
+int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock);
+int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock);
+int32_t taosThreadMutexUnlock(TdThreadMutex *mutex);
+int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock);
+int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock);
+void taosThreadTestCancel(void);
+int32_t taosThreadAttrInit(TdThreadAttr *attr);
+int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg);
+int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void));
+int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState);
+int32_t taosThreadAttrDestroy(TdThreadAttr *attr);
+int32_t taosThreadJoin(TdThread thread, void **pValue);
+int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr);
+int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock);
+int32_t taosThreadCondSignal(TdThreadCond *cond);
+int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr);
+int32_t taosThreadCondBroadcast(TdThreadCond *cond);
+int32_t taosThreadCondDestroy(TdThreadCond *cond);
+int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex);
+TdThread taosThreadSelf(void);
+int32_t taosThreadEqual(TdThread t1, TdThread t2);
+int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset);
+int32_t taosThreadCancel(TdThread thread);
+int32_t taosThreadKill(TdThread thread, int sig);
 #ifdef __cplusplus
 }
 #endif

include/os/osTime.h

@@ -20,12 +20,21 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+// If the error is in a third-party library, place this header file under the third-party library header file.
-  #ifdef _TD_GO_DLL_
+// When you want to use this feature, you should find or add the same function in the following section.
-    #define MILLISECOND_PER_SECOND (1000LL)
+#ifndef ALLOW_FORBID_FUNC
-  #else
+    #define strptime STRPTIME_FUNC_TAOS_FORBID
-    #define MILLISECOND_PER_SECOND (1000i64)
+    #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID
+    #define localtime_s LOCALTIMES_FUNC_TAOS_FORBID
+    #define localtime_r LOCALTIMER_FUNC_TAOS_FORBID
+    #define time TIME_FUNC_TAOS_FORBID
 #endif
+
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+
+  #define CLOCK_REALTIME 	0
+
+  #define MILLISECOND_PER_SECOND (1000i64)
 #else
   #define MILLISECOND_PER_SECOND ((int64_t)1000L)
 #endif
@@ -61,6 +70,9 @@ static FORCE_INLINE int64_t taosGetTimestampNs() {
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 
+char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
+struct tm *taosLocalTime(const time_t *timep, struct tm *result);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osTimer.h

@@ -20,6 +20,16 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define timer_create TIMER_CREATE_FUNC_TAOS_FORBID
+    #define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID
+    #define timer_delete TIMER_DELETE_FUNC_TAOS_FORBID
+    #define timeSetEvent TIMESETEVENT_SETTIME_FUNC_TAOS_FORBID
+    #define timeKillEvent TIMEKILLEVENT_SETTIME_FUNC_TAOS_FORBID
+#endif
+
 #define MSECONDS_PER_TICK 5
 
 int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);

include/os/osTimezone.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_TIMEZONE_H_
+#define _TD_OS_TIMEZONE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// If the error is in a third-party library, place this header file under the third-party library header file.
+// When you want to use this feature, you should find or add the same function in the following section.
+#ifndef ALLOW_FORBID_FUNC
+    #define tzset TZSET_FUNC_TAOS_FORBID
+#endif
+
+enum TdTimezone
+{
+      TdWestZone12=-12,
+      TdWestZone11,
+      TdWestZone10,
+      TdWestZone9,
+      TdWestZone8,
+      TdWestZone7,
+      TdWestZone6,
+      TdWestZone5,
+      TdWestZone4,
+      TdWestZone3,
+      TdWestZone2,
+      TdWestZone1,
+      TdZeroZone,
+      TdEastZone1,
+      TdEastZone2,
+      TdEastZone3,
+      TdEastZone4,
+      TdEastZone5,
+      TdEastZone6,
+      TdEastZone7,
+      TdEastZone8,
+      TdEastZone9,
+      TdEastZone10,
+      TdEastZone11,
+      TdEastZone12
+};
+
+void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone);
+void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_TIMEZONE_H_*/

include/util/exception.h

@@ -1,125 +0,0 @@
-/*
- * Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_EXCEPTION_H
-#define _TD_UTIL_EXCEPTION_H
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * cleanup actions
- */
-typedef struct SCleanupAction {
-    bool failOnly;
-    uint8_t wrapper;
-    uint16_t reserved;
-    void* func;
-    union {
-        void* Ptr;
-        bool Bool;
-        char Char;
-        int8_t Int8;
-        uint8_t Uint8;
-        int16_t Int16;
-        uint16_t Uint16;
-        int Int;
-        unsigned int Uint;
-        int32_t Int32;
-        uint32_t Uint32;
-        int64_t Int64;
-        uint64_t Uint64;
-        float Float;
-        double Double;
-    } arg1, arg2;
-} SCleanupAction;
-
-
-/*
- * exception hander registration
- */
-typedef struct SExceptionNode {
-   struct SExceptionNode* prev;
-   jmp_buf jb;
-   int32_t code;
-   int32_t maxCleanupAction;
-   int32_t numCleanupAction;
-   SCleanupAction* cleanupActions;
-} SExceptionNode;
-
-////////////////////////////////////////////////////////////////////////////////
-// functions & macros for auto-cleanup
-
-void cleanupPush_void_ptr_ptr   ( bool failOnly, void* func, void* arg1, void* arg2 );
-void cleanupPush_void_ptr_bool  ( bool failOnly, void* func, void* arg1, bool arg2 );
-void cleanupPush_void_ptr       ( bool failOnly, void* func, void* arg );
-void cleanupPush_int_int        ( bool failOnly, void* func, int arg );
-void cleanupPush_void           ( bool failOnly, void* func );
-void cleanupPush_int_ptr        ( bool failOnly, void* func, void* arg );
-
-int32_t cleanupGetActionCount();
-void cleanupExecuteTo( int32_t anchor, bool failed );
-void cleanupExecute( SExceptionNode* node, bool failed );
-bool cleanupExceedLimit();
-
-#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 )  cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) )
-#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) )
-#define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg )             cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) )
-#define CLEANUP_PUSH_INT_INT( failOnly, func, arg )              cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) )
-#define CLEANUP_PUSH_VOID( failOnly, func )                      cleanupPush_void( (failOnly), (void*)(func) )
-#define CLEANUP_PUSH_INT_PTR( failOnly, func, arg )              cleanupPush_int_ptr( (failOnly), (void*)(func), (void*)(arg) )
-#define CLEANUP_PUSH_FREE( failOnly, arg )                       cleanupPush_void_ptr( (failOnly), free, (void*)(arg) )
-#define CLEANUP_PUSH_CLOSE( failOnly, arg )                      cleanupPush_int_int( (failOnly), close, (int)(arg) )
-#define CLEANUP_PUSH_FCLOSE( failOnly, arg )                     cleanupPush_int_ptr( (failOnly), fclose, (void*)(arg) )
-
-#define CLEANUP_GET_ANCHOR()          cleanupGetActionCount()
-#define CLEANUP_EXECUTE_TO( anchor, failed )  cleanupExecuteTo( (anchor), (failed) )
-#define CLEANUP_EXCEED_LIMIT()        cleanupExceedLimit() 
-
-////////////////////////////////////////////////////////////////////////////////
-// functions & macros for exception handling
-
-void exceptionPushNode( SExceptionNode* node );
-int32_t exceptionPopNode();
-void exceptionThrow( int32_t code );
-
-#define TRY(maxCleanupActions) do { \
-    SExceptionNode exceptionNode = { 0 }; \
-    SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \
-    exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \
-    exceptionNode.cleanupActions = cleanupActions; \
-    exceptionPushNode( &exceptionNode ); \
-    int caughtException = setjmp( exceptionNode.jb ); \
-    if( caughtException == 0 )
-
-#define CATCH( code ) int32_t code = exceptionPopNode(); \
-    if( caughtException == 1 )
-
-#define FINALLY( code ) int32_t code = exceptionPopNode();
-
-#define END_TRY } while( 0 );
-
-#define THROW( x )          exceptionThrow( (x) )
-#define CAUGHT_EXCEPTION()  ((bool)(caughtException == 1))
-#define CLEANUP_EXECUTE()   cleanupExecute( &exceptionNode, CAUGHT_EXCEPTION() )
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_EXCEPTION_H*/

include/util/talgo.h

@@ -13,8 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_UTIL_TALGO_H
+#ifndef _TD_UTIL_TALGO_H_
-#define _TD_UTIL_TALGO_H
+#define _TD_UTIL_TALGO_H_
+
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -22,7 +24,7 @@ extern "C" {
 
 #ifndef __COMPAR_FN_T
 #define __COMPAR_FN_T
-typedef int (*__compar_fn_t) (const void *, const void *);
+typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
 #define TD_EQ 0x1
@@ -45,7 +47,7 @@ typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param);
  * @param param
  * @param comparFn
  */
-void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn);
+void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
 
 /**
  * binary search, with range support
@@ -58,7 +60,7 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex
  * @param flags
  * @return
  */
-void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags);
+void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags);
 
 /**
  * adjust heap
@@ -74,7 +76,8 @@ void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size,
  * @param maxroot: if heap is max root heap
  * @return
  */
-void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
+                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -89,10 +92,11 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @param maxroot: if heap is max root heap
  * @return
  */
-void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-
+                  const void *parswap, __ext_swap_fn_t swap, bool maxroot);
 
 #ifdef __cplusplus
 }
 #endif
-#endif  /*_TD_UTIL_TALGO_H*/
+
+#endif /*_TD_UTIL_TALGO_H_*/