refactor: merge from 3.0

cmake/addr2line_CMakeLists.txt.in

@@ -0,0 +1,12 @@
+
+# addr2line
+ExternalProject_Add(addr2line
+        GIT_REPOSITORY https://github.com/davea42/libdwarf-addr2line.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/addr2line"
+        BINARY_DIR "${TD_CONTRIB_DIR}/addr2line"
+        CONFIGURE_COMMAND "" 
+        BUILD_COMMAND ""  
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

cmake/cmake.define

@@ -15,12 +15,12 @@ MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH
 MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
 
 find_package(Git QUIET)
-if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
+if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git")
 # Update submodules as needed
     option(GIT_SUBMODULE "Check submodules during build" ON)
     if(GIT_SUBMODULE)
         message(STATUS "Submodule update")
-        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive
+        execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive
                         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
                         RESULT_VARIABLE GIT_SUBMOD_RESULT)
         if(NOT GIT_SUBMOD_RESULT EQUAL "0")
@@ -29,7 +29,7 @@ if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
     endif()
 endif()
 
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt")
+if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt")
     message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.")
 endif()
 
@@ -75,6 +75,14 @@ IF (TD_WINDOWS)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
 
 ELSE ()
+    IF (${COVER} MATCHES "true")
+        MESSAGE(STATUS "Test coverage mode, add extra flags")
+        SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+        SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+    ENDIF ()
+
     IF (${SANITIZER} MATCHES "true")
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")

cmake/cmake.options

@@ -48,6 +48,12 @@ IF(${TD_WINDOWS})
 
 ENDIF ()
 
+option(
+    BUILD_ADDR2LINE
+    "If build addr2line"
+    OFF
+    )
+
 option(
     BUILD_TEST 
     "If build unit tests using googletest" 

cmake/libdwarf_CMakeLists.txt.in

@@ -0,0 +1,12 @@
+
+# libdwarf
+ExternalProject_Add(libdwarf
+        GIT_REPOSITORY https://github.com/davea42/libdwarf-code.git
+        GIT_TAG libdwarf-0.3.1
+        SOURCE_DIR "${TD_CONTRIB_DIR}/libdwarf"
+        BINARY_DIR "${TD_CONTRIB_DIR}/libdwarf"
+        CONFIGURE_COMMAND "" 
+        BUILD_COMMAND ""  
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

contrib/CMakeLists.txt

@@ -98,6 +98,12 @@ if(${BUILD_WITH_NURAFT})
     cat("${TD_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_NURAFT})
 
+# addr2line
+if(${BUILD_ADDR2LINE})
+    cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_ADDR2LINE})
+
 # download dependencies
 configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -327,7 +333,48 @@ if(${BUILD_WITH_SQLITE})
     endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
-# pthread
+# addr2line
+if(${BUILD_ADDR2LINE})
+    check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H) 
+    check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H ) 
+    check_include_file( "inttypes.h"      HAVE_INTTYPES_H   ) 
+    check_include_file( "stddef.h"        HAVE_STDDEF_H   ) 
+    check_include_file( "stdlib.h"        HAVE_STDLIB_H   ) 
+    check_include_file( "string.h"        HAVE_STRING_H   ) 
+    check_include_file( "memory.h"        HAVE_MEMORY_H   ) 
+    check_include_file( "strings.h"       HAVE_STRINGS_H  ) 
+    check_include_file( "stdint.h"        HAVE_STDINT_H   )
+    check_include_file( "unistd.h"        HAVE_UNISTD_H   )
+    check_include_file( "sgidefs.h"       HAVE_SGIDEFS_H  )
+    check_include_file( "stdafx.h"        HAVE_STDAFX_H   )
+    check_include_file( "elf.h"           HAVE_ELF_H      ) 
+    check_include_file( "libelf.h"        HAVE_LIBELF_H   ) 
+    check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) 
+    check_include_file( "alloca.h"        HAVE_ALLOCA_H   )
+    check_include_file( "elfaccess.h"     HAVE_ELFACCESS_H)
+    check_include_file( "sys/elf_386.h"   HAVE_SYS_ELF_386_H  )
+    check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H)
+    check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
+    check_include_file( "sys/ia64/elf.h"  HAVE_SYS_IA64_ELF_H )
+    set(VERSION 0.3.1)
+    set(PACKAGE_VERSION "\"${VERSION}\"") 
+    configure_file(libdwarf/cmake/config.h.cmake config.h)
+    file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
+    add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
+    set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf")
+    if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H)
+        target_link_libraries(libdwarf PUBLIC libelf)
+    endif()
+    target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_BINARY_DIR}/contrib)
+    file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT)
+    string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
+    file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}")
+    add_library(addr2line STATIC "addr2line/addr2line.c")
+    target_link_libraries(addr2line PUBLIC libdwarf dl z)
+    target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
+endif(${BUILD_ADDR2LINE})
 
 
 # ================================================================================================

example/src/tstream.c

@@ -83,7 +83,7 @@ int32_t create_stream() {
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
   pRes = taos_query(
       pConn,
-      "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
+      "create stream stream1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
       "from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));

include/client/taos.h

@@ -221,15 +221,12 @@ DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
 DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
-#if 0
-DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
-#endif
-
 DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
+
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
@@ -240,6 +237,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 #endif
+
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {
@@ -268,12 +266,9 @@ DLL_EXPORT char   *tmq_get_table_name(TAOS_RES *res);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
 #endif
-/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
+
-#if 0
-DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
-DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
-#endif
 /* ------------------------------ TMQ END -------------------------------- */
+
 #if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 #endif

include/common/systable.h

@@ -45,7 +45,6 @@ extern "C" {
 
 #define TSDB_PERFORMANCE_SCHEMA_DB     "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS          "smas"
-#define TSDB_PERFS_TABLE_SUBSCRIBES    "subscribes"
 #define TSDB_PERFS_TABLE_CONNECTIONS   "connections"
 #define TSDB_PERFS_TABLE_QUERIES       "queries"
 #define TSDB_PERFS_TABLE_TOPICS        "topics"

include/common/tdata.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDATA_H_
+#define _TD_TDATA_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+typedef struct STaosData TDATA, tdata_t;
+
+typedef enum {
+  TAOS_META_STABLE_DATA = 0,  // super table meta
+  TAOS_META_TABLE_DATA,       // non-super table meta
+  TAOS_TS_ROW_DATA,           // row time-series data
+  TAOS_TS_COL_DATA,           // col time-series data
+  TAOS_DATA_MAX
+} ETaosDataT;
+
+struct STaosData {
+  ETaosDataT type;
+  uint32_t   nPayload;
+  uint8_t   *pPayload;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDATA_H_*/

include/common/tdataformat.h

@@ -150,29 +150,6 @@ int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags,
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition
-#ifdef TD_2_0
-
-typedef uint64_t TKEY;
-
-#define TKEY_INVALID       UINT64_MAX
-#define TKEY_NULL          TKEY_INVALID
-#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
-#define TKEY_DELETE_FLAG   (((TKEY)1) << 62)
-#define TKEY_VALUE_FILTER  (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
-
-#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
-#define TKEY_IS_DELETED(tkey)  (((tkey)&TKEY_DELETE_FLAG) != 0)
-#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
-#define tdGetTKEY(key)         (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
-#define tdGetKey(tkey)         (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
-
-#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
-#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)
-
-#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
-
-#else
-
 // typedef uint64_t TKEY;
 #define TKEY TSKEY
 
@@ -192,8 +169,6 @@ typedef uint64_t TKEY;
 
 #define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
 
-#endif
-
 static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
   TSKEY lkey = key;
   if (key > MAX_TS_KEY) {
@@ -218,157 +193,6 @@ static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
   }
 }
 
-#if 0
-// ----------------- Data row structure
-
-/* A data row, the format is like below:
- * |<------------------------------------------------ len ---------------------------------->|
- * |<--     Head      -->|<---------   flen -------------->|                                 |
- * +---------------------+---------------------------------+---------------------------------+
- * | uint16_t |  int16_t |                                 |                                 |
- * +----------+----------+---------------------------------+---------------------------------+
- * |   len    | sversion |           First part            |             Second part         |
- * +----------+----------+---------------------------------+---------------------------------+
- *
- * NOTE: timestamp in this row structure is TKEY instead of TSKEY
- */
-typedef void *SDataRow;
-
-#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
-
-#define dataRowLen(r)                (*(TDRowLenT *)(r))  // 0~65535
-#define dataRowEnd(r)                POINTER_SHIFT(r, dataRowLen(r))
-#define dataRowVersion(r)            (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
-#define dataRowTuple(r)              POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowTKey(r)               (*(TKEY *)(dataRowTuple(r)))
-#define dataRowKey(r)                tdGetKey(dataRowTKey(r))
-#define dataRowSetLen(r, l)          (dataRowLen(r) = (l))
-#define dataRowSetVersion(r, v)      (dataRowVersion(r) = (v))
-#define dataRowCpy(dst, r)           memcpy((dst), (r), dataRowLen(r))
-#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
-#define dataRowDeleted(r)            TKEY_IS_DELETED(dataRowTKey(r))
-
-SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
-void     tdFreeDataRow(SDataRow row);
-void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-SDataRow tdDataRowDup(SDataRow row);
-
-// offset here not include dataRow header length
-static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
-                                           int32_t offset) {
-  assert(value != NULL);
-  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-
-  if (IS_VAR_DATA_TYPE(type)) {
-    *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-    if (isCopyVarData) {
-      memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
-    }
-    dataRowLen(row) += varDataTLen(value);
-  } else {
-    if (offset == 0) {
-      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
-      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
-      memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]);
-    } else {
-      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-    }
-  }
-
-  return 0;
-}
-
-// offset here not include dataRow header length
-static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
-  return tdAppendDataColVal(row, value, true, type, offset);
-}
-
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
-  if (IS_VAR_DATA_TYPE(type)) {
-    return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-  } else {
-    return POINTER_SHIFT(row, offset);
-  }
-}
-
-static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) {
-  return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset);
-}
-
-static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-
-  return tdGetRowDataOfCol(row, type, offset);
-}
-
-static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-
-  return isNull(tdGetRowDataOfCol(row, type, offset), type);
-}
-
-static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
-  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
-  int8_t  type = pSchema->columns[idx].type;
-  int16_t bytes = pSchema->columns[idx].bytes;
-
-  setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
-}
-
-static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src,
-                                                STSchema *pSrcSchema, int32_t srcIdx) {
-  int8_t type = pDstSchema->columns[dstIdx].type;
-  assert(type == pSrcSchema->columns[srcIdx].type);
-  void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
-  void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx);
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value;
-      pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData);
-      value = POINTER_SHIFT(src, *(VarDataOffsetT *)value);
-      memcpy(pData, value, varDataTLen(value));
-      break;
-    case TSDB_DATA_TYPE_NULL:
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_TINYINT:
-    case TSDB_DATA_TYPE_UTINYINT:
-      *(uint8_t *)pData = *(uint8_t *)value;
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_USMALLINT:
-      *(uint16_t *)pData = *(uint16_t *)value;
-      break;
-    case TSDB_DATA_TYPE_INT:
-    case TSDB_DATA_TYPE_UINT:
-      *(uint32_t *)pData = *(uint32_t *)value;
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_UBIGINT:
-      *(uint64_t *)pData = *(uint64_t *)value;
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-      SET_FLOAT_PTR(pData, value);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      SET_DOUBLE_PTR(pData, value);
-      break;
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
-      } else {
-        *(TSKEY *)pData = *(TSKEY *)value;
-      }
-      break;
-    default:
-      memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes);
-  }
-}
-#endif
 // ----------------- Data column structure
 // SDataCol arrangement: data => bitmap => dataOffset
 typedef struct SDataCol {
@@ -398,29 +222,6 @@ void   *dataColSetOffset(SDataCol *pCol, int32_t nEle);
 
 bool isNEleNull(SDataCol *pCol, int32_t nEle);
 
-#if 0
-// Get the data pointer from a column-wised data
-static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) {
-  if (isAllRowsNull(pCol)) {
-    return getNullValue(pCol->type);
-  }
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
-  } else {
-    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
-  }
-}
-
-static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
-  assert(rows > 0);
-
-  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-    return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
-  } else {
-    return TYPE_BYTES[pDataCol->type] * rows;
-  }
-}
-#endif
 typedef struct {
   col_id_t  maxCols;    // max number of columns
   col_id_t  numOfCols;  // Total number of cols
@@ -479,7 +280,8 @@ void       tdResetDataCols(SDataCols *pCols);
 int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer);
+int32_t    tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update,
+                           TDRowVerT maxVer);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|
@@ -542,54 +344,6 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
   return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
 }
 
-#if 0
-// offset here not include kvRow header length
-static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
-                                         int32_t offset) {
-  assert(value != NULL);
-  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
-  SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
-  char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
-
-  pColIdx->colId = colId;
-  pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
-
-  if (IS_VAR_DATA_TYPE(type)) {
-    if (isCopyValData) {
-      memcpy(ptr, value, varDataTLen(value));
-    }
-    kvRowLen(row) += varDataTLen(value);
-  } else {
-    if (offset == 0) {
-      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
-      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
-      memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
-    } else {
-      memcpy(ptr, value, TYPE_BYTES[type]);
-    }
-    kvRowLen(row) += TYPE_BYTES[type];
-  }
-
-  return 0;
-}
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); }
-
-static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) {
-  while (*nIdx < kvRowNCols(row)) {
-    SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx);
-    if (pColIdx->colId == colId) {
-      ++(*nIdx);
-      return tdGetKvRowDataOfCol(row, pColIdx->offset);
-    } else if (pColIdx->colId > colId) {
-      return NULL;
-    } else {
-      ++(*nIdx);
-    }
-  }
-  return NULL;
-}
-#endif
 // ----------------- K-V data row builder
 typedef struct {
   int16_t  tCols;
@@ -632,166 +386,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   return 0;
 }
-#if 0
-// ----------------- SMemRow appended with tuple row structure
-/*
- * |---------|------------------------------------------------- len ---------------------------------->|
- * |<--------     Head      ------>|<---------   flen -------------->|                                 |
- * |---------+---------------------+---------------------------------+---------------------------------+
- * | uint8_t | uint16_t |  int16_t |                                 |                                 |
- * |---------+----------+----------+---------------------------------+---------------------------------+
- * |  flag   |   len    | sversion |           First part            |             Second part         |
- * +---------+----------+----------+---------------------------------+---------------------------------+
- *
- * NOTE: timestamp in this row structure is TKEY instead of TSKEY
- */
-
-// ----------------- SMemRow appended with extended K-V data row structure
-/* |--------------------|------------------------------------------------  len ---------------------------------->|
- * |<-------------     Head      ------------>|<---------   flen -------------->|                                 |
- * |--------------------+----------+--------------------------------------------+---------------------------------+
- * | uint8_t | int16_t  | uint16_t |  int16_t |                                 |                                 |
- * |---------+----------+----------+----------+---------------------------------+---------------------------------+
- * |   flag  | sversion |   len    |   ncols  |           cols index            |             data part           |
- * |---------+----------+----------+----------+---------------------------------+---------------------------------+
- */
-
-typedef void *SMemRow;
-
-#define TD_MEM_ROW_TYPE_SIZE        sizeof(uint8_t)
-#define TD_MEM_ROW_KV_VER_SIZE      sizeof(int16_t)
-#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE)
-#define TD_MEM_ROW_DATA_HEAD_SIZE   (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
-#define TD_MEM_ROW_KV_HEAD_SIZE     (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
-
-#define SMEM_ROW_DATA 0x0U   // SDataRow
-#define SMEM_ROW_KV   0x01U  // SKVRow
-
-#define KVRatioConvert (0.9f)
-
-#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
-
-#define memRowSetType(r, t)  ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
-#define isDataRowT(t)        (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
-#define isDataRow(r)         (SMEM_ROW_DATA == memRowType(r))
-#define isKvRowT(t)          (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
-#define isKvRow(r)           (SMEM_ROW_KV == memRowType(r))
-#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
-
-#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
-#define memRowKvBody(r) \
-  POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE)  // section after flag + sversion as to reuse SKVRow
-
-#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r))  //  0~65535
-#define memRowKvLen(r)   (*(TDRowLenT *)memRowKvBody(r))    //  0~65535
-
-#define memRowDataTLen(r) \
-  ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
-
-#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
-
-#define memRowLen(r)  (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
-#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
-
-static FORCE_INLINE char *memRowEnd(SMemRow row) {
-  if (isDataRow(row)) {
-    return (char *)dataRowEnd(memRowDataBody(row));
-  } else {
-    return (char *)kvRowEnd(memRowKvBody(row));
-  }
-}
-
-#define memRowDataVersion(r)     dataRowVersion(memRowDataBody(r))
-#define memRowKvVersion(r)       (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
-#define memRowVersion(r)         (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
-#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
-#define memRowTuple(r)           (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
-
-#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
-#define memRowKey(r)  (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
-#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
-#define memRowSetTKey(r, k)                 \
-  do {                                      \
-    if (isDataRow(r)) {                     \
-      dataRowTKey(memRowDataBody(r)) = (k); \
-    } else {                                \
-      kvRowTKey(memRowKvBody(r)) = (k);     \
-    }                                       \
-  } while (0)
-
-#define memRowSetLen(r, l)          (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
-#define memRowSetVersion(r, v)      (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
-#define memRowCpy(dst, r)           memcpy((dst), (r), memRowTLen(r))
-#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE)
-#define memRowDeleted(r)            TKEY_IS_DELETED(memRowTKey(r))
-
-SMemRow tdMemRowDup(SMemRow row);
-void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
-
-// NOTE: offset here including the header size
-static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) {
-  if (isDataRow(row)) {
-    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
-  } else {
-    return tdGetKVRowValOfCol(memRowKvBody(row), colId);
-  }
-}
-
-/**
- * NOTE:
- *  1. Applicable to scan columns one by one
- *  2. offset here including the header size
- */
-static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset,
-                                                 int32_t *kvNIdx) {
-  if (isDataRow(row)) {
-    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
-  } else {
-    return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx);
-  }
-}
-
-static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
-                                             int8_t type, int32_t offset) {
-  if (isDataRow(row)) {
-    tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
-  } else {
-    tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
-  }
-  return 0;
-}
-
-// make sure schema->flen appended for SDataRow
-static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) {
-  int32_t len = 0;
-  if (IS_VAR_DATA_TYPE(colType)) {
-    len += varDataTLen(value);
-    if (rowType == SMEM_ROW_KV) {
-      len += sizeof(SColIdx);
-    }
-  } else {
-    if (rowType == SMEM_ROW_KV) {
-      len += TYPE_BYTES[colType];
-      len += sizeof(SColIdx);
-    }
-  }
-  return len;
-}
-
-typedef struct {
-  int16_t colId;
-  uint8_t colType;
-  char *  colVal;
-} SColInfo;
-
-static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) {
-  colInfo->colId = colId;
-  colInfo->colType = colType;
-  colInfo->colVal = colVal;
-}
-
-SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
-#endif
 
 #ifdef __cplusplus
 }

include/common/tglobal.h

@@ -121,6 +121,9 @@ extern char     tsCompressor[];
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
+// udf
+extern bool  tsStartUdfd;
+
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;

include/common/tmsg.h

@@ -99,7 +99,7 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_VGROUP,
   TSDB_MGMT_TABLE_TOPICS,
   TSDB_MGMT_TABLE_CONSUMERS,
-  TSDB_MGMT_TABLE_SUBSCRIBES,
+  TSDB_MGMT_TABLE_SUBSCRIPTIONS,
   TSDB_MGMT_TABLE_TRANS,
   TSDB_MGMT_TABLE_SMAS,
   TSDB_MGMT_TABLE_CONFIGS,
@@ -131,12 +131,10 @@ typedef enum _mgmt_table {
 #define TSDB_ALTER_USER_SUPERUSER       0x2
 #define TSDB_ALTER_USER_ADD_READ_DB     0x3
 #define TSDB_ALTER_USER_REMOVE_READ_DB  0x4
-#define TSDB_ALTER_USER_CLEAR_READ_DB   0x5
+#define TSDB_ALTER_USER_ADD_WRITE_DB    0x5
-#define TSDB_ALTER_USER_ADD_WRITE_DB    0x6
+#define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x6
-#define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x7
+#define TSDB_ALTER_USER_ADD_ALL_DB      0x7
-#define TSDB_ALTER_USER_CLEAR_WRITE_DB  0x8
+#define TSDB_ALTER_USER_REMOVE_ALL_DB   0x8
-#define TSDB_ALTER_USER_ADD_ALL_DB      0x9
-#define TSDB_ALTER_USER_REMOVE_ALL_DB   0xA
 
 #define TSDB_ALTER_USER_PRIVILEGES 0x2
 
@@ -330,6 +328,9 @@ typedef struct {
   int8_t  alterType;
   int32_t numOfFields;
   SArray* pFields;
+  int32_t ttl;
+  int32_t commentLen;
+  char*   comment;
 } SMAlterStbReq;
 
 int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq);
@@ -1457,7 +1458,7 @@ typedef struct {
 static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
   SMqRebInfo* pRebInfo = (SMqRebInfo*)taosMemoryCalloc(1, sizeof(SMqRebInfo));
   if (pRebInfo == NULL) {
-    goto _err;
+    return NULL;
   }
   strcpy(pRebInfo->key, key);
   pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t));
@@ -1492,7 +1493,7 @@ typedef struct {
 } SMVSubscribeRsp;
 
 typedef struct {
-  char   name[TSDB_TABLE_FNAME_LEN];
+  char   name[TSDB_TOPIC_FNAME_LEN];
   int8_t igNotExists;
 } SMDropTopicReq;
 

include/common/tmsgcb.h

@@ -57,6 +57,7 @@ typedef struct {
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
   ReportStartup           reportStartupFp;
+  void*                   clientRpc;
 } SMsgCb;
 
 void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);

include/common/tname.h

@@ -16,8 +16,8 @@
 #ifndef _TD_COMMON_NAME_H_
 #define _TD_COMMON_NAME_H_
 
-#include "tdef.h"
 #include "tarray.h"
+#include "tdef.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,19 +65,19 @@ bool tNameDBNameEqual(SName* left, SName* right);
 
 typedef struct {
   // input
-  SArray         *tags;         // element is SSmlKV
+  SArray*     tags;           // element is SSmlKv
-  const char     *sTableName;   // super table name
+  const char* sTableName;     // super table name
-  uint8_t        sTableNameLen; // the length of super table name
+  uint8_t     sTableNameLen;  // the length of super table name
 
   // output
-  char           *childTableName;   // must have size of TSDB_TABLE_NAME_LEN;
+  char*    childTableName;  // must have size of TSDB_TABLE_NAME_LEN;
-  uint64_t       uid;               // child table uid, may be useful
+  uint64_t uid;             // child table uid, may be useful
 } RandTableName;
 
-void buildChildTableName(RandTableName *rName);
+void buildChildTableName(RandTableName* rName);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  /*_TD_COMMON_NAME_H_*/
+#endif /*_TD_COMMON_NAME_H_*/

include/libs/executor/executor.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "query.h"
 #include "tcommon.h"
+#include "tmsgcb.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -29,11 +30,12 @@ struct SRpcMsg;
 struct SSubplan;
 
 typedef struct SReadHandle {
-  void* reader;
+  void*   reader;
-  void* meta;
+  void*   meta;
-  void* config;
+  void*   config;
-  void* vnode;
+  void*   vnode;
-  void* mnd;
+  void*   mnd;
+  SMsgCb* pMsgCb;
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1

include/libs/function/function.h

@@ -336,6 +336,17 @@ int32_t udfcOpen();
  */
 int32_t udfcClose();
 
+/**
+ * start udfd that serves udf function invocation under dnode startDnodeId
+ * @param startDnodeId
+ * @return
+ */
+int32_t udfStartUdfd(int32_t startDnodeId);
+/**
+ * stop udfd
+ * @return
+ */
+int32_t udfStopUdfd();
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -31,13 +31,13 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ELAPSED,
   FUNCTION_TYPE_IRATE,
   FUNCTION_TYPE_LAST_ROW,
-  FUNCTION_TYPE_LEASTSQUARES,
   FUNCTION_TYPE_MAX,
   FUNCTION_TYPE_MIN,
   FUNCTION_TYPE_MODE,
   FUNCTION_TYPE_PERCENTILE,
   FUNCTION_TYPE_SPREAD,
   FUNCTION_TYPE_STDDEV,
+  FUNCTION_TYPE_LEASTSQUARES,
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
   FUNCTION_TYPE_HISTOGRAM,
@@ -55,6 +55,8 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_TAIL,
   FUNCTION_TYPE_TOP,
   FUNCTION_TYPE_UNIQUE,
+  FUNCTION_TYPE_STATE_COUNT,
+  FUNCTION_TYPE_STATE_DURATION,
 
   // math function
   FUNCTION_TYPE_ABS = 1000,
@@ -154,6 +156,7 @@ bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);
 bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
 bool fmIsMultiResFunc(int32_t funcId);
+bool fmIsRepeatScanFunc(int32_t funcId);
 bool fmIsUserDefinedFunc(int32_t funcId);
 
 typedef enum EFuncDataRequired {
@@ -170,7 +173,7 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
 int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet);
-bool fmIsInvertible(int32_t funcId);
+bool    fmIsInvertible(int32_t funcId);
 
 #ifdef __cplusplus
 }

include/libs/nodes/nodes.h

@@ -179,6 +179,7 @@ typedef enum ENodeType {
   QUERY_NODE_KILL_CONNECTION_STMT,
   QUERY_NODE_KILL_QUERY_STMT,
   QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_QUERY,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -206,6 +207,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
   QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_FILL,
   QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,

include/libs/nodes/plannodes.h

@@ -274,6 +274,8 @@ typedef struct SIntervalPhysiNode {
   int8_t           slidingUnit;
 } SIntervalPhysiNode;
 
+typedef SIntervalPhysiNode SStreamIntervalPhysiNode;
+
 typedef struct SFillPhysiNode {
   SPhysiNode  node;
   EFillMode   mode;

include/libs/nodes/querynodes.h

@@ -233,6 +233,7 @@ typedef struct SSelectStmt {
   uint8_t     precision;
   bool        isEmptyResult;
   bool        hasAggFuncs;
+  bool        hasRepeatScanFuncs;
   bool        isTimeOrderQuery;
 } SSelectStmt;
 
@@ -294,6 +295,37 @@ typedef struct SExplainStmt {
   SNode*           pQuery;
 } SExplainStmt;
 
+typedef struct SCmdMsgInfo {
+  int16_t msgType;
+  SEpSet  epSet;
+  void*   pMsg;
+  int32_t msgLen;
+  void*   pExtension;  // todo remove it soon
+} SCmdMsgInfo;
+
+typedef enum EQueryExecMode {
+  QUERY_EXEC_MODE_LOCAL = 1,
+  QUERY_EXEC_MODE_RPC,
+  QUERY_EXEC_MODE_SCHEDULE,
+  QUERY_EXEC_MODE_EMPTY_RESULT
+} EQueryExecMode;
+
+typedef struct SQuery {
+  ENodeType      type;
+  EQueryExecMode execMode;
+  bool           haveResultSet;
+  SNode*         pRoot;
+  int32_t        numOfResCols;
+  SSchema*       pResSchema;
+  int8_t         precision;
+  SCmdMsgInfo*   pCmdMsg;
+  int32_t        msgType;
+  SArray*        pDbList;
+  SArray*        pTableList;
+  bool           showRewrite;
+  int32_t        placeholderNum;
+} SQuery;
+
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 

include/libs/parser/parser.h

@@ -48,36 +48,6 @@ typedef struct SParseContext {
   bool             isSuperUser;
 } SParseContext;
 
-typedef struct SCmdMsgInfo {
-  int16_t msgType;
-  SEpSet  epSet;
-  void*   pMsg;
-  int32_t msgLen;
-  void*   pExtension;  // todo remove it soon
-} SCmdMsgInfo;
-
-typedef enum EQueryExecMode {
-  QUERY_EXEC_MODE_LOCAL = 1,
-  QUERY_EXEC_MODE_RPC,
-  QUERY_EXEC_MODE_SCHEDULE,
-  QUERY_EXEC_MODE_EMPTY_RESULT
-} EQueryExecMode;
-
-typedef struct SQuery {
-  EQueryExecMode execMode;
-  bool           haveResultSet;
-  SNode*         pRoot;
-  int32_t        numOfResCols;
-  SSchema*       pResSchema;
-  int8_t         precision;
-  SCmdMsgInfo*   pCmdMsg;
-  int32_t        msgType;
-  SArray*        pDbList;
-  SArray*        pTableList;
-  bool           showRewrite;
-  int32_t        placeholderNum;
-} SQuery;
-
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
 bool    isInsertSql(const char* pStr, size_t length);
 
@@ -103,9 +73,10 @@ void    destroyBoundColumnInfo(void* pBoundInfo);
 int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
                      int32_t msgBufLen);
 
-void*   smlInitHandle(SQuery *pQuery);
+void*   smlInitHandle(SQuery* pQuery);
-void    smlDestroyHandle(void *pHandle);
+void    smlDestroyHandle(void* pHandle);
-int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *colsSchema, SArray *cols, bool format, STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen);
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* colsSchema, SArray* cols, bool format,
+                    STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 #ifdef __cplusplus

include/libs/qcom/query.h

@@ -24,6 +24,7 @@ extern "C" {
 #include "thash.h"
 #include "tlog.h"
 #include "tmsg.h"
+#include "tmsgcb.h"
 
 typedef enum {
   JOB_TASK_STATUS_NULL = 0,
@@ -149,7 +150,8 @@ int32_t cleanupTaskQueue();
  */
 int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
 
-int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx);
+int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo,
+                                bool persistHandle, void* ctx);
 
 /**
  * Asynchronously send message to server, after the response received, the callback will be incured.
@@ -182,7 +184,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
+#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
 #define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

include/libs/stream/tstream.h

@@ -16,6 +16,7 @@
 #include "tdatablock.h"
 #include "tmsg.h"
 #include "tmsgcb.h"
+#include "tqueue.h"
 #include "trpc.h"
 
 #ifdef __cplusplus
@@ -154,6 +155,10 @@ struct SStreamTask {
     STaskDispatcherShuffle shuffleDispatcher;
   };
 
+  // msg buffer
+  int32_t     memUsed;
+  STaosQueue* inputQ;
+
   // application storage
   void* ahandle;
 };
@@ -194,6 +199,8 @@ typedef struct {
   SArray* res;  // SArray<SSDataBlock>
 } SStreamSinkReq;
 
+int32_t streamEnqueueData(SStreamTask* pTask, const void* input, int32_t inputType);
+
 int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
 
 #ifdef __cplusplus

include/os/osFile.h

@@ -80,6 +80,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
 void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
 
 int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf);
+int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf);
 
 int32_t taosEOFFile(TdFilePtr pFile);
  

include/os/osMemory.h

@@ -35,6 +35,7 @@ void *taosMemoryRealloc(void *ptr, int32_t size);
 void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(void *ptr);
 int32_t taosMemorySize(void *ptr);
+void taosPrintBackTrace();
 
 #define taosMemoryFreeClear(ptr)   \
   do {                             \

include/util/taoserror.h

@@ -88,6 +88,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x0114)
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x0115)
 #define TSDB_CODE_DUP_KEY                       TAOS_DEF_ERROR_CODE(0, 0x0116)
+#define TSDB_CODE_NEED_RETRY                    TAOS_DEF_ERROR_CODE(0, 0x0117)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0140)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0141)
@@ -632,6 +633,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN    TAOS_DEF_ERROR_CODE(0, 0x2642)
 #define TSDB_CODE_PAR_INVALID_TAGS_NUM          TAOS_DEF_ERROR_CODE(0, 0x2643)
 #define TSDB_CODE_PAR_PERMISSION_DENIED         TAOS_DEF_ERROR_CODE(0, 0x2644)
+#define TSDB_CODE_PAR_INVALID_STREAM_QUERY      TAOS_DEF_ERROR_CODE(0, 0x2645)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tdef.h

@@ -86,10 +86,9 @@ extern const int32_t TYPE_BYTES[15];
 #define TS_PATH_DELIMITER "."
 #define TS_ESCAPE_CHAR    '`'
 
-
+#define TSDB_TIME_PRECISION_MILLI   0
-#define TSDB_TIME_PRECISION_MILLI 0
+#define TSDB_TIME_PRECISION_MICRO   1
-#define TSDB_TIME_PRECISION_MICRO 1
+#define TSDB_TIME_PRECISION_NANO    2
-#define TSDB_TIME_PRECISION_NANO  2
 #define TSDB_TIME_PRECISION_HOURS   3
 #define TSDB_TIME_PRECISION_MINUTES 4
 #define TSDB_TIME_PRECISION_SECONDS 5
@@ -249,7 +248,6 @@ typedef enum ELogicConditionType {
 #define TSDB_SHOW_SQL_LEN       512
 #define TSDB_SLOW_QUERY_SQL_LEN 512
 #define TSDB_SHOW_SUBQUERY_LEN  1000
-#define TSDB_SHOW_LIST_LEN      1000
 
 #define TSDB_TRANS_STAGE_LEN 12
 #define TSDB_TRANS_TYPE_LEN  16
@@ -376,9 +374,9 @@ typedef enum ELogicConditionType {
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag
  */
-#define TSDB_QUERY_TYPE_INSERT           0x100u  // insert type
+#define TSDB_QUERY_TYPE_INSERT      0x100u  // insert type
-#define TSDB_QUERY_TYPE_FILE_INSERT      0x400u   // insert data from file
+#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u  // insert data from file
-#define TSDB_QUERY_TYPE_STMT_INSERT      0x800u   // stmt insert type
+#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u  // stmt insert type
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)   (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)   ((x) |= (_type))

include/util/tskiplist2.h

@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_UTIL_SKIPLIST2_H_
-#define _TD_UTIL_SKIPLIST2_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define SL_MAX_LEVEL 15
-
-typedef struct SSkipList2 SSkipList2;
-typedef struct SSLCursor  SSLCursor;
-typedef struct SSLCfg     SSLCfg;
-typedef struct SSLNode    SSLNode;
-
-typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
-
-// SSkipList2
-int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
-int32_t slClose(SSkipList2 *pSl);
-int32_t slClear(SSkipList2 *pSl);
-
-// SSLCursor
-int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
-int32_t slcClose(SSLCursor *pSlc);
-int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey);
-int32_t slcMoveToNext(SSLCursor *pSlc);
-int32_t slcMoveToPrev(SSLCursor *pSlc);
-int32_t slcMoveToFirst(SSLCursor *pSlc);
-int32_t slcMoveToLast(SSLCursor *pSlc);
-int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData);
-int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData);
-int32_t slcDrop(SSLCursor *pSlc);
-
-// struct
-struct SSLCfg {
-  int8_t    maxLevel;
-  int32_t   nKey;
-  int32_t   nData;
-  tslCmprFn cmprFn;
-  void     *pPool;
-  void *(*xMalloc)(void *, int32_t size);
-  void (*xFree)(void *, void *);
-};
-
-struct SSLCursor {
-  SSkipList2 *pSl;
-  SSLNode   **forwards[SL_MAX_LEVEL];
-};
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_SKIPLIST2_H_*/

source/client/inc/clientInt.h

@@ -254,7 +254,7 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
+void*    createTscObj(const char* user, const char* auth, const char* db, int32_t connType, SAppInstInfo* pAppInfo);
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);
 int32_t  releaseTscObj(int64_t rid);
@@ -310,6 +310,7 @@ void hbMgrInitMqHbRspHandle();
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
 int32_t      getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
 int32_t      scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
+int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
 
 #ifdef __cplusplus
 }

source/client/inc/clientStmt.h

@@ -60,6 +60,7 @@ typedef struct SStmtBindInfo {
   int32_t      sBindRowNum;
   int32_t      sBindLastIdx;
   int8_t       tbType;
+  bool         tagsCached;
   void*        boundTags;  
   char*        tbName;
   SName        sname;

source/client/src/clientEnv.c

@@ -131,7 +131,7 @@ void destroyTscObj(void *pObj) {
   taosMemoryFreeClear(pTscObj);
 }
 
-void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) {
+void *createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo) {
   STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -145,6 +145,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
     return NULL;
   }
 
+  pObj->connType = connType;
   pObj->pAppInfo = pAppInfo;
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);

source/client/src/clientImpl.c

@@ -25,7 +25,7 @@
 #include "tref.h"
 
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
-static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType);
+static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
 static void          destroySendMsgInfo(SMsgSendInfo* pMsgBody);
 
 static bool stringLengthCheck(const char* str, size_t maxsize) {
@@ -491,7 +491,7 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
 
 STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
                          SAppInstInfo* pAppInfo, int connType) {
-  STscObj* pTscObj = createTscObj(user, auth, db, pAppInfo);
+  STscObj* pTscObj = createTscObj(user, auth, db, connType, pAppInfo);
   if (NULL == pTscObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return pTscObj;
@@ -504,7 +504,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
     return NULL;
   }
 
-  SMsgSendInfo* body = buildConnectMsg(pRequest, connType);
+  SMsgSendInfo* body = buildConnectMsg(pRequest);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
@@ -527,7 +527,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
   return pTscObj;
 }
 
-static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) {
+static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (pMsgSendInfo == NULL) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -550,9 +550,10 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) {
   }
   taosMemoryFreeClear(db);
 
-  connectReq.connType = connType;
+  connectReq.connType  = pObj->connType;
-  connectReq.pid = htonl(appInfo.pid);
+  connectReq.pid       = htonl(appInfo.pid);
   connectReq.startTime = htobe64(appInfo.startTime);
+
   tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
   tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user));
   tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd));

source/client/src/clientMain.c

@@ -563,7 +563,11 @@ const char *taos_get_server_info(TAOS *taos) {
 }
 
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
-  // TODO
+  if (taos == NULL || sql == NULL) {
+    // todo directly call fp
+  }
+
+  taos_query_l(taos, sql, (int32_t) strlen(sql));
 }
 
 void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {

source/client/src/clientSml.c

@@ -401,7 +401,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
 
     code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta);
 
-    if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) {
+    if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) {
       SSchemaAction schemaAction = {.action = SCHEMA_ACTION_CREATE_STABLE};
       memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen);
       schemaAction.createSTable.tags = sTableData->tags;

source/client/src/clientStmt.c

@@ -29,6 +29,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
       if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) {
         code = TSDB_CODE_TSC_STMT_API_ERROR;
       }
+/*      
+      if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+*/
       break;
     case STMT_BIND_COL:
       if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) {
@@ -123,6 +128,7 @@ int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
   pStmt->bInfo.tbSuid = pTableMeta->suid;
   pStmt->bInfo.tbType = pTableMeta->tableType;
   pStmt->bInfo.boundTags = tags;
+  pStmt->bInfo.tagsCached = false;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -207,8 +213,6 @@ int32_t stmtParseSql(STscStmt* pStmt) {
       STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR);
   }
 
-  STMT_ERR_RET(stmtCacheBlock(pStmt));
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -219,8 +223,10 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
   pStmt->bInfo.needParse = true;
 
   taosMemoryFreeClear(pStmt->bInfo.tbName);
-  destroyBoundColumnInfo(pStmt->bInfo.boundTags);
+  if (!pStmt->bInfo.tagsCached) {
-  taosMemoryFreeClear(pStmt->bInfo.boundTags);
+    destroyBoundColumnInfo(pStmt->bInfo.boundTags);
+    taosMemoryFreeClear(pStmt->bInfo.boundTags);
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -275,6 +281,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
 
     qDestroyStmtDataBlock(pCache->pDataBlock);
     destroyBoundColumnInfo(pCache->boundTags);
+    taosMemoryFreeClear(pCache->boundTags);
     
     pIter = taosHashIterate(pStmt->sql.pTableCache, pIter);
   }
@@ -302,7 +309,15 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
 
   STableMeta *pTableMeta = NULL;
   SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
-  STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+  int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta);
+  if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
+    STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STMT_ERR_RET(code);
+  
   uint64_t uid = pTableMeta->uid;
   uint64_t suid = pTableMeta->suid;
   int8_t tableType = pTableMeta->tableType;
@@ -328,6 +343,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
     pStmt->bInfo.tbSuid = suid;
     pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
+    pStmt->bInfo.tagsCached = true;
 
     return TSDB_CODE_SUCCESS;
   }
@@ -340,6 +356,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
     pStmt->bInfo.tbSuid = suid;
     pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
+    pStmt->bInfo.tagsCached = true;
 
     STableDataBlocks* pNewBlock = NULL;
     STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock));
@@ -448,10 +465,12 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
 
-  if (pStmt->bInfo.needParse) {
+  if (!pStmt->bInfo.needParse) {
-    STMT_ERR_RET(stmtParseSql(pStmt));
+    return TSDB_CODE_SUCCESS;
   }
 
+  STMT_ERR_RET(stmtParseSql(pStmt));
+
   STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
   if (NULL == pDataBlock) {
     tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
@@ -501,8 +520,6 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel
 int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
-
   if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
     pStmt->bInfo.needParse = false;
   }
@@ -520,6 +537,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
     STMT_ERR_RET(stmtParseSql(pStmt));
   }
 
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     if (NULL == pStmt->sql.pQueryPlan) {
       STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
@@ -586,6 +605,16 @@ int stmtExec(TAOS_STMT *stmt) {
     STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
     launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
   }
+
+  if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) {
+    code = refreshMeta(pStmt->exec.pRequest->pTscObj, pStmt->exec.pRequest);
+    if (code) {
+      pStmt->exec.pRequest->code = code;
+    } else {
+      STMT_ERR_RET(stmtResetStmt(pStmt));
+      STMT_ERR_RET(TSDB_CODE_NEED_RETRY);
+    }
+  }
   
   STMT_ERR_JRET(pStmt->exec.pRequest->code);
 
@@ -613,13 +642,11 @@ int stmtClose(TAOS_STMT *stmt) {
 const char *stmtErrstr(TAOS_STMT *stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  if (stmt == NULL) {
+  if (stmt == NULL || NULL == pStmt->exec.pRequest) {
     return (char*) tstrerror(terrno);
   }
 
-  if (pStmt->exec.pRequest) {
+  pStmt->exec.pRequest->code = terrno;
-    pStmt->exec.pRequest->code = terrno;
-  }
 
   return taos_errstr(pStmt->exec.pRequest);
 }

source/client/src/tmq.c

@@ -187,7 +187,7 @@ typedef struct {
 
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
-  conf->autoCommit = false;
+  conf->autoCommit = true;
   conf->autoCommitInterval = 5000;
   conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
   return conf;
@@ -395,7 +395,7 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
   }
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
-    tmq_list_append(*topics, topic->topicName);
+    tmq_list_append(*topics, strchr(topic->topicName, '.') + 1);
   }
   return TMQ_RESP_ERR__SUCCESS;
 }
@@ -1307,7 +1307,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
 }
 
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
-  // TODO
+  if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
+    tmq_list_t*    lst = tmq_list_new();
+    tmq_resp_err_t rsp = tmq_subscribe(tmq, lst);
+    tmq_list_destroy(lst);
+    if (rsp == TMQ_RESP_ERR__SUCCESS) {
+      // TODO: free resources
+      return TMQ_RESP_ERR__SUCCESS;
+    } else {
+      return TMQ_RESP_ERR__FAIL;
+    }
+  }
+  // TODO: free resources
   return TMQ_RESP_ERR__SUCCESS;
 }
 

source/common/src/systable.c

@@ -14,9 +14,9 @@
  */
 
 #include "systable.h"
+#include "taos.h"
 #include "tdef.h"
 #include "types.h"
-#include "taos.h"
 
 #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
 #define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
@@ -262,10 +262,10 @@ static const SSysDbTableSchema topicSchema[] = {
 
 static const SSysDbTableSchema consumerSchema[] = {
     {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "consumer_group", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
@@ -274,8 +274,8 @@ static const SSysDbTableSchema consumerSchema[] = {
 };
 
 static const SSysDbTableSchema subscriptionSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "topic_name", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "consumer_group", .bytes = TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
 };

source/common/src/tdata.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */

source/common/src/tdatablock.c

@@ -493,7 +493,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
     for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
       bool isNull = false;
       if (pBlock->pBlockAgg == NULL) {
-        isNull = colDataIsNull_s(pColData, pBlock->info.rows);
+        isNull = colDataIsNull_s(pColData, j);
       } else {
         isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
       }

source/common/src/tdataformat.c

@@ -129,50 +129,6 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
   return buf;
 }
 
-#if 0
-int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) {
-  if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1;
-  if (tEncodeI8(pEncoder, pCol->type) < 0) return -1;
-  if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1;
-  if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1;
-  if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1;
-  return pEncoder->pos;
-}
-
-int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) {
-  if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1;
-  return 0;
-}
-
-int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) {
-  if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1;
-  if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1;
-  if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1;
-  if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1;
-  if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1;
-
-  for (int32_t i = 0; i < schemaNCols(pSchema); i++) {
-    const STColumn *pCol = schemaColAt(pSchema, i);
-    if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1;
-  }
-  return 0;
-}
-
-int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) {
-  if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1;
-  if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1;
-  if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1;
-
-  return 0;
-}
-#endif
-
 int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   if (pBuilder == NULL) return -1;
 
@@ -260,49 +216,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   return pSchema;
 }
 
-#if 0
-/**
- * Initialize a data row
- */
-void tdInitDataRow(SDataRow row, STSchema *pSchema) {
-  dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema));
-  dataRowSetVersion(row, schemaVersion(pSchema));
-}
-
-SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
-  int32_t size = dataRowMaxBytesFromSchema(pSchema);
-
-  SDataRow row = taosMemoryMalloc(size);
-  if (row == NULL) return NULL;
-
-  tdInitDataRow(row, pSchema);
-  return row;
-}
-
-/**
- * Free the SDataRow object
- */
-void tdFreeDataRow(SDataRow row) {
-  if (row) taosMemoryFree(row);
-}
-
-SDataRow tdDataRowDup(SDataRow row) {
-  SDataRow trow = taosMemoryMalloc(dataRowLen(row));
-  if (trow == NULL) return NULL;
-
-  dataRowCpy(trow, row);
-  return trow;
-}
-
-SMemRow tdMemRowDup(SMemRow row) {
-  SMemRow trow = taosMemoryMalloc(memRowTLen(row));
-  if (trow == NULL) return NULL;
-
-  memRowCpy(trow, row);
-  return trow;
-}
-#endif
-
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->type = colType(pCol);
   pDataCol->colId = colColId(pCol);
@@ -312,39 +225,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->len = 0;
 }
 
-#if 0
-// value from timestamp should be TKEY here instead of TSKEY
-int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
-  ASSERT(pCol != NULL && value != NULL);
-
-  if (isAllRowsNull(pCol)) {
-    if (isNull(value, pCol->type)) {
-      // all null value yet, just return
-      return 0;
-    }
-
-    if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
-    if (numOfRows > 0) {
-      // Find the first not null value, fill all previouse values as NULL
-      dataColSetNEleNull(pCol, numOfRows);
-    }
-  }
-
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    // set offset
-    pCol->dataOff[numOfRows] = pCol->len;
-    // Copy data
-    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
-    // Update the length
-    pCol->len += varDataTLen(value);
-  } else {
-    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
-    pCol->len += pCol->bytes;
-  }
-  return 0;
-}
-#endif
 static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
@@ -361,31 +241,6 @@ bool isNEleNull(SDataCol *pCol, int nEle) {
   return true;
 }
 
-#if 0
-static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    pCol->dataOff[index] = pCol->len;
-    char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
-    setVardataNull(ptr, pCol->type);
-    pCol->len += varDataTLen(ptr);
-  } else {
-    setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
-    pCol->len += TYPE_BYTES[pCol->type];
-  }
-}
-
-static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) {
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    pCol->len = 0;
-    for (int i = 0; i < nEle; ++i) {
-      dataColSetNullAt(pCol, i);
-    }
-  } else {
-    setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
-    pCol->len = TYPE_BYTES[pCol->type] * nEle;
-  }
-}
-#endif
 void *dataColSetOffset(SDataCol *pCol, int nEle) {
   ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
 
@@ -483,42 +338,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
   return NULL;
 }
 
-#if 0
-SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
-  SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
-  if (pRet == NULL) return NULL;
-
-  pRet->numOfCols = pDataCols->numOfCols;
-  pRet->sversion = pDataCols->sversion;
-  if (keepData) pRet->numOfRows = pDataCols->numOfRows;
-
-  for (int i = 0; i < pDataCols->numOfCols; i++) {
-    pRet->cols[i].type = pDataCols->cols[i].type;
-    pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
-    pRet->cols[i].colId = pDataCols->cols[i].colId;
-    pRet->cols[i].bytes = pDataCols->cols[i].bytes;
-    pRet->cols[i].offset = pDataCols->cols[i].offset;
-
-    if (keepData) {
-      if (pDataCols->cols[i].len > 0) {
-        if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
-          tdFreeDataCols(pRet);
-          return NULL;
-        }
-        pRet->cols[i].len = pDataCols->cols[i].len;
-        memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
-        if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
-          int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints;
-          memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
-        }
-      }
-    }
-  }
-
-  return pRet;
-}
-#endif
-
 void tdResetDataCols(SDataCols *pCols) {
   if (pCols != NULL) {
     pCols->numOfRows = 0;
@@ -528,180 +347,6 @@ void tdResetDataCols(SDataCols *pCols) {
     }
   }
 }
-#if 0
-static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
-  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
-
-  int rcol = 0;
-  int dcol = 0;
-
-  while (dcol < pCols->numOfCols) {
-    bool setCol = 0;
-    SDataCol *pDataCol = &(pCols->cols[dcol]);
-    if (rcol >= schemaNCols(pSchema)) {
-      dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
-      dcol++;
-      continue;
-    }
-
-    STColumn *pRowCol = schemaColAt(pSchema, rcol);
-    if (pRowCol->colId == pDataCol->colId) {
-      void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
-      if(!isNull(value, pDataCol->type)) setCol = 1;
-      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
-      dcol++;
-      rcol++;
-    } else if (pRowCol->colId < pDataCol->colId) {
-      rcol++;
-    } else {
-      if(forceSetNull || setCol) {
-        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
-      }
-      dcol++;
-    }
-  }
-  pCols->numOfRows++;
-}
-
-static void tdAppendKVRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
-  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row));
-
-  int rcol = 0;
-  int dcol = 0;
-
-  int nRowCols = kvRowNCols(row);
-
-  while (dcol < pCols->numOfCols) {
-    bool setCol = 0;
-    SDataCol *pDataCol = &(pCols->cols[dcol]);
-    if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) {
-      dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
-      ++dcol;
-      continue;
-    }
-
-    SColIdx *colIdx = kvRowColIdxAt(row, rcol);
-
-    if (colIdx->colId == pDataCol->colId) {
-      void *value = tdGetKvRowDataOfCol(row, colIdx->offset);
-      if(!isNull(value, pDataCol->type)) setCol = 1;
-      dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
-      ++dcol;
-      ++rcol;
-    } else if (colIdx->colId < pDataCol->colId) {
-      ++rcol;
-    } else {
-      if(forceSetNull || setCol) {
-        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
-      }
-      ++dcol;
-    }
-  }
-  pCols->numOfRows++;
-}
-
-void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
-  if (isDataRow(row)) {
-    tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull);
-  } else if (isKvRow(row)) {
-    tdAppendKVRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull);
-  } else {
-    ASSERT(0);
-  }
-}
-
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) {
-  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
-  ASSERT(target->numOfCols == source->numOfCols);
-  int offset = 0;
-
-  if (pOffset == NULL) {
-    pOffset = &offset;
-  }
-
-  SDataCols *pTarget = NULL;
-
-  if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) {  // No overlap
-    ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
-    for (int i = 0; i < rowsToMerge; i++) {
-      for (int j = 0; j < source->numOfCols; j++) {
-        if (source->cols[j].len > 0 || target->cols[j].len > 0) {
-          dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i + (*pOffset)), target->numOfRows,
-                           target->maxPoints);
-        }
-      }
-      target->numOfRows++;
-    }
-    (*pOffset) += rowsToMerge;
-  } else {
-    pTarget = tdDupDataCols(target, true);
-    if (pTarget == NULL) goto _err;
-
-    int iter1 = 0;
-    tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows,
-                       pTarget->numOfRows + rowsToMerge, forceSetNull);
-  }
-
-  tdFreeDataCols(pTarget);
-  return 0;
-
-_err:
-  tdFreeDataCols(pTarget);
-  return -1;
-}
-
-// src2 data has more priority than src1
-static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
-                               int limit2, int tRows, bool forceSetNull) {
-  tdResetDataCols(target);
-  ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
-
-  while (target->numOfRows < tRows) {
-    if (*iter1 >= limit1 && *iter2 >= limit2) break;
-
-    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
-    TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
-    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
-    TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
-
-    ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
-
-    if (key1 < key2) {
-      for (int i = 0; i < src1->numOfCols; i++) {
-        ASSERT(target->cols[i].type == src1->cols[i].type);
-        if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
-          dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
-                           target->maxPoints);
-        }
-      }
-
-      target->numOfRows++;
-      (*iter1)++;
-    } else if (key1 >= key2) {
-      if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
-        for (int i = 0; i < src2->numOfCols; i++) {
-          ASSERT(target->cols[i].type == src2->cols[i].type);
-          if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
-            dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
-                             target->maxPoints);
-          } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
-            dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
-                             target->maxPoints);
-          } else if(target->cols[i].len > 0) {
-            dataColSetNullAt(&target->cols[i], target->numOfRows);
-          }
-        }
-        target->numOfRows++;
-      }
-
-      (*iter2)++;
-      if (key1 == key2) (*iter1)++;
-    }
-
-    ASSERT(target->numOfRows <= target->maxPoints);
-  }
-}
-#endif
 
 SKVRow tdKVRowDup(SKVRow row) {
   SKVRow trow = taosMemoryMalloc(kvRowLen(row));
@@ -859,98 +504,3 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
 
   return row;
 }
-#if 0
-SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
-#if 0
-  ASSERT(memRowKey(row1) == memRowKey(row2));
-  ASSERT(schemaVersion(pSchema1) == memRowVersion(row1));
-  ASSERT(schemaVersion(pSchema2) == memRowVersion(row2));
-  ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2));
-#endif
-
-  SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo));
-  if (stashRow == NULL) {
-    return NULL;
-  }
-
-  SMemRow  pRow = buffer;
-  SDataRow dataRow = memRowDataBody(pRow);
-  memRowSetType(pRow, SMEM_ROW_DATA);
-  dataRowSetVersion(dataRow, schemaVersion(pSchema1));  // use latest schema version
-  dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen));
-
-  TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE;
-
-  int32_t  i = 0;  // row1
-  int32_t  j = 0;  // row2
-  int32_t  nCols1 = schemaNCols(pSchema1);
-  int32_t  nCols2 = schemaNCols(pSchema2);
-  SColInfo colInfo = {0};
-  int32_t  kvIdx1 = 0, kvIdx2 = 0;
-
-  while (i < nCols1) {
-    STColumn *pCol = schemaColAt(pSchema1, i);
-    void *    val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1);
-    // if val1 != NULL, use val1;
-    if (val1 != NULL && !isNull(val1, pCol->type)) {
-      tdAppendColVal(dataRow, val1, pCol->type, pCol->offset);
-      kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type);
-      setSColInfo(&colInfo, pCol->colId, pCol->type, val1);
-      taosArrayPush(stashRow, &colInfo);
-      ++i;  // next col
-      continue;
-    }
-
-    void *val2 = NULL;
-    while (j < nCols2) {
-      STColumn *tCol = schemaColAt(pSchema2, j);
-      if (tCol->colId < pCol->colId) {
-        ++j;
-        continue;
-      }
-      if (tCol->colId == pCol->colId) {
-        val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2);
-      } else if (tCol->colId > pCol->colId) {
-        // set NULL
-      }
-      break;
-    }  // end of while(j<nCols2)
-    if (val2 == NULL) {
-      val2 = (void *)getNullValue(pCol->type);
-    }
-    tdAppendColVal(dataRow, val2, pCol->type, pCol->offset);
-    if (!isNull(val2, pCol->type)) {
-      kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type);
-      setSColInfo(&colInfo, pCol->colId, pCol->type, val2);
-      taosArrayPush(stashRow, &colInfo);
-    }
-
-    ++i;  // next col
-  }
-
-  dataLen = memRowTLen(pRow);
-
-  if (kvLen < dataLen) {
-    // scan stashRow and generate SKVRow
-    memset(buffer, 0, sizeof(dataLen));
-    SMemRow tRow = buffer;
-    memRowSetType(tRow, SMEM_ROW_KV);
-    SKVRow kvRow = (SKVRow)memRowKvBody(tRow);
-    int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow);
-    kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols));
-    kvRowSetNCols(kvRow, nKvNCols);
-    memRowSetKvVersion(tRow, pSchema1->version);
-
-    int32_t toffset = 0;
-    int16_t k;
-    for (k = 0; k < nKvNCols; ++k) {
-      SColInfo *pColInfo = taosArrayGet(stashRow, k);
-      tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
-      toffset += sizeof(SColIdx);
-    }
-    ASSERT(kvLen == memRowTLen(tRow));
-  }
-  taosArrayDestroy(stashRow);
-  return buffer;
-}
-#endif

source/common/src/tglobal.c

@@ -169,6 +169,9 @@ uint32_t tsMaxRange = 500;                      // max range
 uint32_t tsCurRange = 100;                      // range
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
+// udf
+bool     tsStartUdfd = true;
+
 // internal
 int32_t tsTransPullupInterval = 6;
 int32_t tsMqRebalanceInterval = 2;
@@ -348,7 +351,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
 }
 
 static int32_t taosAddServerCfg(SConfig *pCfg) {
-  if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 4096, 0) != 0) return -1;
   if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1;
   if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1;
@@ -441,6 +444,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "startUdfd", tsStartUdfd, 0) != 0) return -1;
   return 0;
 }
 
@@ -581,6 +585,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
 
+  tsStartUdfd = cfgGetItem(pCfg, "startUdfd")->bval;
+
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
   }

source/common/src/tmsg.c

@@ -607,6 +607,11 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
     if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
+  if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
+  if (pReq->commentLen > 0) {
+    if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1;
+  }
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -639,6 +644,14 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
     }
   }
 
+  if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
+  if (pReq->commentLen > 0) {
+    pReq->comment = taosMemoryMalloc(pReq->commentLen);
+    if (pReq->comment == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
+  }
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -1233,21 +1246,21 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
   for (int32_t i = 0; i < numOfCreatedDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
-    int32_t len = strlen(db) + 1;
+    int32_t len = strlen(db);
     taosHashPut(pRsp->createdDbs, db, len, db, len);
   }
 
   for (int32_t i = 0; i < numOfReadDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
-    int32_t len = strlen(db) + 1;
+    int32_t len = strlen(db);
     taosHashPut(pRsp->readDbs, db, len, db, len);
   }
 
   for (int32_t i = 0; i < numOfWriteDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
-    int32_t len = strlen(db) + 1;
+    int32_t len = strlen(db);
     taosHashPut(pRsp->writeDbs, db, len, db, len);
   }
 

source/common/src/tname.c

@@ -18,11 +18,9 @@
 #include "tcommon.h"
 #include "tstrbuild.h"
 
-#define VALID_NAME_TYPE(x)  ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T)
+#define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T)
 
-bool tscValidateTableNameLength(size_t len) {
+bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; }
-  return len < TSDB_TABLE_NAME_LEN;
-}
 
 #if 0
 // TODO refactor
@@ -95,12 +93,12 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
       * but in case of DST, the start time of one day need to be dynamically decided.
       */
       // todo refactor to extract function that is available for Linux/Windows/Mac platform
-  #if defined(WINDOWS) && _MSC_VER >= 1900
+#if defined(WINDOWS) && _MSC_VER >= 1900
       // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
       int64_t timezone = _timezone;
       int32_t daylight = _daylight;
       char**  tzname = _tzname;
-  #endif
+#endif
 
       int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
       start += timezone * t;
@@ -142,10 +140,10 @@ int32_t tNameExtractFullName(const SName* name, char* dst) {
 int32_t tNameLen(const SName* name) {
   assert(name != NULL);
 
-  char tmp[12] = {0};
+  char    tmp[12] = {0};
   int32_t len = sprintf(tmp, "%d", name->acctId);
-  int32_t len1 = (int32_t) strlen(name->dbname);
+  int32_t len1 = (int32_t)strlen(name->dbname);
-  int32_t len2 = (int32_t) strlen(name->tname);
+  int32_t len2 = (int32_t)strlen(name->tname);
 
   if (name->type == TSDB_DB_NAME_T) {
     assert(len2 == 0);
@@ -200,9 +198,7 @@ const char* tNameGetTableName(const SName* name) {
   return &name->tname[0];
 }
 
-void tNameAssign(SName* dst, const SName* src) {
+void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); }
-  memcpy(dst, src, sizeof(SName));
-}
 
 int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) {
   assert(dst != NULL && dbName != NULL && nameLen > 0);
@@ -244,7 +240,6 @@ bool tNameDBNameEqual(SName* left, SName* right) {
   return (0 == strcmp(left->dbname, right->dbname));
 }
 
-
 int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
   assert(dst != NULL && str != NULL && strlen(str) > 0);
 
@@ -260,14 +255,14 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
 
   if ((type & T_NAME_DB) == T_NAME_DB) {
     dst->type = TSDB_DB_NAME_T;
-    char* start = (char*)((p == NULL)? str:(p+1));
+    char* start = (char*)((p == NULL) ? str : (p + 1));
 
     int32_t len = 0;
     p = strstr(start, TS_PATH_DELIMITER);
     if (p == NULL) {
-      len = (int32_t) strlen(start);
+      len = (int32_t)strlen(start);
     } else {
-      len = (int32_t) (p - start);
+      len = (int32_t)(p - start);
     }
 
     // too long account id or too long db name
@@ -275,21 +270,21 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
       return -1;
     }
 
-    memcpy (dst->dbname, start, len);
+    memcpy(dst->dbname, start, len);
     dst->dbname[len] = 0;
   }
 
   if ((type & T_NAME_TABLE) == T_NAME_TABLE) {
     dst->type = TSDB_TABLE_NAME_T;
-    char* start = (char*) ((p == NULL)? str: (p+1));
+    char* start = (char*)((p == NULL) ? str : (p + 1));
 
     // too long account id or too long db name
-    int32_t len = (int32_t) strlen(start);
+    int32_t len = (int32_t)strlen(start);
     if ((len >= tListLen(dst->tname)) || (len <= 0)) {
       return -1;
     }
 
-    memcpy (dst->tname, start, len);
+    memcpy(dst->tname, start, len);
     dst->tname[len] = 0;
   }
 
@@ -305,14 +300,14 @@ static int compareKv(const void* p1, const void* p2) {
   if (res != 0) {
     return res;
   } else {
-    return kvLen1-kvLen2;
+    return kvLen1 - kvLen2;
   }
 }
 
 /*
  * use stable name and tags to grearate child table name
  */
-void buildChildTableName(RandTableName *rName) {
+void buildChildTableName(RandTableName* rName) {
   int32_t size = taosArrayGetSize(rName->tags);
   ASSERT(size > 0);
   taosArraySort(rName->tags, compareKv);
@@ -320,19 +315,19 @@ void buildChildTableName(RandTableName *rName) {
   SStringBuilder sb = {0};
   taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen);
   for (int j = 0; j < size; ++j) {
-    SSmlKv *tagKv = taosArrayGetP(rName->tags, j);
+    SSmlKv* tagKv = taosArrayGetP(rName->tags, j);
     taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
     taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen);
   }
-  size_t len = 0;
+  size_t    len = 0;
-  char* keyJoined = taosStringBuilderGetResult(&sb, &len);
+  char*     keyJoined = taosStringBuilderGetResult(&sb, &len);
   T_MD5_CTX context;
   tMD5Init(&context);
-  tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len);
+  tMD5Update(&context, (uint8_t*)keyJoined, (uint32_t)len);
   tMD5Final(&context);
   uint64_t digest1 = *(uint64_t*)(context.digest);
   uint64_t digest2 = *(uint64_t*)(context.digest + 8);
-  snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2);
+  snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016" PRIx64 "%016" PRIx64, digest1, digest2);
   taosStringBuilderDestroy(&sb);
   rName->uid = digest1;
 }

source/common/src/trow.c

@@ -33,6 +33,7 @@ const uint8_t tdVTypeByte[2][3] = {{
 
 // declaration
 static uint8_t tdGetBitmapByte(uint8_t byte);
+static int32_t tdCompareColId(const void *arg1, const void *arg2);
 
 // static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 
@@ -916,4 +917,937 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
   }
 
   return pRet;
+}
+
+void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) {
+  STSRowIter iter = {0};
+  tdSTSRowIterInit(&iter, pSchema);
+  tdSTSRowIterReset(&iter, row);
+  printf("%s >>>", tag);
+  for (int i = 0; i < pSchema->numOfCols; ++i) {
+    STColumn *stCol = pSchema->columns + i;
+    SCellVal  sVal = {255, NULL};
+    if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
+      break;
+    }
+    ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2);
+    tdSCellValPrint(&sVal, stCol->type);
+  }
+  printf("\n");
+}
+
+void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
+  if (tdValTypeIsNull(pVal->valType)) {
+    printf("NULL ");
+    return;
+  } else if (tdValTypeIsNone(pVal->valType)) {
+    printf("NONE ");
+    return;
+  }
+  switch (colType) {
+    case TSDB_DATA_TYPE_BOOL:
+      printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true");
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      printf("%" PRIi8 " ", *(int8_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      printf("%" PRIi16 " ", *(int16_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      printf("%" PRIi32 " ", *(int32_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      printf("%" PRIi64 " ", *(int64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      printf("%f ", *(float *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      printf("%lf ", *(double *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_VARCHAR:
+      printf("VARCHAR ");
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      printf("%" PRIi64 " ", *(int64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_NCHAR:
+      printf("NCHAR ");
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      printf("%" PRIu8 " ", *(uint8_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      printf("%" PRIu16 " ", *(uint16_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      printf("%" PRIu32 " ", *(uint32_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      printf("%" PRIu64 " ", *(uint64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+}
+
+int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) {
+  ASSERT(rows > 0);
+  int32_t result = 0;
+
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    result += pDataCol->dataOff[rows - 1];
+    SCellVal val = {0};
+    if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) {
+      TASSERT(0);
+    }
+
+    // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4
+    result += varDataTLen(val.val);
+    // TODO: later on, don't save Null/None for VarDataT for 3.0
+    // if (tdValTypeIsNorm(val.valType)) {
+    //   result += varDataTLen(val.val);
+    // }
+  } else {
+    result += TYPE_BYTES[pDataCol->type] * rows;
+  }
+
+  ASSERT(pDataCol->len == result);
+
+  return result;
+}
+
+bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) {
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
+    return true;
+  }
+  void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
+  tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx);
+  return true;
+}
+
+bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx,
+                    SCellVal *pVal) {
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
+    return true;
+  }
+  void *pBitmap = tdGetBitmapAddrTp(pRow, flen);
+  tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx);
+  return true;
+}
+
+int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) {
+  if (isAllRowsNone(pCol)) {
+    pVal->valType = TD_VTYPE_NULL;
+#ifdef TD_SUPPORT_READ2
+    pVal->val = (void *)getNullValue(pCol->type);
+#else
+    pVal->val = NULL;
+#endif
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (TD_COL_ROWS_NORM(pCol)) {
+    pVal->valType = TD_VTYPE_NORM;
+  } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) {
+    return terrno;
+  }
+
+  if (tdValTypeIsNorm(pVal->valType)) {
+    if (IS_VAR_DATA_TYPE(pCol->type)) {
+      pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+    } else {
+      pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+    }
+  } else {
+    pVal->valType = TD_VTYPE_NULL;
+#ifdef TD_SUPPORT_READ2
+    pVal->val = (void *)getNullValue(pCol->type);
+#else
+    pVal->val = NULL;
+#endif
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) {
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    pVal->val = &pIter->pRow->ts;
+    pVal->valType = TD_VTYPE_NORM;
+    return true;
+  }
+
+  if (TD_IS_TP_ROW(pIter->pRow)) {
+    STColumn *pCol = NULL;
+    STSchema *pSchema = pIter->pSchema;
+    while (pIter->colIdx < pSchema->numOfCols) {
+      pCol = &pSchema->columns[pIter->colIdx];  // 1st column of schema is primary TS key
+      if (colId == pCol->colId) {
+        break;
+      } else if (pCol->colId < colId) {
+        ++pIter->colIdx;
+        continue;
+      } else {
+        return false;
+      }
+    }
+    tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    ++pIter->colIdx;
+  } else if (TD_IS_KV_ROW(pIter->pRow)) {
+    return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal);
+  } else {
+    pVal->valType = TD_VTYPE_NONE;
+    terrno = TSDB_CODE_INVALID_PARA;
+    if (COL_REACH_END(colId, pIter->maxColId)) return false;
+  }
+  return true;
+}
+
+bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) {
+  STSRow    *pRow = pIter->pRow;
+  SKvRowIdx *pKvIdx = NULL;
+  bool       colFound = false;
+  col_id_t   kvNCols = tdRowGetNCols(pRow) - 1;
+  while (*nIdx < kvNCols) {
+    pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
+    if (pKvIdx->colId == colId) {
+      ++(*nIdx);
+      pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset);
+      colFound = true;
+      break;
+    } else if (pKvIdx->colId > colId) {
+      pVal->valType = TD_VTYPE_NONE;
+      return true;
+    } else {
+      ++(*nIdx);
+    }
+  }
+
+  if (!colFound) {
+    if (colId <= pIter->maxColId) {
+      pVal->valType = TD_VTYPE_NONE;
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+#ifdef TD_SUPPORT_BITMAP
+  int16_t colIdx = -1;
+  if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx);
+  if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
+    pVal->valType = TD_VTYPE_NONE;
+  }
+#else
+  pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
+#endif
+
+  return true;
+}
+
+bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) {
+  STSRow *pRow = pIter->pRow;
+  if (IS_VAR_DATA_TYPE(colType)) {
+    pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
+  } else {
+    pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
+  }
+
+#ifdef TD_SUPPORT_BITMAP
+  if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
+    pVal->valType = TD_VTYPE_NONE;
+  }
+#else
+  pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
+#endif
+
+  return true;
+}
+
+static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
+    return 1;
+  } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
+    return -1;
+  } else {
+    return 0;
+  }
+}
+
+bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) {
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    pVal->val = &pIter->pRow->ts;
+    pVal->valType = TD_VTYPE_NORM;
+    return true;
+  }
+
+  STSRow *pRow = pIter->pRow;
+  int16_t colIdx = -1;
+  if (TD_IS_TP_ROW(pRow)) {
+    STSchema *pSchema = pIter->pSchema;
+    STColumn *pCol =
+        (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ);
+    if (!pCol) {
+      pVal->valType = TD_VTYPE_NONE;
+      if (COL_REACH_END(colId, pIter->maxColId)) return false;
+      return true;
+    }
+#ifdef TD_SUPPORT_BITMAP
+    colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn);
+#endif
+    tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1);
+  } else if (TD_IS_KV_ROW(pRow)) {
+    SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx),
+                                               compareKvRowColId, TD_EQ);
+#ifdef TD_SUPPORT_BITMAP
+    if (pIdx) {
+      colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx);
+    }
+#endif
+    tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx);
+  } else {
+    if (COL_REACH_END(colId, pIter->maxColId)) return false;
+    pVal->valType = TD_VTYPE_NONE;
+  }
+
+  return true;
+}
+
+static int32_t tdCompareColId(const void *arg1, const void *arg2) {
+  int32_t   colId = *(int32_t *)arg1;
+  STColumn *pCol = (STColumn *)arg2;
+
+  if (colId < pCol->colId) {
+    return -1;
+  } else if (colId == pCol->colId) {
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      *pValType = (((*pDestByte) & 0xC0) >> 6);
+      break;
+    case 1:
+      *pValType = (((*pDestByte) & 0x30) >> 4);
+      break;
+    case 2:
+      *pValType = (((*pDestByte) & 0x0C) >> 2);
+      break;
+    case 3:
+      *pValType = ((*pDestByte) & 0x03);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      *pValType = (((*pDestByte) & 0x80) >> 7);
+      break;
+    case 1:
+      *pValType = (((*pDestByte) & 0x40) >> 6);
+      break;
+    case 2:
+      *pValType = (((*pDestByte) & 0x20) >> 5);
+      break;
+    case 3:
+      *pValType = (((*pDestByte) & 0x10) >> 4);
+      break;
+    case 4:
+      *pValType = (((*pDestByte) & 0x08) >> 3);
+      break;
+    case 5:
+      *pValType = (((*pDestByte) & 0x04) >> 2);
+      break;
+    case 6:
+      *pValType = (((*pDestByte) & 0x02) >> 1);
+      break;
+    case 7:
+      *pValType = ((*pDestByte) & 0x01);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7);
+      // set the value and clear other partitions for offset 0
+      // *pDestByte |= (valType << 7);
+      break;
+    case 1:
+      *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6);
+      // *pDestByte |= (valType << 6);
+      break;
+    case 2:
+      *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5);
+      // *pDestByte |= (valType << 5);
+      break;
+    case 3:
+      *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4);
+      // *pDestByte |= (valType << 4);
+      break;
+    case 4:
+      *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3);
+      // *pDestByte |= (valType << 3);
+      break;
+    case 5:
+      *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2);
+      // *pDestByte |= (valType << 2);
+      break;
+    case 6:
+      *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1);
+      // *pDestByte |= (valType << 1);
+      break;
+    case 7:
+      *pDestByte = ((*pDestByte) & 0xFE) | valType;
+      // *pDestByte |= (valType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) {
+#ifdef TD_SUPPORT_BITMAP
+  TASSERT(colIdx < tdRowGetNCols(pRow) - 1);
+  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
+    output->valType = TD_VTYPE_NONE;
+    return terrno;
+  }
+  if (tdValTypeIsNorm(output->valType)) {
+    if (offset < 0) {
+      terrno = TSDB_CODE_INVALID_PARA;
+      output->valType = TD_VTYPE_NONE;
+      return terrno;
+    }
+    output->val = POINTER_SHIFT(pRow, offset);
+  }
+#else
+  TASSERT(0);
+  if (offset < 0) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    output->valType = TD_VTYPE_NONE;
+    return terrno;
+  }
+  output->val = POINTER_SHIFT(pRow, offset);
+  output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset,
+                           int16_t colIdx) {
+#ifdef TD_SUPPORT_BITMAP
+  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
+    output->valType = TD_VTYPE_NONE;
+    return terrno;
+  }
+  if (tdValTypeIsNorm(output->valType)) {
+    if (IS_VAR_DATA_TYPE(colType)) {
+      output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
+    } else {
+      output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
+    }
+  }
+#else
+  if (IS_VAR_DATA_TYPE(colType)) {
+    output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
+  } else {
+    output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
+  }
+  output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val,
+                            bool isCopyVarData, int32_t offset, col_id_t colIdx) {
+  STSRow *pRow = pBuilder->pBuf;
+  if (!val) {
+#ifdef TD_SUPPORT_BITMAP
+    if (tdValTypeIsNorm(valType)) {
+      terrno = TSDB_CODE_INVALID_PTR;
+      return terrno;
+    }
+#else
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+#endif
+  }
+  // TS KEY is stored in STSRow.ts and not included in STSRow.data field.
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    TD_ROW_KEY(pRow) = *(TSKEY *)val;
+    // The primary TS key is Norm all the time, thus its valType is not stored in bitmap.
+    return TSDB_CODE_SUCCESS;
+  }
+  // TODO:  We can avoid the type judegement by FP, but would prevent the inline scheme.
+  if (TD_IS_TP_ROW(pRow)) {
+    tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset);
+  } else {
+    tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
+                              int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) {
+  if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  offset -= sizeof(SKvRowIdx);
+  --colIdx;
+
+#ifdef TD_SUPPORT_BITMAP
+  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
+    return terrno;
+  }
+#endif
+
+  STSRow *row = pBuilder->pBuf;
+  // No need to store None/Null values.
+  if (tdValIsNorm(valType, val, colType)) {
+    // ts key stored in STSRow.ts
+    SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    pColIdx->colId = colId;
+    pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
+
+    if (IS_VAR_DATA_TYPE(colType)) {
+      if (isCopyVarData) {
+        memcpy(ptr, val, varDataTLen(val));
+      }
+      TD_ROW_LEN(row) += varDataTLen(val);
+    } else {
+      memcpy(ptr, val, TYPE_BYTES[colType]);
+      TD_ROW_LEN(row) += TYPE_BYTES[colType];
+    }
+  }
+#ifdef TD_SUPPORT_BACK2
+  // NULL/None value
+  else {
+    SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    pColIdx->colId = colId;
+    pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
+    const void *nullVal = getNullValue(colType);
+
+    if (IS_VAR_DATA_TYPE(colType)) {
+      if (isCopyVarData) {
+        memcpy(ptr, nullVal, varDataTLen(nullVal));
+      }
+      TD_ROW_LEN(row) += varDataTLen(nullVal);
+    } else {
+      memcpy(ptr, nullVal, TYPE_BYTES[colType]);
+      TD_ROW_LEN(row) += TYPE_BYTES[colType];
+    }
+  }
+#endif
+
+  return 0;
+}
+
+int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
+                              int8_t colType, int16_t colIdx, int32_t offset) {
+  if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  offset -= sizeof(TSKEY);
+  --colIdx;
+
+#ifdef TD_SUPPORT_BITMAP
+  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
+    return terrno;
+  }
+#endif
+
+  STSRow *row = pBuilder->pBuf;
+
+  // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap
+  // should be updated simultaneously if Norm val overwrite Null/None cols.
+  // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null.
+  if (tdValIsNorm(valType, val, colType)) {
+    // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar.
+    if (IS_VAR_DATA_TYPE(colType)) {
+      // ts key stored in STSRow.ts
+      *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row);
+      if (isCopyVarData) {
+        memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val));
+      }
+      TD_ROW_LEN(row) += varDataTLen(val);
+    } else {
+      memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]);
+    }
+  }
+#ifdef TD_SUPPORT_BACK2
+  // NULL/None value
+  else {
+    // TODO: Null value for new data types imported since 3.0 need to be defined.
+    const void *nullVal = getNullValue(colType);
+    if (IS_VAR_DATA_TYPE(colType)) {
+      // ts key stored in STSRow.ts
+      *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row);
+
+      if (isCopyVarData) {
+        memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal));
+      }
+      TD_ROW_LEN(row) += varDataTLen(nullVal);
+    } else {
+      memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]);
+    }
+  }
+#endif
+
+  return 0;
+}
+
+int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,
+                              int32_t allNullLen, int32_t boundNullLen) {
+  if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) {
+    uint32_t tpLen = allNullLen;
+    uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen;
+    if (isSelectKVRow(kvLen, tpLen)) {
+      pBuilder->rowType = TD_ROW_KV;
+    } else {
+      pBuilder->rowType = TD_ROW_TP;
+    }
+
+  } else {
+    pBuilder->rowType = TD_ROW_TP;
+  }
+
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  pBuilder->nBoundCols = nBoundCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+  if (nBoundCols > 0) {
+    pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
+  } else {
+    pBuilder->nBoundBitmaps = 0;
+  }
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
+  pBuilder->pBuf = (STSRow *)pBuf;
+  if (!pBuilder->pBuf) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  TD_ROW_SET_INFO(pBuilder->pBuf, 0);
+  TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
+
+  TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
+
+  uint32_t len = 0;
+  switch (pBuilder->rowType) {
+    case TD_ROW_TP:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
+      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
+#endif
+      // the primary TS key is stored separatedly
+      len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps;
+      TD_ROW_SET_LEN(pBuilder->pBuf, len);
+      TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
+      break;
+    case TD_ROW_KV:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
+      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
+#endif
+      len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
+            pBuilder->nBoundBitmaps;  // add
+      TD_ROW_SET_LEN(pBuilder->pBuf, len);
+      TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
+      TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) {
+  pBuilder->pBuf = (STSRow *)pBuf;
+  if (!pBuilder->pBuf) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
+
+  uint32_t len = 0;
+  switch (pBuilder->rowType) {
+    case TD_ROW_TP:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
+#endif
+      break;
+    case TD_ROW_KV:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
+#endif
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols,
+                     int32_t nBoundCols, int32_t flen) {
+  if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) {
+    return terrno;
+  }
+  return tdSRowResetBuf(pBuilder, pBuf);
+}
+
+void tdSRowReset(SRowBuilder *pBuilder) {
+  pBuilder->rowType = TD_ROW_TP;
+  pBuilder->pBuf = NULL;
+  pBuilder->nBoundCols = -1;
+  pBuilder->nCols = -1;
+  pBuilder->flen = -1;
+  pBuilder->pBitmap = NULL;
+}
+
+int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) {
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) {
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  pBuilder->nBoundCols = nBoundCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+  if (nBoundCols > 0) {
+    pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
+  } else {
+    pBuilder->nBoundBitmaps = 0;
+  }
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+  switch (bitmapMode) {
+    case 0:
+      tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
+      break;
+    case -1:
+    case 1:
+      tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) {
+  TDRowValT valType = 0;
+  tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode);
+  if (tdValTypeIsNorm(valType)) {
+    return true;
+  }
+  return false;
+}
+
+int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
+      // set the value and clear other partitions for offset 0
+      // *pDestByte |= (valType << 6);
+      break;
+    case 1:
+      *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
+      // *pDestByte |= (valType << 4);
+      break;
+    case 2:
+      *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
+      // *pDestByte |= (valType << 2);
+      break;
+    case 3:
+      *pDestByte = ((*pDestByte) & 0xFC) | valType;
+      // *pDestByte |= (valType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) {
+  switch (bitmapMode) {
+    case 0:
+      tdSetBitmapValTypeII(pBitmap, colIdx, valType);
+      break;
+    case -1:
+    case 1:
+      tdSetBitmapValTypeI(pBitmap, colIdx, valType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) {
+#ifdef TD_SUPPORT_BITMAP
+  switch (rowType) {
+    case TD_ROW_TP:
+      return tdGetBitmapAddrTp(pRow, flen);
+    case TD_ROW_KV:
+      return tdGetBitmapAddrKv(pRow, nKvCols);
+    default:
+      break;
+  }
+#endif
+  return NULL;
+}
+
+void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) {
+  pIter->pRow = pRow;
+  pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow));
+  pIter->offset = 0;
+  pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID;
+  pIter->kvIdx = 0;
+}
+
+void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) {
+  pIter->pSchema = pSchema;
+  pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId;
 }

source/dnode/mgmt/implement/inc/dmImp.h

@@ -26,8 +26,10 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper);
 void    dmCloseNode(SMgmtWrapper *pWrapper);
 
 // dmTransport.c
-int32_t  dmInitTrans(SDnode *pDnode);
+int32_t  dmInitServer(SDnode *pDnode);
-void     dmCleanupTrans(SDnode *pDnode);
+void     dmCleanupServer(SDnode *pDnode);
+int32_t  dmInitClient(SDnode *pDnode);
+void     dmCleanupClient(SDnode *pDnode);
 SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper);
 SMsgCb   dmGetMsgcb(SMgmtWrapper *pWrapper);
 int32_t  dmInitMsgHandle(SDnode *pDnode);

source/dnode/mgmt/implement/src/dmExec.c

@@ -213,10 +213,12 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
     }
 
     pWrapper->procType = DND_PROC_CHILD;
+    if (dmInitClient(pDnode) != 0) {
+      return -1;
+    }
 
-    SMsgCb msgCb = pDnode->data.msgCb;
+    pDnode->data.msgCb = dmGetMsgcb(pWrapper);
-    msgCb.pWrapper = pWrapper;
+    tmsgSetDefaultMsgCb(&pDnode->data.msgCb);
-    tmsgSetDefaultMsgCb(&msgCb);
 
     if (dmOpenNode(pWrapper) != 0) {
       dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
@@ -234,6 +236,15 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
         pWrapper->procType = DND_PROC_SINGLE;
       }
 
+      if (n == DNODE) {
+        if (dmInitClient(pDnode) != 0) {
+          return -1;
+        }
+
+        pDnode->data.msgCb = dmGetMsgcb(pWrapper);
+        tmsgSetDefaultMsgCb(&pDnode->data.msgCb);
+      }
+
       if (dmOpenNode(pWrapper) != 0) {
         dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
         return -1;
@@ -281,21 +292,21 @@ static void dmProcessProcHandle(void *handle) {
 }
 
 static void dmWatchNodes(SDnode *pDnode) {
-  taosThreadMutexLock(&pDnode->mutex);
+  if (pDnode->ptype != DND_PROC_PARENT) return;
-  if (pDnode->ptype == DND_PROC_PARENT) {
+  if (pDnode->ntype == NODE_END) return;
-    for (EDndNodeType n = DNODE + 1; n < NODE_END; ++n) {
-      SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-      if (!pWrapper->required) continue;
-      if (pWrapper->procType != DND_PROC_PARENT) continue;
-      if (pDnode->ntype == NODE_END) continue;
 
-      if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
+  taosThreadMutexLock(&pDnode->mutex);
-        dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
+  for (EDndNodeType n = DNODE + 1; n < NODE_END; ++n) {
-        if (pWrapper->procObj) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-          taosProcCloseHandles(pWrapper->procObj, dmProcessProcHandle);
+    if (!pWrapper->required) continue;
-        }
+    if (pWrapper->procType != DND_PROC_PARENT) continue;
-        dmNewNodeProc(pWrapper, n);
+
+    if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
+      dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
+      if (pWrapper->procObj) {
+        taosProcCloseHandles(pWrapper->procObj, dmProcessProcHandle);
       }
+      dmNewNodeProc(pWrapper, n);
     }
   }
   taosThreadMutexUnlock(&pDnode->mutex);

source/dnode/mgmt/implement/src/dmHandle.c

@@ -217,145 +217,6 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) {
   dmStopStatusThread(pWrapper->pDnode);
 }
 
-static int32_t dmSpawnUdfd(SDnode *pDnode);
-
-void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
-  dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
-  SDnode *pDnode = process->data;
-  if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pDnode->udfdData.stopCalled)) {
-    dInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
-  } else {
-    dInfo("udfd process restart");
-    dmSpawnUdfd(pDnode);
-  }
-}
-
-static int32_t dmSpawnUdfd(SDnode *pDnode) {
-  dInfo("dnode start spawning udfd");
-  uv_process_options_t options = {0};
-
-  char path[PATH_MAX] = {0};
-  if (tsProcPath == NULL) {
-    path[0] = '.';
-  } else {
-    strncpy(path, tsProcPath, strlen(tsProcPath));
-    taosDirName(path);
-  }
-#ifdef WINDOWS
-  strcat(path, "udfd.exe");
-#else
-  strcat(path, "/udfd");
-#endif
-  char* argsUdfd[] = {path, "-c", configDir, NULL};
-  options.args = argsUdfd;
-  options.file = path;
-
-  options.exit_cb = dmUdfdExit;
-
-  SUdfdData *pData = &pDnode->udfdData;
-  uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
-
-  uv_stdio_container_t child_stdio[3];
-  child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
-  child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
-  child_stdio[1].flags = UV_IGNORE;
-  child_stdio[2].flags = UV_INHERIT_FD;
-  child_stdio[2].data.fd = 2;
-  options.stdio_count = 3;
-  options.stdio = child_stdio;
-
-  options.flags = UV_PROCESS_DETACHED;
-
-  char dnodeIdEnvItem[32] = {0};
-  char thrdPoolSizeEnvItem[32] = {0};
-  snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
-  float numCpuCores = 4;
-  taosGetCpuCores(&numCpuCores);
-  snprintf(thrdPoolSizeEnvItem,32,  "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
-  char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
-  options.env = envUdfd;
-
-  int err = uv_spawn(&pData->loop, &pData->process, &options);
-  pData->process.data = (void*)pDnode;
-
-  if (err != 0) {
-    dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
-  }
-  return err;
-}
-
-static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
-  if (!uv_is_closing(handle)) {
-    uv_close(handle, NULL);
-  }
-}
-
-static void dmUdfdStopAsyncCb(uv_async_t *async) {
-  SDnode *pDnode = async->data;
-  SUdfdData *pData = &pDnode->udfdData;
-  uv_stop(&pData->loop);
-}
-
-static void dmWatchUdfd(void *args) {
-  SDnode *pDnode = args;
-  SUdfdData *pData = &pDnode->udfdData;
-  uv_loop_init(&pData->loop);
-  uv_async_init(&pData->loop, &pData->stopAsync, dmUdfdStopAsyncCb);
-  pData->stopAsync.data = pDnode;
-  int32_t err = dmSpawnUdfd(pDnode);
-  atomic_store_32(&pData->spawnErr, err);
-  uv_barrier_wait(&pData->barrier);
-  uv_run(&pData->loop, UV_RUN_DEFAULT);
-  uv_loop_close(&pData->loop);
-
-  uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
-  uv_run(&pData->loop, UV_RUN_DEFAULT);
-  uv_loop_close(&pData->loop);
-  return;
-}
-
-static int32_t dmStartUdfd(SDnode *pDnode) {
-  char dnodeId[8] = {0};
-  snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId);
-  uv_os_setenv("DNODE_ID", dnodeId);
-  SUdfdData *pData = &pDnode->udfdData;
-  if (pData->startCalled) {
-    dInfo("dnode-mgmt start udfd already called");
-    return 0;
-  }
-  pData->startCalled = true;
-  uv_barrier_init(&pData->barrier, 2);
-  uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
-  uv_barrier_wait(&pData->barrier);
-  int32_t err = atomic_load_32(&pData->spawnErr);
-  if (err != 0) {
-    uv_barrier_destroy(&pData->barrier);
-    uv_async_send(&pData->stopAsync);
-    uv_thread_join(&pData->thread);
-    pData->needCleanUp = false;
-    dInfo("dnode-mgmt udfd cleaned up after spawn err");
-  } else {
-    pData->needCleanUp = true;
-  }
-  return err;
-}
-
-static int32_t dmStopUdfd(SDnode *pDnode) {
-  dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
-        pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
-  SUdfdData *pData = &pDnode->udfdData;
-  if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
-    return 0;
-  }
-  atomic_store_32(&pData->stopCalled, 1);
-  pData->needCleanUp = false;
-  uv_barrier_destroy(&pData->barrier);
-  uv_async_send(&pData->stopAsync);
-  uv_thread_join(&pData->thread);
-  dInfo("dnode-mgmt udfd cleaned up");
-  return 0;
-}
-
 static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to init");
   SDnode *pDnode = pWrapper->pDnode;
@@ -381,13 +242,13 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
     return -1;
   }
 
-  if (dmInitTrans(pDnode) != 0) {
+  if (dmInitServer(pDnode) != 0) {
     dError("failed to init transport since %s", terrstr());
     return -1;
   }
   dmReportStartup(pDnode, "dnode-transport", "initialized");
 
-  if (dmStartUdfd(pDnode) != 0) {
+  if (udfStartUdfd(pDnode->data.dnodeId) != 0) {
     dError("failed to start udfd");
   }
 
@@ -398,7 +259,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
 static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to clean up");
   SDnode *pDnode = pWrapper->pDnode;
-  dmStopUdfd(pDnode);
+
+  udfStopUdfd();
+
   dmStopWorker(pDnode);
 
   taosWLockLatch(&pDnode->data.latch);
@@ -412,7 +275,8 @@ static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
   }
   taosWUnLockLatch(&pDnode->data.latch);
 
-  dmCleanupTrans(pDnode);
+  dmCleanupClient(pDnode);
+  dmCleanupServer(pDnode);
   dInfo("dnode-mgmt is cleaned up");
 }
 

source/dnode/mgmt/implement/src/dmObj.c

@@ -124,9 +124,6 @@ SDnode *dmCreate(const SDnodeOpt *pOption) {
     goto _OVER;
   }
 
-  pDnode->data.msgCb = dmGetMsgcb(&pDnode->wrappers[DNODE]);
-  tmsgSetDefaultMsgCb(&pDnode->data.msgCb);
-
   dInfo("dnode is created, data:%p", pDnode);
   code = 0;
 

source/dnode/mgmt/implement/src/dmTransport.c

@@ -16,6 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "dmImp.h"
 
+#include "qworker.h"
+
 #define INTERNAL_USER   "_dnd"
 #define INTERNAL_CKEY   "_key"
 #define INTERNAL_SECRET "_pwd"
@@ -85,17 +87,14 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
   if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
   if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER;
 
-  if (pWrapper->procType == DND_PROC_SINGLE) {
+  if (pWrapper->procType != DND_PROC_PARENT) {
     dTrace("msg:%p, created, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user);
     code = (*msgFp)(pWrapper, pMsg);
-  } else if (pWrapper->procType == DND_PROC_PARENT) {
+  } else {
     dTrace("msg:%p, created and put into child queue, type:%s handle:%p code:0x%04x user:%s contLen:%d", pMsg,
            TMSG_INFO(msgType), pRpc->handle, pMsg->rpcMsg.code & 0XFFFF, pMsg->user, pRpc->contLen);
     code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen,
                                (isReq && (pMsg->rpcMsg.code == 0)) ? pRpc->handle : NULL, pRpc->refId, PROC_FUNC_REQ);
-  } else {
-    dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user);
-    ASSERT(1);
   }
 
 _OVER:
@@ -108,7 +107,7 @@ _OVER:
   } else {
     dError("msg:%p, type:%s handle:%p failed to process since 0x%04x:%s", pMsg, TMSG_INFO(msgType), pRpc->handle,
            code & 0XFFFF, terrstr());
-    if (msgType & 1U) {
+    if (isReq) {
       if (terrno != 0) code = terrno;
       if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) {
         if (msgType > TDMT_MND_MSG && msgType < TDMT_VND_MSG) {
@@ -130,22 +129,27 @@ _OVER:
 }
 
 static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SDnodeTrans * pTrans = &pDnode->trans;
+  SDnodeTrans  *pTrans = &pDnode->trans;
   tmsg_t        msgType = pMsg->msgType;
   bool          isReq = msgType & 1u;
-  SMsgHandle *  pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
+  SMsgHandle   *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
   SMgmtWrapper *pWrapper = pHandle->pNdWrapper;
 
-  if (msgType == TDMT_DND_SERVER_STATUS) {
+  switch (msgType) {
-    dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
+    case TDMT_DND_SERVER_STATUS:
-    dmProcessServerStatusReq(pDnode, pMsg);
+      dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
-    return;
+      dmProcessServerStatusReq(pDnode, pMsg);
-  }
+      return;
-
+    case TDMT_DND_NET_TEST:
-  if (msgType == TDMT_DND_NET_TEST) {
+      dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
-    dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
+      dmProcessNetTestReq(pDnode, pMsg);
-    dmProcessServerStatusReq(pDnode, pMsg);
+      return;
-    return;
+    case TDMT_MND_SYSTABLE_RETRIEVE_RSP:
+    case TDMT_VND_FETCH_RSP:
+      dTrace("retrieve rsp is received");
+      qWorkerProcessFetchRsp(NULL, NULL, pMsg);
+      pMsg->pCont = NULL;  // already freed in qworker
+      return;
   }
 
   if (pDnode->status != DND_STAT_RUNNING) {
@@ -233,16 +237,6 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
   return 0;
 }
 
-static inline int32_t dmSendRpcReq(SDnode *pDnode, const SEpSet *pEpSet, SRpcMsg *pReq) {
-  if (pDnode->trans.clientRpc == NULL) {
-    terrno = TSDB_CODE_NODE_OFFLINE;
-    return -1;
-  }
-
-  rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pReq, NULL);
-  return 0;
-}
-
 static void dmSendRpcRedirectRsp(SDnode *pDnode, const SRpcMsg *pReq) {
   SEpSet epSet = {0};
   dmGetMnodeEpSet(pDnode, &epSet);
@@ -288,28 +282,20 @@ void dmSendToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) {
 }
 
 static inline int32_t dmSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
-  if (pWrapper->pDnode->status != DND_STAT_RUNNING) {
+  SDnode *pDnode = pWrapper->pDnode;
+  if (pDnode->status != DND_STAT_RUNNING) {
     terrno = TSDB_CODE_NODE_OFFLINE;
     dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle);
     return -1;
   }
 
-  if (pWrapper->procType != DND_PROC_CHILD) {
+  if (pDnode->trans.clientRpc == NULL) {
-    return dmSendRpcReq(pWrapper->pDnode, pEpSet, pReq);
+    terrno = TSDB_CODE_NODE_OFFLINE;
-  } else {
+    return -1;
-    char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet));
-    if (pHead == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-
-    memcpy(pHead, pReq, sizeof(SRpcMsg));
-    memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet));
-    taosProcPutToParentQ(pWrapper->procObj, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen,
-                         PROC_FUNC_REQ);
-    taosMemoryFree(pHead);
-    return 0;
   }
+
+  rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pReq, NULL);
+  return 0;
 }
 
 static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
@@ -396,9 +382,10 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
   pMsg->pCont = pCont;
 
   if (ftype == PROC_FUNC_REQ) {
+    ASSERT(1);
     dTrace("msg:%p, get from parent queue, send req:%s handle:%p code:0x%04x, app:%p", pMsg, TMSG_INFO(pMsg->msgType),
            pMsg->handle, code, pMsg->ahandle);
-    dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg);
+    dmSendReq(pWrapper, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg);
   } else if (ftype == PROC_FUNC_RSP) {
     dTrace("msg:%p, get from parent queue, rsp handle:%p code:0x%04x, app:%p", pMsg, pMsg->handle, code, pMsg->ahandle);
     pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle);
@@ -421,23 +408,25 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
 }
 
 SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper) {
-  SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dmConsumeChildQueue,
+  SProcCfg cfg = {
-                  .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
+      .childConsumeFp = (ProcConsumeFp)dmConsumeChildQueue,
-                  .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
+      .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
-                  .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+      .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
-                  .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+      .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                  .parentConsumeFp = (ProcConsumeFp)dmConsumeParentQueue,
+      .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                  .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
+      .parentConsumeFp = (ProcConsumeFp)dmConsumeParentQueue,
-                  .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
+      .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
-                  .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+      .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
-                  .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+      .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                  .shm = pWrapper->procShm,
+      .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                  .parent = pWrapper,
+      .shm = pWrapper->procShm,
-                  .name = pWrapper->name};
+      .parent = pWrapper,
+      .name = pWrapper->name,
+  };
   return cfg;
 }
 
-bool rpcRfp(int32_t code) {
+static bool rpcRfp(int32_t code) {
   if (code == TSDB_CODE_RPC_REDIRECT) {
     return true;
   } else {
@@ -445,7 +434,7 @@ bool rpcRfp(int32_t code) {
   }
 }
 
-static int32_t dmInitClient(SDnode *pDnode) {
+int32_t dmInitClient(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
 
   SRpcInit rpcInit = {0};
@@ -471,11 +460,15 @@ static int32_t dmInitClient(SDnode *pDnode) {
     return -1;
   }
 
+  pDnode->data.msgCb = dmGetMsgcb(&pDnode->wrappers[DNODE]);
+  tmsgSetDefaultMsgCb(&pDnode->data.msgCb);
+
   dDebug("dnode rpc client is initialized");
+
   return 0;
 }
 
-static void dmCleanupClient(SDnode *pDnode) {
+void dmCleanupClient(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
   if (pTrans->clientRpc) {
     rpcClose(pTrans->clientRpc);
@@ -517,7 +510,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s
   SAuthReq authReq = {0};
   tstrncpy(authReq.user, user, TSDB_USER_LEN);
   int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void *  pReq = rpcMallocCont(contLen);
+  void   *pReq = rpcMallocCont(contLen);
   tSerializeSAuthReq(pReq, contLen, &authReq);
 
   SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
@@ -543,7 +536,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s
   return rpcRsp.code;
 }
 
-static int32_t dmInitServer(SDnode *pDnode) {
+int32_t dmInitServer(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
 
   SRpcInit rpcInit = {0};
@@ -569,7 +562,7 @@ static int32_t dmInitServer(SDnode *pDnode) {
   return 0;
 }
 
-static void dmCleanupServer(SDnode *pDnode) {
+void dmCleanupServer(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
   if (pTrans->serverRpc) {
     rpcClose(pTrans->serverRpc);
@@ -578,17 +571,6 @@ static void dmCleanupServer(SDnode *pDnode) {
   }
 }
 
-int32_t dmInitTrans(SDnode *pDnode) {
-  if (dmInitServer(pDnode) != 0) return -1;
-  if (dmInitClient(pDnode) != 0) return -1;
-  return 0;
-}
-
-void dmCleanupTrans(SDnode *pDnode) {
-  dmCleanupServer(pDnode);
-  dmCleanupClient(pDnode);
-}
-
 SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) {
   SMsgCb msgCb = {
       .sendReqFp = dmSendReq,
@@ -598,6 +580,7 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) {
       .releaseHandleFp = dmReleaseHandle,
       .reportStartupFp = dmReportStartupByWrapper,
       .pWrapper = pWrapper,
+      .clientRpc = pWrapper->pDnode->trans.clientRpc,
   };
   return msgCb;
 }

source/dnode/mgmt/interface/inc/dmDef.h

@@ -156,6 +156,8 @@ typedef struct SUdfdData {
   uv_pipe_t     ctrlPipe;
   uv_async_t    stopAsync;
   int32_t        stopCalled;
+
+  int32_t         dnodeId;
 } SUdfdData;
 
 typedef struct SDnode {

source/dnode/mgmt/interface/inc/dmInt.h

@@ -37,7 +37,7 @@ void   dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF
 void   dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
 void   dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc);
 void   dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg);
-void   dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg);
+void   dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg);
 void   dmGetMonitorSysInfo(SMonSysInfo *pInfo);
 
 // dmFile.c

source/dnode/mgmt/interface/src/dmInt.c

@@ -171,16 +171,17 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
   }
 }
 
-void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
+void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) {
   dDebug("net test req is received");
-  SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
+  SRpcMsg rsp = {.handle = pReq->handle, .refId = pReq->refId, .ahandle = pReq->ahandle, .code = 0};
-  rsp.pCont = rpcMallocCont(pRpc->contLen);
+  rsp.pCont = rpcMallocCont(pReq->contLen);
   if (rsp.pCont == NULL) {
     rsp.code = TSDB_CODE_OUT_OF_MEMORY;
   } else {
-    rsp.contLen = pRpc->contLen;
+    rsp.contLen = pReq->contLen;
   }
   rpcSendResponse(&rsp);
+  rpcFreeCont(pReq->pCont);
 }
 
 void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
@@ -208,6 +209,7 @@ void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
 
 _OVER:
   rpcSendResponse(&rspMsg);
+  rpcFreeCont(pReq->pCont);
 }
 
 void dmGetMonitorSysInfo(SMonSysInfo *pInfo) {

source/dnode/mgmt/mgmt_snode/src/smInt.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "smInt.h"
+#include "libs/function/function.h"
 
 static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dmReadFile(pWrapper, required); }
 
@@ -29,6 +30,9 @@ static void smClose(SMgmtWrapper *pWrapper) {
   if (pMgmt == NULL) return;
 
   dInfo("snode-mgmt start to cleanup");
+
+  udfcClose();
+
   if (pMgmt->pSnode != NULL) {
     smStopWorker(pMgmt);
     sndClose(pMgmt->pSnode);
@@ -68,6 +72,10 @@ int32_t smOpen(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pWrapper->pDnode, "snode-worker", "initialized");
 
+  if (udfcOpen() != 0) {
+    dError("failed to open udfc in snode");
+  }
+
   return 0;
 }
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -289,7 +289,6 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, vmProcessFetchMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, vmProcessFetchMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -324,7 +324,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   dmReportStartup(pDnode, "vnode-vnodes", "initialized");
 
   if (udfcOpen() != 0) {
-    dError("failed to open udfc in dnode");
+    dError("failed to open udfc in vnode");
   }
 
   code = 0;

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -198,6 +198,9 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
       rsp.refId = pMsg->rpcMsg.refId;
       tmsgSendRsp(&rsp);
     }
+
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
   }
 }
 
@@ -211,6 +214,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
     // todo
     SRpcMsg *pRsp = NULL;
     (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
   }
 }
 

source/dnode/mnode/impl/inc/mndAuth.h

@@ -26,7 +26,7 @@ int32_t mndInitAuth(SMnode *pMnode);
 void    mndCleanupAuth(SMnode *pMnode);
 
 int32_t mndCheckCreateUserAuth(SUserObj *pOperUser);
-int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, SAlterUserReq *pAlter);
+int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
 int32_t mndCheckDropUserAuth(SUserObj *pOperUser);
 
 int32_t mndCheckNodeAuth(SUserObj *pOperUser);

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -29,6 +29,7 @@ enum {
   MQ_CONSUMER_STATUS__LOST,
   MQ_CONSUMER_STATUS__LOST_IN_REB,
   MQ_CONSUMER_STATUS__LOST_REBD,
+  MQ_CONSUMER_STATUS__REMOVED,
 };
 
 int32_t mndInitConsumer(SMnode *pMnode);

source/dnode/mnode/impl/inc/mndTopic.h

@@ -33,6 +33,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw);
 
 int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 
+const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndAuth.c

@@ -79,14 +79,12 @@ int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) {
   return -1;
 }
 
-int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, SAlterUserReq *pAlter) {
+int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) {
   if (pAlter->alterType == TSDB_ALTER_USER_PASSWD) {
     if (pOperUser->superUser || strcmp(pUser->user, pOperUser->user) == 0) {
       return 0;
     }
-  }
+  } else if (pAlter->alterType == TSDB_ALTER_USER_SUPERUSER) {
-
-  if (pAlter->alterType == TSDB_ALTER_USER_SUPERUSER) {
     if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) {
       terrno = TSDB_CODE_MND_NO_RIGHTS;
       return -1;
@@ -95,21 +93,12 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb,
     if (pOperUser->superUser) {
       return 0;
     }
-  }
+  } else {
-
-  if (pAlter->alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB || pAlter->alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
     if (pOperUser->superUser) {
       return 0;
     }
   }
 
-  if (pAlter->alterType == TSDB_ALTER_USER_ADD_READ_DB || pAlter->alterType == TSDB_ALTER_USER_REMOVE_READ_DB ||
-      pAlter->alterType == TSDB_ALTER_USER_ADD_WRITE_DB || pAlter->alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
-    if (pOperUser->superUser || strcmp(pUser->user, pDb->createUser) == 0) {
-      return 0;
-    }
-  }
-
   terrno = TSDB_CODE_MND_NO_RIGHTS;
   return -1;
 }

source/dnode/mnode/impl/src/mndConsumer.c

@@ -486,6 +486,14 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
       }
     }
 
+    if (pConsumerOld && taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 &&
+        taosArrayGetSize(pConsumerNew->rebRemovedTopics) == 0) {
+      /*if (taosArrayGetSize(pConsumerNew->assignedTopics) == 0) {*/
+      /*pConsumerNew->updateType = */
+      /*}*/
+      goto SUBSCRIBE_OVER;
+    }
+
     STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
     if (pTrans == NULL) goto SUBSCRIBE_OVER;
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
@@ -789,72 +797,88 @@ static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock
   while (numOfRows < rowsCapacity) {
     pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
     if (pShow->pIter == NULL) break;
-
+    if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {
-    SColumnInfoData *pColInfo;
+      sdbRelease(pSdb, pConsumer);
-    int32_t          cols = 0;
+      continue;
+    }
 
     taosRLockLatch(&pConsumer->lock);
 
-    // consumer id
+    int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics);
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    bool    hasTopic = true;
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
+    if (topicSz == 0) {
+      hasTopic = false;
+      topicSz = 1;
+    }
 
-    // group id
+    for (int32_t i = 0; i < topicSz; i++) {
-    char groupId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+      if (numOfRows + topicSz > rowsCapacity) {
-    tstrncpy(varDataVal(groupId), pConsumer->cgroup, TSDB_CGROUP_LEN);
+        blockDataEnsureCapacity(pBlock, numOfRows + topicSz);
-    varDataSetLen(groupId, strlen(varDataVal(groupId)));
+      }
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      SColumnInfoData *pColInfo;
-    colDataAppend(pColInfo, numOfRows, (const char *)groupId, false);
+      int32_t          cols = 0;
 
-    // app id
+      // consumer id
-    char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
-    varDataSetLen(appId, strlen(varDataVal(appId)));
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)appId, false);
 
-    // status
+      // consumer group
-    char status[20 + VARSTR_HEADER_SIZE] = {0};
+      char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
-    tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
+      tstrncpy(varDataVal(cgroup), pConsumer->cgroup, TSDB_CGROUP_LEN);
-    varDataSetLen(status, strlen(varDataVal(status)));
+      varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)status, false);
+      colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
 
-    // subscribed topics
+      // app id
-    // TODO: split into multiple rows
+      char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
-    char  topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
+      tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN);
-    char *showStr = taosShowStrArray(pConsumer->assignedTopics);
+      varDataSetLen(appId, strlen(varDataVal(appId)));
-    tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    taosMemoryFree(showStr);
+      colDataAppend(pColInfo, numOfRows, (const char *)appId, false);
-    varDataSetLen(topics, strlen(varDataVal(topics)));
 
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      // status
-    colDataAppend(pColInfo, numOfRows, (const char *)topics, false);
+      char status[20 + VARSTR_HEADER_SIZE] = {0};
+      tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
+      varDataSetLen(status, strlen(varDataVal(status)));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)status, false);
 
-    // pid
+      // one subscribed topic
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true);
+      if (hasTopic) {
+        char        topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+        const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i));
+        tstrncpy(varDataVal(topic), topicName, TSDB_TOPIC_FNAME_LEN);
+        varDataSetLen(topic, strlen(varDataVal(topic)));
+        colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
+      } else {
+        colDataAppend(pColInfo, numOfRows, NULL, true);
+      }
 
-    // end point
+      // pid
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true);
 
-    // up time
+      // end point
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);
 
-    // subscribe time
+      // up time
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
 
-    // rebalance time
+      // subscribe time
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
 
+      // rebalance time
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
+
+      numOfRows++;
+    }
     taosRUnLockLatch(&pConsumer->lock);
     sdbRelease(pSdb, pConsumer);
-
-    numOfRows++;
   }
 
   pShow->numOfRows += numOfRows;

source/dnode/mnode/impl/src/mndDb.c

@@ -1550,10 +1550,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)t, false);
 
-    //  single stable model
-    int8_t m = 0;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&m, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     colDataAppend(pColInfo, rows, (const char *)b, false);

source/dnode/mnode/impl/src/mndDef.c

@@ -196,7 +196,9 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
   return pVgEpNew;
 }
 
-void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); }
+void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
+  if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
+}
 
 int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
   int32_t tlen = 0;

source/dnode/mnode/impl/src/mndQuery.c

@@ -20,7 +20,7 @@
 
 int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
   SMnode     *pMnode = pReq->pNode;
-  SReadHandle handle = {.mnd = pMnode};
+  SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb};
 
   mTrace("msg:%p, in query queue is processing", pReq);
   switch (pReq->rpcMsg.msgType) {

source/dnode/mnode/impl/src/mndShow.c

@@ -85,8 +85,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
     type = TSDB_MGMT_TABLE_VGROUP;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONSUMERS, len) == 0) {
     type = TSDB_MGMT_TABLE_CONSUMERS;
-  } else if (strncasecmp(name, TSDB_PERFS_TABLE_SUBSCRIBES, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_SUBSCRIPTIONS, len) == 0) {
-    type = TSDB_MGMT_TABLE_SUBSCRIBES;
+    type = TSDB_MGMT_TABLE_SUBSCRIPTIONS;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_TRANS, len) == 0) {
     type = TSDB_MGMT_TABLE_TRANS;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_SMAS, len) == 0) {

source/dnode/mnode/impl/src/mndStb.c

@@ -832,6 +832,8 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) {
 }
 
 static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
+  if (pAlter->commentLen != 0) return 0;
+
   if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) {
     terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
     return -1;
@@ -839,15 +841,6 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
 
   for (int32_t i = 0; i < pAlter->numOfFields; ++i) {
     SField *pField = taosArrayGet(pAlter->pFields, i);
-
-    if (pField->type <= 0) {
-      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
-      return -1;
-    }
-    if (pField->bytes <= 0) {
-      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
-      return -1;
-    }
     if (pField->name[0] == 0) {
       terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
       return -1;
@@ -890,6 +883,23 @@ static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) {
   return 0;
 }
 
+static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen) {
+  if (commentLen > 0) {
+    pNew->commentLen = commentLen;
+    pNew->comment = taosMemoryCalloc(1, commentLen);
+    if (pNew->comment == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    memcpy(pNew->comment, pComment, commentLen);
+  }
+
+  if (mndAllocStbSchemas(pOld, pNew) != 0) {
+    return -1;
+  }
+  return 0;
+}
+
 static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ntags) {
   if (pOld->numOfTags + ntags > TSDB_MAX_TAGS) {
     terrno = TSDB_CODE_MND_TOO_MANY_TAGS;
@@ -908,12 +918,12 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
 
   for (int32_t i = 0; i < ntags; i++) {
     SField *pField = taosArrayGet(pFields, i);
-    if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) {
+    if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
       terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
       return -1;
     }
 
-    if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) {
+    if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
       terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
       return -1;
     }
@@ -1034,12 +1044,12 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
 
   for (int32_t i = 0; i < ncols; i++) {
     SField *pField = taosArrayGet(pFields, i);
-    if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) {
+    if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
       terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
       return -1;
     }
 
-    if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) {
+    if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
       terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
       return -1;
     }
@@ -1193,32 +1203,39 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq *
 
   int32_t code = -1;
   STrans *pTrans = NULL;
-  SField *pField0 = taosArrayGet(pAlter->pFields, 0);
+  SField *pField0 = NULL;
 
   switch (pAlter->alterType) {
     case TSDB_ALTER_TABLE_ADD_TAG:
       code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
       break;
     case TSDB_ALTER_TABLE_DROP_TAG:
+      pField0 = taosArrayGet(pAlter->pFields, 0);
       code = mndDropSuperTableTag(pOld, &stbObj, pField0->name);
       break;
     case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
       code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields);
       break;
     case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES:
+      pField0 = taosArrayGet(pAlter->pFields, 0);
       code = mndAlterStbTagBytes(pOld, &stbObj, pField0);
       break;
     case TSDB_ALTER_TABLE_ADD_COLUMN:
       code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
       break;
     case TSDB_ALTER_TABLE_DROP_COLUMN:
+      pField0 = taosArrayGet(pAlter->pFields, 0);
       code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name);
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
+      pField0 = taosArrayGet(pAlter->pFields, 0);
       code = mndAlterStbColumnBytes(pOld, &stbObj, pField0);
       break;
+    case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
+      code = mndUpdateStbComment(pOld, &stbObj, pAlter->comment, pAlter->commentLen);
+      break;
     default:
-      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
+      terrno = TSDB_CODE_OPS_NOT_SUPPORT;
       break;
   }
 

source/dnode/mnode/impl/src/mndStream.c

@@ -259,7 +259,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
     return -1;
   }
 
-#if 1
+#if 0
   printf("|");
   for (int i = 0; i < pStream->outputSchema.nCols; i++) {
     printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -44,6 +44,9 @@ static int32_t  mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubs
 static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg);
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg);
 
+static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
+static void    mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
+
 static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
   SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
   if (pRedoRaw == NULL) return -1;
@@ -71,6 +74,10 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
 
   mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
+
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe);
+
   return sdbSetTable(pMnode->pSdb, table);
 }
 
@@ -164,14 +171,21 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM
   return 0;
 }
 
-static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) {
+static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) {
   int32_t i = 0;
   while (key[i] != TMQ_SEPARATOR) {
     i++;
   }
   memcpy(cgroup, key, i);
   cgroup[i] = 0;
-  strcpy(topic, &key[i + 1]);
+  if (fullName) {
+    strcpy(topic, &key[i + 1]);
+  } else {
+    while (key[i] != '.') {
+      i++;
+    }
+    strcpy(topic, &key[i + 1]);
+  }
   return 0;
 }
 
@@ -419,7 +433,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
     pConsumerNew->updateType = CONSUMER_UPDATE__ADD;
     char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
     char  cgroup[TSDB_CGROUP_LEN];
-    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
     taosArrayPush(pConsumerNew->rebNewTopics, &topic);
     mndReleaseConsumer(pMnode, pConsumerOld);
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
@@ -437,7 +451,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
     pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE;
     char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
     char  cgroup[TSDB_CGROUP_LEN];
-    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
     taosArrayPush(pConsumerNew->rebRemovedTopics, &topic);
     mndReleaseConsumer(pMnode, pConsumerOld);
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
@@ -487,7 +501,7 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
       // split sub key and extract topic
       char topic[TSDB_TOPIC_FNAME_LEN];
       char cgroup[TSDB_CGROUP_LEN];
-      mndSplitSubscribeKey(pRebInfo->key, topic, cgroup);
+      mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
       ASSERT(pTopic);
       taosRLockLatch(&pTopic->lock);
@@ -706,3 +720,124 @@ int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
 END:
   return code;
 }
+
+static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
+  SMnode          *pMnode = pReq->pNode;
+  SSdb            *pSdb = pMnode->pSdb;
+  int32_t          numOfRows = 0;
+  SMqSubscribeObj *pSub = NULL;
+
+  while (numOfRows < rowsCapacity) {
+    pShow->pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pShow->pIter, (void **)&pSub);
+    if (pShow->pIter == NULL) break;
+
+    taosRLockLatch(&pSub->lock);
+
+    if (numOfRows + pSub->vgNum > rowsCapacity) {
+      blockDataEnsureCapacity(pBlock, numOfRows + pSub->vgNum);
+    }
+
+    SMqConsumerEp *pConsumerEp = NULL;
+    void          *pIter = NULL;
+    while (1) {
+      pIter = taosHashIterate(pSub->consumerHash, pIter);
+      if (pIter == NULL) break;
+      pConsumerEp = (SMqConsumerEp *)pIter;
+
+      int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
+      for (int32_t j = 0; j < sz; j++) {
+        SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
+
+        SColumnInfoData *pColInfo;
+        int32_t          cols = 0;
+
+        // topic and cgroup
+        char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+        char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+        mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false);
+        varDataSetLen(topic, strlen(varDataVal(topic)));
+        varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
+
+        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+        colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
+
+        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+        colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
+
+        // vg id
+        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+        colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
+
+        // consumer id
+        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+        colDataAppend(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
+
+        // offset
+#if 0
+      // subscribe time
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
+
+      // rebalance time
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
+#endif
+
+        numOfRows++;
+      }
+    }
+
+    int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
+    for (int32_t i = 0; i < sz; i++) {
+      SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);
+
+      SColumnInfoData *pColInfo;
+      int32_t          cols = 0;
+
+      // topic and cgroup
+      char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
+      char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+      mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false);
+      varDataSetLen(topic, strlen(varDataVal(topic)));
+      varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
+
+      // vg id
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
+
+      // consumer id
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, NULL, true);
+
+      // offset
+#if 0
+      // subscribe time
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
+
+      // rebalance time
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
+#endif
+
+      numOfRows++;
+    }
+
+    taosRUnLockLatch(&pSub->lock);
+    sdbRelease(pSdb, pSub);
+  }
+
+  pShow->numOfRows += numOfRows;
+  return numOfRows;
+}
+
+static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndTopic.c

@@ -35,6 +35,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj
 static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq);
 static int32_t mndProcessDropTopicReq(SNodeMsg *pReq);
 static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp);
+
 static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
 
@@ -61,6 +62,11 @@ int32_t mndInitTopic(SMnode *pMnode) {
 
 void mndCleanupTopic(SMnode *pMnode) {}
 
+const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) {
+  //
+  return strchr(topic, '.') + 1;
+}
+
 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
@@ -292,6 +298,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
 
     SNode *pAst = NULL;
     if (nodesStringToNode(pCreate->ast, &pAst) != 0) {
+      taosMemoryFree(topicObj.ast);
+      taosMemoryFree(topicObj.sql);
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
       return -1;
     }
@@ -301,16 +309,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
     SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
     if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      taosMemoryFree(topicObj.ast);
+      taosMemoryFree(topicObj.sql);
       return -1;
     }
 
     if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) {
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      taosMemoryFree(topicObj.ast);
+      taosMemoryFree(topicObj.sql);
       return -1;
     }
 
     if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) {
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      taosMemoryFree(topicObj.ast);
+      taosMemoryFree(topicObj.sql);
       return -1;
     }
   } else {
@@ -325,6 +339,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg);
   if (pTrans == NULL) {
     mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+    taosMemoryFreeClear(topicObj.ast);
+    taosMemoryFreeClear(topicObj.sql);
     taosMemoryFreeClear(topicObj.physicalPlan);
     return -1;
   }

source/dnode/mnode/impl/src/mndUser.c

@@ -394,6 +394,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) {
 
 static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
   SMnode       *pMnode = pReq->pNode;
+  SSdb         *pSdb = pMnode->pSdb;
+  void         *pIter = NULL;
   int32_t       code = -1;
   SUserObj     *pUser = NULL;
   SUserObj     *pOperUser = NULL;
@@ -429,7 +431,13 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
+  if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) {
+    goto _OVER;
+  }
+
   memcpy(&newUser, pUser, sizeof(SUserObj));
+  newUser.authVersion++;
+  newUser.updateTime = taosGetTimestampMs();
 
   taosRLockLatch(&pUser->lock);
   newUser.readDbs = mndDupDbHash(pUser->readDbs);
@@ -440,63 +448,90 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  int32_t len = strlen(alterReq.dbname) + 1;
-  SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
-  mndReleaseDb(pMnode, pDb);
-
   if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) {
     char pass[TSDB_PASSWORD_LEN + 1] = {0};
     taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass);
     memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN);
-  } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) {
-    newUser.superUser = alterReq.superUser;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) {
-    if (pDb == NULL) {
-      terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto _OVER;
-    }
-    if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto _OVER;
-    }
-    newUser.authVersion++;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) {
-    if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) {
-      terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto _OVER;
-    }
-    newUser.authVersion++;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
-    taosHashClear(newUser.readDbs);
-    newUser.authVersion++;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) {
-    if (pDb == NULL) {
-      terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto _OVER;
-    }
-    if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto _OVER;
-    }
-    newUser.authVersion++;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
-    if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) {
-      terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto _OVER;
-    }
-    newUser.authVersion++;
-  } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) {
-    taosHashClear(newUser.writeDbs);
-    newUser.authVersion++;
-  } else {
-    terrno = TSDB_CODE_MND_INVALID_ALTER_OPER;
-    goto _OVER;
   }
 
-  newUser.updateTime = taosGetTimestampMs();
+  if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) {
+    newUser.superUser = alterReq.superUser;
+  }
 
-  if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) {
+  if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
-    goto _OVER;
+    if (strcmp(alterReq.dbname, "1.*") != 0) {
+      int32_t len = strlen(alterReq.dbname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+      if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+    } else {
+      while (1) {
+        SDbObj *pDb = NULL;
+        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
+        if (pIter == NULL) break;
+        int32_t len = strlen(pDb->name) + 1;
+        taosHashPut(newUser.readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
+        sdbRelease(pSdb, pDb);
+      }
+    }
+  }
+
+  if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
+    if (strcmp(alterReq.dbname, "1.*") != 0) {
+      int32_t len = strlen(alterReq.dbname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+      if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+    } else {
+      while (1) {
+        SDbObj *pDb = NULL;
+        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
+        if (pIter == NULL) break;
+        int32_t len = strlen(pDb->name) + 1;
+        taosHashPut(newUser.writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
+        sdbRelease(pSdb, pDb);
+      }
+    }
+  }
+
+  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
+    if (strcmp(alterReq.dbname, "1.*") != 0) {
+      int32_t len = strlen(alterReq.dbname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+      taosHashRemove(newUser.readDbs, alterReq.dbname, len);
+    } else {
+      taosHashClear(newUser.readDbs);
+    }
+  }
+
+  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
+    if (strcmp(alterReq.dbname, "1.*") != 0) {
+      int32_t len = strlen(alterReq.dbname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        goto _OVER;
+      }
+      taosHashRemove(newUser.writeDbs, alterReq.dbname, len);
+    } else {
+      taosHashClear(newUser.writeDbs);
+    }
   }
 
   code = mndAlterUser(pMnode, pUser, &newUser, pReq);

source/dnode/mnode/impl/src/mndVgroup.c

@@ -682,7 +682,11 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *
         colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false);
 
         char        buf1[20] = {0};
-        const char *role = syncStr(pVgroup->vnodeGid[i].role);
+        SDnodeObj *pDnodeObj = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId);
+        ASSERT(pDnodeObj != NULL);
+        bool isOffLine = !mndIsDnodeOnline(pMnode, pDnodeObj, taosGetTimestampMs());
+        const char *role = isOffLine ? "OFFLINE" : syncStr(pVgroup->vnodeGid[i].role);
+        
         STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);

source/dnode/mnode/impl/test/user/user.cpp

@@ -238,9 +238,10 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_CLEAR_WRITE_DB;
+    alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
+    strcpy(alterReq.dbname, "1.*");
 
     int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq);
     void*   pReq = rpcMallocCont(contLen);
@@ -253,9 +254,10 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_CLEAR_READ_DB;
+    alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
+    strcpy(alterReq.dbname, "1.*");
 
     int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq);
     void*   pReq = rpcMallocCont(contLen);
@@ -365,8 +367,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
     EXPECT_EQ(numOfReadDbs, 1);
     EXPECT_EQ(numOfWriteDbs, 0);
 
-    char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 5);
+    char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 4);
-    EXPECT_STREQ(dbname, "1.d2");
+    EXPECT_TRUE(dbname != NULL);
 
     taosHashCleanup(authRsp.readDbs);
     taosHashCleanup(authRsp.writeDbs);

source/dnode/qnode/src/qnode.c

@@ -51,7 +51,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; }
 
 int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) {
   qTrace("message in qnode query queue is processing");
-  SReadHandle handle = {0};
+  SReadHandle handle = {.pMsgCb = &pQnode->msgCb};
 
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY: {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -34,6 +34,7 @@
 #include "tlockfree.h"
 #include "tlosertree.h"
 #include "tmallocator.h"
+#include "tmsgcb.h"
 #include "tskiplist.h"
 #include "tstream.h"
 #include "ttime.h"
@@ -121,7 +122,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 
 // sma
 
-int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq);
+int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
 int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
 int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
 void    tsdbUidStoreDestory(STbUidStore* pStore);

source/dnode/vnode/src/tq/tq.c

@@ -233,17 +233,19 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
   if (msgType != TDMT_VND_SUBMIT) return 0;
 
+  // make sure msgType == TDMT_VND_SUBMIT
+  if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
+    return -1;
+  }
+
+  if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
+
   void* data = taosMemoryMalloc(msgLen);
   if (data == NULL) {
     return -1;
   }
   memcpy(data, msg, msgLen);
 
-  // make sure msgType == TDMT_VND_SUBMIT
-  if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
-    return -1;
-  }
-
   SRpcMsg req = {
       .msgType = TDMT_VND_STREAM_TRIGGER,
       .pCont = data,
@@ -378,6 +380,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
       SReadHandle    handle = {
              .reader = pReadHandle,
              .meta = pTq->pVnode->pMeta,
+             .pMsgCb = &pTq->pVnode->msgCb,
       };
       pTopic->buffer.output[j].pReadHandle = pReadHandle;
       pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
@@ -857,6 +860,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
         SReadHandle handle = {
             .reader = pExec->pExecReader[i],
             .meta = pTq->pVnode->pMeta,
+            .pMsgCb = &pTq->pVnode->msgCb,
         };
         pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
         ASSERT(pExec->task[i]);
@@ -914,6 +918,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
       SReadHandle    handle = {
              .reader = pStreamReader,
              .meta = pTq->pVnode->pMeta,
+             .pMsgCb = &pTq->pVnode->msgCb,
       };
       pTask->exec.runners[i].inputHandle = pStreamReader;
       pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -70,6 +70,7 @@ static int  tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static void tsdbResetCommitFile(SCommitH *pCommith);
 static int  tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static int  tsdbCommitToTable(SCommitH *pCommith, int tid);
+static int  tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
 static int  tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
 static int  tsdbComparKeyBlock(const void *arg1, const void *arg2);
 static int  tsdbWriteBlockInfo(SCommitH *pCommih);
@@ -211,7 +212,7 @@ int tsdbCommit(STsdb *pRepo) {
 
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
   STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
-  TSKEY     minKey, midKey, maxKey, now;
+  TSKEY         minKey, midKey, maxKey, now;
 
   now = taosGetTimestamp(pCfg->precision);
   minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
@@ -349,7 +350,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
   if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) {
     return -1;
   }
-
+#if 0
   // Loop to commit each table data
   for (int tid = 0; tid < pCommith->niters; tid++) {
     SCommitIter *pIter = pCommith->iters + tid;
@@ -363,6 +364,46 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
       return -1;
     }
   }
+#endif
+  // Loop to commit each table data in mem and file
+  int mIter = 0, fIter = 0;
+  int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx);
+
+  while (true) {
+    SBlockIdx   *pIdx = NULL;
+    SCommitIter *pIter = NULL;
+    if (mIter < pCommith->niters) {
+      pIter = pCommith->iters + mIter;
+      if (fIter < nBlkIdx) {
+        pIdx = taosArrayGet(pCommith->readh.aBlkIdx, fIter);
+      }
+    } else if (fIter < nBlkIdx) {
+      pIdx = taosArrayGet(pCommith->readh.aBlkIdx, fIter);
+    } else {
+      break;
+    }
+    if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->uid <= pIdx->uid))) {
+      if (tsdbCommitToTable(pCommith, mIter) < 0) {
+        tsdbCloseCommitFile(pCommith, true);
+        // revert the file change
+        tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
+        return -1;
+      }
+
+      if (pIdx && (pIter->pTable->uid == pIdx->uid)) {
+        ++fIter;
+      }
+      ++mIter;
+    } else if (pIdx) {
+      if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) {
+        tsdbCloseCommitFile(pCommith, true);
+        // revert the file change
+        tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
+        return -1;
+      }
+      ++fIter;
+    }
+  }
 
   if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) <
       0) {
@@ -838,6 +879,40 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
   return 0;
 }
 
+static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
+  SReadH *pReadh = &pCommith->readh;
+  int     nBlocks = pIdx->numOfBlocks;
+  int     bidx = 0;
+
+  tsdbResetCommitTable(pCommith);
+
+  pReadh->pBlkIdx = pIdx;
+
+  if (tsdbLoadBlockInfo(pReadh, NULL) < 0) {
+    return -1;
+  }
+
+  while (bidx < nBlocks) {
+    if (tsdbMoveBlock(pCommith, bidx) < 0) {
+      tsdbError("vgId:%d failed to move block into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
+                TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
+      return -1;
+    }
+    ++bidx;
+  }
+
+  STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL};
+  TSDB_COMMIT_TABLE(pCommith) = &table;
+
+  if (tsdbWriteBlockInfo(pCommith) < 0) {
+    tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
+              TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
 static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) {
   STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
 
@@ -1386,34 +1461,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
-#if 0
+      if (lastKey != key1) {
-      if (update != TD_ROW_OVERWRITE_UPDATE) {
-        // copy disk data
-        for (int i = 0; i < pDataCols->numOfCols; ++i) {
-          // TODO: dataColAppendVal may fail
-          SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
-            TASSERT(0);
-          }
-          tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode);
-        }
-
-        if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++;
-      }
-      if (update != TD_ROW_DISCARD_UPDATE) {
-        // copy mem data
-        if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
-          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
-          ASSERT(pSchema != NULL);
-        }
-
-        tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
-      }
-      ++(*iter);
-      tSkipListIterNext(pCommitIter->pIter);
-#endif
-
-      if(lastKey != key1) {
         lastKey = key1;
         ++pTarget->numOfRows;
       }
@@ -1484,29 +1532,4 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
   }
 
   return false;
-}
+}
-
-// int tsdbApplyRtn(STsdbRepo *pRepo) {
-//   SRtn       rtn;
-//   SFSIter    fsiter;
-//   STsdbFS *  pfs = REPO_FS(pRepo);
-//   SDFileSet *pSet;
-
-//   // Get retention snapshot
-//   tsdbGetRtnSnap(pRepo, &rtn);
-
-//   tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
-//   while ((pSet = tsdbFSIterNext(&fsiter))) {
-//     if (pSet->fid < rtn.minFid) {
-//       tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
-//                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
-//       continue;
-//     }
-
-//     if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) {
-//       return -1;
-//     }
-//   }
-
-//   return 0;
-// }

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -248,11 +248,13 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
             pMergeInfo->nOperations++;
             pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
             pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
-            lastKey = rowKey;
             if (pCols) {
-              ++pCols->numOfRows;
+              if (lastKey != TSKEY_INITIAL_VAL) {
+                ++pCols->numOfRows;
+              }
               tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
             }
+            lastKey = rowKey;
           } else {
             tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true);
           }
@@ -326,6 +328,11 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo
 
   tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
 
+#ifdef TD_DEBUG_PRINT_ROW
+  printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid,
+         SL_SIZE(pTbData->pData));
+#endif
+
   // Set statistics
   keyMax = TD_ROW_KEY(blkIter.row);
 

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -21,16 +21,19 @@ typedef struct SMemSkipList       SMemSkipList;
 typedef struct SMemSkipListNode   SMemSkipListNode;
 typedef struct SMemSkipListCurosr SMemSkipListCurosr;
 
+#define SL_MAX_LEVEL 5
+
 struct SMemTable {
-  STsdb     *pTsdb;
+  STsdb              *pTsdb;
-  TSKEY      minKey;
+  TSKEY               minKey;
-  TSKEY      maxKey;
+  TSKEY               maxKey;
-  int64_t    minVer;
+  int64_t             minVer;
-  int64_t    maxVer;
+  int64_t             maxVer;
-  int64_t    nRows;
+  int64_t             nRows;
-  int32_t    nHash;
+  int32_t             nHash;
-  int32_t    nBucket;
+  int32_t             nBucket;
-  SMemData **pBuckets;
+  SMemData          **pBuckets;
+  SMemSkipListCurosr *pSlc;
 };
 
 struct SMemSkipListNode {
@@ -60,9 +63,15 @@ struct SMemData {
 
 struct SMemSkipListCurosr {
   SMemSkipList     *pSl;
-  SMemSkipListNode *pNodeC;
+  SMemSkipListNode *pNodes[SL_MAX_LEVEL];
 };
 
+typedef struct {
+  int64_t       version;
+  uint32_t      szRow;
+  const STSRow *pRow;
+} STsdbRow;
+
 #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET))
 
 #define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
@@ -76,7 +85,19 @@ struct SMemSkipListCurosr {
 #define SL_HEAD_NODE_FORWARD(n, l)  SL_NODE_FORWARD(n, l)
 #define SL_TAIL_NODE_BACKWARD(n, l) SL_NODE_FORWARD(n, l)
 
-static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
+static int8_t  tsdbMemSkipListRandLevel(SMemSkipList *pSl);
+static int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow);
+static int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow);
+static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc);
+static void    tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc);
+static void    tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl);
+static void    tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode);
+static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags);
+static void    tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc);
+static void    tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc);
+static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc);
+static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc);
+static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow);
 
 // SMemTable
 int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
@@ -102,6 +123,11 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
     taosMemoryFree(pMemTb);
     return -1;
   }
+  if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pMemTb->pBuckets);
+    taosMemoryFree(pMemTb);
+  }
 
   *ppMemTb = pMemTb;
   return 0;
@@ -110,6 +136,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
 int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) {
   if (pMemTb) {
     // loop to destroy the contents (todo)
+    tsdbMemSkipListCursorDestroy(pMemTb->pSlc);
     taosMemoryFree(pMemTb->pBuckets);
     taosMemoryFree(pMemTb);
   }
@@ -177,52 +204,42 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
   }
 
   // do insert data to SMemData
-  SMemSkipListCurosr slc = {0};
+  SMemSkipListNode *forwards[SL_MAX_LEVEL];
-  const STSRow      *pRow;
+  SMemSkipListNode *pNode;
-  uint32_t           szRow;
+  int32_t           iRow;
-  SDecoder           decoder = {0};
+  STsdbRow          tRow = {.version = version};
+  SEncoder          ec = {0};
+  SDecoder          dc = {0};
 
-  tDecoderInit(&decoder, pSubmitBlk->pData, pSubmitBlk->nData);
+  tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData);
-  for (;;) {
+  tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl);
-    if (tDecodeIsEnd(&decoder)) break;
+  for (iRow = 0;; iRow++) {
+    if (tDecodeIsEnd(&dc)) break;
 
-    if (tDecodeBinary(&decoder, (const uint8_t **)&pRow, &szRow) < 0) {
+    // decode row
+    if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) {
       terrno = TSDB_CODE_INVALID_MSG;
       return -1;
     }
 
-    // check the row (todo)
+    // move cursor
-
+    tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0);
-    //   // move the cursor to position to write (todo)
-    //   int32_t c;
-    //   tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c);
-    //   ASSERT(c);
 
     // encode row
-    int8_t            level = tsdbMemSkipListRandLevel(&pMemData->sl);
+    pNode = tsdbMemSkipListNodeCreate(pPool, &pMemData->sl, &tRow);
-    int32_t           tsize = SL_NODE_SIZE(level) + sizeof(version) + (0 /*todo*/);
-    SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize);
     if (pNode == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
 
-    pNode->level = level;
+    // put the node
-
+    tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode);
-    //   uint8_t *pData = SL_NODE_DATA(pSlNode);
-    //   *(int64_t *)pData = version;
-    //   pData += sizeof(version);
-    //   memcpy(pData, pt, p - pt);
-
-    //   // insert row
-    //   tsdbMemSkipListCursorPut(&slc, pSlNode);
 
     // update status
-    if (pRow->ts < pMemData->minKey) pMemData->minKey = pRow->ts;
+    if (tRow.pRow->ts < pMemData->minKey) pMemData->minKey = tRow.pRow->ts;
-    if (pRow->ts > pMemData->maxKey) pMemData->maxKey = pRow->ts;
+    if (tRow.pRow->ts > pMemData->maxKey) pMemData->maxKey = tRow.pRow->ts;
   }
-  tDecoderClear(&decoder);
+  tDecoderClear(&dc);
-  // tsdbMemSkipListCursorClose(&slc);
 
   // update status
   if (pMemData->minVer == -1) pMemData->minVer = version;
@@ -236,17 +253,128 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
   return 0;
 }
 
-static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
+static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
   int8_t         level = 1;
-  int8_t         tlevel;
+  int8_t         tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
   const uint32_t factor = 4;
 
-  if (pSl->size) {
+  while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
-    tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
+    level++;
-    while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
-      level++;
-    }
   }
 
   return level;
+}
+
+static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) {
+  if (tEncodeI64(pEncoder, pRow->version) < 0) return -1;
+  if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1;
+  return 0;
+}
+
+static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) {
+  if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1;
+  if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1;
+  return 0;
+}
+
+static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc) {
+  *ppSlc = (SMemSkipListCurosr *)taosMemoryCalloc(1, sizeof(**ppSlc) + sizeof(SMemSkipListNode *) * maxLevel);
+  if (*ppSlc == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
+static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryFree(pSlc); }
+
+static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) {
+  SMemSkipListNode *pHead = SL_HEAD_NODE(pSl);
+  pSlc->pSl = pSl;
+  // for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
+  //   pSlc->forwards[iLevel] = pHead;
+  // }
+}
+
+static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) {
+  SMemSkipList     *pSl = pSlc->pSl;
+  SMemSkipListNode *pNodeNext;
+
+  for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
+    // todo
+
+    ASSERT(0);
+  }
+
+  if (pSl->level < pNode->level) {
+    pSl->level = pNode->level;
+  }
+
+  pSl->size += 1;
+}
+
+static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags) {
+  SMemSkipListNode **pForwards = NULL;
+  SMemSkipList      *pSl = pSlc->pSl;
+  int8_t             maxLevel = pSl->maxLevel;
+  SMemSkipListNode  *pHead = SL_HEAD_NODE(pSl);
+  SMemSkipListNode  *pTail = SL_TAIL_NODE(pSl);
+
+  if (pSl->size == 0) {
+    for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
+      pForwards[iLevel] = pHead;
+    }
+  }
+
+  return 0;
+}
+
+static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) {
+  SMemSkipList     *pSl = pSlc->pSl;
+  SMemSkipListNode *pHead = SL_HEAD_NODE(pSl);
+
+  for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
+    pSlc->pNodes[iLevel] = pHead;
+  }
+
+  tsdbMemSkipListCursorMoveToNext(pSlc);
+}
+
+static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) {
+  SMemSkipList     *pSl = pSlc->pSl;
+  SMemSkipListNode *pTail = SL_TAIL_NODE(pSl);
+
+  for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
+    pSlc->pNodes[iLevel] = pTail;
+  }
+
+  tsdbMemSkipListCursorMoveToPrev(pSlc);
+}
+
+static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) {
+  // TODO
+  return 0;
+}
+
+static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) {
+  // TODO
+  return 0;
+}
+
+static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow) {
+  int32_t           tsize;
+  int32_t           ret;
+  int8_t            level = tsdbMemSkipListRandLevel(pSl);
+  SMemSkipListNode *pNode = NULL;
+  SEncoder          ec = {0};
+
+  tEncodeSize(tsdbEncodeRow, pTRow, tsize, ret);
+  pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level));
+  if (pNode) {
+    pNode->level = level;
+    tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize);
+    tsdbEncodeRow(&ec, pTRow);
+    tEncoderClear(&ec);
+  }
+
+  return pNode;
 }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -351,47 +351,38 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData
               pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
   }
 }
-#if 1
+
-int nQUERY = 0;
-#endif
 static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) {
   if (vnodeIsRollup(pVnode)) {
-    int level = 0;
+    int     level = 0;
-#if 0
     int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
+
     for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
-      SRetention* pRetention = retentions + i;
+      SRetention* pRetention = retentions + level;
-      if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) {
+      if (pRetention->keep <= 0) {
+        if (level > 0) {
+          --level;
+        }
+        break;
+      }
+      if ((now - pRetention->keep) <= winSKey) {
         break;
       }
       ++level;
     }
-#endif
+
-#if 1
-    switch ((nQUERY++) % 3) {
-      case 0:
-        level = 0;
-        break;
-      case 1:
-        level = 1;
-        break;
-      default:
-        level = 2;
-        break;
-    }
-#endif
     if (level == TSDB_RETENTION_L0) {
-      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
+      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L0);
       return VND_RSMA0(pVnode);
     } else if (level == TSDB_RETENTION_L1) {
-      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
+      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L1);
       return VND_RSMA1(pVnode);
     } else {
-      tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
+      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L2);
       return VND_RSMA2(pVnode);
     }
   }
-  return pVnode->pTsdb;
+  return VND_TSDB(pVnode);
 }
 
 static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) {
@@ -879,14 +870,14 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
   }
 }
 
-
+static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow,
-static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) {
+                                 TDRowVerT maxVer) {
   STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
       rmem = (STSRow*)SL_GET_NODE_DATA(node);
-#if 0 // TODO: skiplist refactor
+#if 0  // TODO: skiplist refactor
       if (TD_ROW_VER(rmem) > maxVer) {
         rmem = NULL;
       }
@@ -898,7 +889,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
       rimem = (STSRow*)SL_GET_NODE_DATA(node);
-#if 0 // TODO: skiplist refactor
+#if 0  // TODO: skiplist refactor
       if (TD_ROW_VER(rimem) > maxVer) {
         rimem = NULL;
       }
@@ -1677,7 +1668,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
       colIdOfRow2 = tdKvRowColIdAt(row2, k);
     }
 
-    if (colIdOfRow1 < colIdOfRow2) { // the most probability
+    if (colIdOfRow1 < colIdOfRow2) {  // the most probability
       if (colIdOfRow1 < pColInfo->info.colId) {
         ++j;
         continue;
@@ -1720,7 +1711,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
             ++(*curRow);
           }
           ++nResult;
-        } else if (update){
+        } else if (update) {
           mergeOption = 2;
         } else {
           mergeOption = 0;
@@ -1741,7 +1732,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
             ++(*curRow);
           }
           ++nResult;
-        } else if(update) {
+        } else if (update) {
           mergeOption = 2;
         } else {
           mergeOption = 0;
@@ -2018,9 +2009,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           rv2 = TD_ROW_SVER(row2);
         }
 
-        numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+        numOfRows +=
-                           pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
+            mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
-        // numOfRows += 1;
+                               pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = key;
         }
@@ -2028,7 +2019,6 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         cur->win.ekey = key;
         cur->lastKey = key + step;
         cur->mixBlock = true;
-
         moveToNextRowInMem(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
 #if 0
@@ -2064,7 +2054,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         }
 #endif
         if (TD_SUPPORT_UPDATE(pCfg->update)) {
-          doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
+          if (lastKeyAppend != key) {
+            lastKeyAppend = key;
+            ++curRow;
+          }
+          numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
 
           if (rv1 != TD_ROW_SVER(row1)) {
             //            pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
@@ -2074,10 +2068,10 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
             //            pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
             rv2 = TD_ROW_SVER(row2);
           }
+          numOfRows +=
+              mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+                                 pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
 
-          numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
-                             pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
-          // ++numOfRows;
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
             cur->win.skey = key;
           }
@@ -2117,15 +2111,20 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
         int32_t qstart = 0, qend = 0;
         getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend);
-        lastKeyAppend = tsArray[qend];
 
-        numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend);
+        if ((lastKeyAppend != TSKEY_INITIAL_VAL) &&
+            (lastKeyAppend != (ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qstart] : tsArray[qend]))) {
+          ++curRow;
+        }
+        numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend);
         pos += (qend - qstart + 1) * step;
-        if(numOfRows > 0) {
+        if (numOfRows > 0) {
           curRow = numOfRows - 1;
         }
+
         cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qend] : tsArray[qstart];
         cur->lastKey = cur->win.ekey + step;
+        lastKeyAppend = cur->win.ekey;
       }
     } while (numOfRows < pTsdbReadHandle->outputCapacity);
 
@@ -2429,7 +2428,7 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi
   int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
 
   STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdbReadHandle->pTsdb);
-  STimeWindow win = TSWINDOW_INITIALIZER;
+  STimeWindow   win = TSWINDOW_INITIALIZER;
 
   while (true) {
     tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
@@ -2735,7 +2734,6 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   STSchema* pSchema = NULL;
   TSKEY     lastRowKey = TSKEY_INITIAL_VAL;
 
-
   do {
     STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX);
     if (row == NULL) {
@@ -2760,8 +2758,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
       pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0);
       rv = TD_ROW_SVER(row);
     }
-    numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema,
+    numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId,
-                       NULL, pCfg->update, &lastRowKey);
+                                    pSchema, NULL, pCfg->update, &lastRowKey);
 
     if (numOfRows >= maxRowsToRead) {
       moveToNextRowInMem(pCheckInfo);
@@ -2770,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
 
   } while (moveToNextRowInMem(pCheckInfo));
 
-  taosMemoryFreeClear(pSchema); // free the STSChema
+  taosMemoryFreeClear(pSchema);  // free the STSChema
 
   assert(numOfRows <= maxRowsToRead);
 
@@ -2898,8 +2896,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
     //    if (ret != TSDB_CODE_SUCCESS) {
     //      return false;
     //    }
-    mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId,
+    mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols,
-                       NULL, NULL, true, &lastRowKey);
+                       pCheckInfo->tableId, NULL, NULL, true, &lastRowKey);
     taosMemoryFreeClear(pRow);
 
     // update the last key value
@@ -3468,7 +3466,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa
 
   pDataBlockInfo->rows = cur->rows;
   pDataBlockInfo->window = cur->win;
-//  ASSERT(pDataBlockInfo->numOfCols >= (int32_t)(QH_GET_NUM_OF_COLS(pHandle));
+  //  ASSERT(pDataBlockInfo->numOfCols >= (int32_t)(QH_GET_NUM_OF_COLS(pHandle));
 }
 
 /*
@@ -3872,7 +3870,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
 
   if (metaGetTableEntryByUid(&mr, uid) < 0) {
     tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
-    terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     goto _error;
   } else {
     tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
@@ -3942,7 +3940,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr
   metaReaderInit(&mr, (SMeta*)pMeta, 0);
 
   if (metaGetTableEntryByUid(&mr, uid) < 0) {
-    terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     goto _error;
   }
 

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -1698,7 +1698,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
  * @param pReq
  * @return int32_t
  */
-int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq) {
+int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) {
   if (!pReq->rollup) {
     tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid);
     return TSDB_CODE_SUCCESS;
@@ -1742,6 +1742,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq) {
   SReadHandle handle = {
       .reader = pReadHandle,
       .meta = pMeta,
+      .pMsgCb = pMsgCb,
   };
 
   if (param->qmsg1) {
@@ -1942,7 +1943,6 @@ static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid
 
 int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) {
   if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
-    tsdbDebug("vgId:%d no need to update tbUids since empty uidStore", REPO_ID(pTsdb));
     return TSDB_CODE_SUCCESS;
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -142,9 +142,9 @@ _err:
 int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   vTrace("message in vnode query queue is processing");
 #if 0
-  SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode};
+  SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
 #endif
-  SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode};
+  SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY:
       return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg);
@@ -305,7 +305,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq,
     goto _err;
   }
 
-  tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req);
+  tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb);
 
   tDecoderClear(&coder);
   return 0;
@@ -497,6 +497,47 @@ _exit:
   return 0;
 }
 
+static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) {
+  ASSERT(pMsg != NULL);
+  SSubmitMsgIter msgIter = {0};
+  SMeta         *pMeta = pVnode->pMeta;
+  SSubmitBlk    *pBlock = NULL;
+  SSubmitBlkIter blkIter = {0};
+  STSRow        *row = NULL;
+  STSchema      *pSchema = NULL;
+  tb_uid_t       suid = 0;
+
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  while (true) {
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (pBlock == NULL) break;
+    tInitSubmitBlkIter(&msgIter, pBlock, &blkIter);
+    if (blkIter.row == NULL) continue;
+    if (!pSchema || (suid != msgIter.suid)) {
+      if (pSchema) {
+        taosMemoryFreeClear(pSchema);
+      }
+      pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0);  // TODO: use the real schema
+      if(pSchema) {
+        suid = msgIter.suid;
+      }
+    }
+    if(!pSchema) {
+      printf("%s:%d no valid schema\n", tags, __LINE__);
+      continue;
+    }
+    char __tags[128] = {0};
+    snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid);
+    while ((row = tGetSubmitBlkNext(&blkIter))) {
+      tdSRowPrint(row, pSchema, __tags);
+    }
+  }
+
+  taosMemoryFreeClear(pSchema);
+
+  return 0;
+}
+
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SSubmitReq    *pSubmitReq = (SSubmitReq *)pReq;
   SSubmitMsgIter msgIter = {0};
@@ -508,6 +549,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
 
   pRsp->code = 0;
 
+#ifdef TD_DEBUG_PRINT_ROW
+  vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__);
+#endif
+
   // handle the request
   if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) {
     pRsp->code = TSDB_CODE_INVALID_MSG;
@@ -550,7 +595,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
       goto _exit;
     }
 
-    rsp.numOfRows += nRows;
+    rsp.affectedRows += nRows;
+    
   }
 
 _exit:

source/libs/catalog/inc/catalogInt.h

@@ -36,7 +36,7 @@ extern "C" {
 
 #define CTG_DEFAULT_INVALID_VERSION (-1)
 
-#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_TDB_INVALID_TABLE_ID
+#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_PAR_TABLE_NOT_EXIST
 
 enum {
   CTG_READ = 1,

source/libs/executor/inc/executorimpl.h

@@ -62,11 +62,6 @@ enum {
    * 2. when all data within queried time window, it is also denoted as query_completed
    */
   TASK_COMPLETED = 0x2u,
-
-  /* when the result is not completed return to client, this status will be
-   * usually used in case of interval query with interpolation option
-   */
-  TASK_OVER = 0x4u,
 };
 
 typedef struct SResultRowCell {
@@ -288,12 +283,6 @@ typedef struct SOperatorInfo {
   SOperatorFpSet          fpSet;
 } SOperatorInfo;
 
-typedef struct {
-  int32_t      numOfTags;
-  int32_t      numOfCols;
-  SColumnInfo* colList;
-} SQueriedTableInfo;
-
 typedef enum {
   EX_SOURCE_DATA_NOT_READY = 0x1,
   EX_SOURCE_DATA_READY     = 0x2,
@@ -392,10 +381,7 @@ typedef struct SStreamBlockScanInfo {
 } SStreamBlockScanInfo;
 
 typedef struct SSysTableScanInfo {
-  union {
+  SReadHandle readHandle;
-    void* pTransporter;
-    SReadHandle readHandle;
-  };
 
   SRetrieveMetaTableRsp* pRsp;
   SRetrieveTableReq      req;
@@ -632,8 +618,7 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf
 void    initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
 void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf);
 
-void    finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
+void    finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
-                                      SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
 void    doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
                          int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order);
 int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes,
@@ -663,8 +648,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
                                    char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
                                    SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup);
 
-SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
-
 SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo,
                                            SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo);
 
@@ -716,8 +699,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pE
 
 #if 0
 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
-SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
-                                                        SExprInfo* pExpr, int32_t numOfOutput);
 #endif
 
 int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
@@ -742,7 +723,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo);
 void    doDestroyTask(SExecTaskInfo* pTaskInfo);
 int32_t getMaximumIdleDurationSec();
 
-void    doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type);
 void    setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
                                EOPTR_EXEC_MODEL model);

source/libs/executor/src/executorMain.c

@@ -202,7 +202,7 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER);
+  return isTaskKilled(pTaskInfo);
 }
 
 void qDestroyTask(qTaskInfo_t qTaskHandle) {

source/libs/executor/src/executorimpl.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <libs/function/function.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@@ -106,7 +107,6 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo);
 
 static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols);
 
-static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo);
 static void    releaseQueryBuf(size_t numOfTables);
 
 static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr);
@@ -203,9 +203,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData idata = {{0}};
     SSlotDescNode*  pDescNode = nodesListGetNode(pNode->pSlots, i);
-//    if (!pDescNode->output) {  // todo disable it temporarily
+    //    if (!pDescNode->output) {  // todo disable it temporarily
-//      continue;
+    //      continue;
-//    }
+    //    }
 
     idata.info.type = pDescNode->dataType.type;
     idata.info.bytes = pDescNode->dataType.bytes;
@@ -619,7 +619,7 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData*
       continue;
     }
 
-    if (functionNeedToExecute(&pCtx[k])) {
+    if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) {
       pCtx[k].fpSet.process(&pCtx[k]);
     }
 
@@ -802,8 +802,12 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
 static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx) {
   for (int32_t k = 0; k < pOperator->numOfExprs; ++k) {
     if (functionNeedToExecute(&pCtx[k])) {
-      pCtx[k].startTs = startTs;  // this can be set during create the struct
+      pCtx[k].startTs = startTs;
-      pCtx[k].fpSet.process(&pCtx[k]);
+      // this can be set during create the struct
+      // todo add a dummy funtion to avoid process check
+      if (pCtx[k].fpSet.process != NULL) {
+        pCtx[k].fpSet.process(&pCtx[k]);
+      }
     }
   }
 }
@@ -833,7 +837,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
     if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) {  // it is a project query
       SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
       if (pResult->info.rows > 0 && !createNewColModel) {
-        colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows);
+        colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0],
+                        pfCtx->input.numOfRows);
       } else {
         colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows);
       }
@@ -919,6 +924,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
 static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) {
   if (IS_VAR_DATA_TYPE(type)) {
     // todo disable this
+
     //    if (pResultRow->key == NULL) {
     //      pResultRow->key = taosMemoryMalloc(varDataTLen(pData));
     //      varDataCopy(pResultRow->key, pData);
@@ -1072,37 +1078,38 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock*
 }
 
 // set the output buffer for the selectivity + tag query
-static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
+static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   int32_t num = 0;
-  int16_t tagLen = 0;
 
   SqlFunctionCtx*  p = NULL;
-  SqlFunctionCtx** pTagCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES);
+  SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES);
-  if (pTagCtx == NULL) {
+  if (pValCtx == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    int32_t functionId = pCtx[i].functionId;
+    if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
-
+      pValCtx[num++] = &pCtx[i];
-    if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
+    } else if (fmIsAggFunc(pCtx[i].functionId)) {
-      tagLen += pCtx[i].resDataInfo.bytes;
-      pTagCtx[num++] = &pCtx[i];
-    } else if (1 /*(aAggs[functionId].status & FUNCSTATE_SELECTIVITY) != 0*/) {
       p = &pCtx[i];
-    } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) {
-      // tag function may be the group by tag column
-      // ts may be the required primary timestamp column
-      continue;
-    } else {
-      // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ
     }
+//    if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) {
+//      tagLen += pCtx[i].resDataInfo.bytes;
+//      pTagCtx[num++] = &pCtx[i];
+//    } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) {
+//      // tag function may be the group by tag column
+//      // ts may be the required primary timestamp column
+//      continue;
+//    } else {
+//      // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ
+//    }
   }
+
   if (p != NULL) {
-    p->subsidiaries.pCtx = pTagCtx;
+    p->subsidiaries.pCtx = pValCtx;
     p->subsidiaries.num = num;
   } else {
-    taosMemoryFreeClear(pTagCtx);
+    taosMemoryFreeClear(pValCtx);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1211,7 +1218,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
         (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize);
   }
 
-  setCtxTagColumnInfo(pFuncCtx, numOfOutput);
+  setSelectValueColumnInfo(pFuncCtx, numOfOutput);
   return pFuncCtx;
 }
 
@@ -1447,8 +1454,6 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in
   }
 }
 
-#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)
-
 // static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis,
 // SqlFunctionCtx *pCtx, int32_t numOfRows) {
 //   STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
@@ -1873,7 +1878,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo)
 }
 
 void initResultRow(SResultRow* pResultRow) {
-//  pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow));
+  //  pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow));
 }
 
 /*
@@ -1885,7 +1890,8 @@ void initResultRow(SResultRow* pResultRow) {
  *           offset[0]                                  offset[1]                                   offset[2]
  */
 // TODO refactor: some function move away
-void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t numOfExprs, SExecTaskInfo* pTaskInfo) {
+void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t numOfExprs,
+                             SExecTaskInfo* pTaskInfo) {
   SqlFunctionCtx* pCtx = pInfo->pCtx;
   SSDataBlock*    pDataBlock = pInfo->pRes;
   int32_t*        rowCellInfoOffset = pInfo->rowCellInfoOffset;
@@ -2004,8 +2010,8 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
 }
 
 // todo merged with the build group result.
-void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
+void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo,
-                                   SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
+                                   int32_t* rowCellInfoOffset) {
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
     SResultRowPosition* pPos = &pResultRowInfo->pPosition[i];
 
@@ -2018,17 +2024,11 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
     //    }
 
     for (int32_t j = 0; j < numOfOutput; ++j) {
-      pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset);
+      struct SResultRowEntryInfo* pResInfo = getResultCell(pRow, j, rowCellInfoOffset);
-
-      struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo;
       if (!isRowEntryInitialized(pResInfo)) {
         continue;
       }
 
-      if (pCtx[j].fpSet.process) {  // TODO set the dummy function, to avoid the check for null ptr.
-                                    //        pCtx[j].fpSet.finalize(&pCtx[j]);
-      }
-
       if (pRow->numOfRows < pResInfo->numOfRes) {
         pRow->numOfRows = pResInfo->numOfRes;
       }
@@ -2182,17 +2182,15 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
   int32_t numOfResult = pBlock->info.rows;  // there are already exists result rows
 
   int32_t start = 0;
-  int32_t step = -1;
+  int32_t step = 1;
 
   // qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv));
   assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC);
 
   if (orderType == TSDB_ORDER_ASC) {
     start = pGroupResInfo->index;
-    step = 1;
   } else {  // desc order copy all data
     start = numOfRows - pGroupResInfo->index - 1;
-    step = -1;
   }
 
   for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
@@ -2219,11 +2217,16 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
       pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset);
       if (pCtx[j].fpSet.process) {
         pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
+        // do nothing, todo refactor
       } else {
-        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
+        // expand the result into multiple rows. E.g., _wstartts, top(k, 20)
-
+        // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
-        char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
+          SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
-        colDataAppend(pColInfoData, pBlock->info.rows, in, pCtx[j].resultInfo->isNullRes);
+          char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
+          for(int32_t k = 0; k < pRow->numOfRows; ++k) {
+            colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+          }
       }
     }
 
@@ -2240,12 +2243,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
   return 0;
 }
 
-void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) {
+void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo,
+                            SDiskbasedBuf* pBuf) {
   assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup);
 
-  int32_t* rowCellOffset = pbInfo->rowCellInfoOffset;
+  int32_t*        rowCellOffset = pbInfo->rowCellInfoOffset;
-  SSDataBlock* pBlock    = pbInfo->pRes;
+  SSDataBlock*    pBlock = pbInfo->pRes;
-  SqlFunctionCtx* pCtx   = pbInfo->pCtx;
+  SqlFunctionCtx* pCtx = pbInfo->pCtx;
 
   blockDataCleanup(pBlock);
   if (!hasRemainDataInCurrentGroup(pGroupResInfo)) {
@@ -2419,7 +2423,8 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) {
 
   SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
   if (pSummary->pRecoder != NULL) {
-    qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, "
+    qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64
+           " us, total blocks:%d, "
            "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
            GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks,
            pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows);
@@ -2954,7 +2959,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
     relocateColumnData(pRes, pColList, pBlock->pDataBlock);
     taosArrayDestroy(pBlock->pDataBlock);
     taosMemoryFree(pBlock);
-//    blockDataDestroy(pBlock);
+    //    blockDataDestroy(pBlock);
   }
 
   pRes->info.rows = numOfRows;
@@ -3259,7 +3264,8 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock,
+                                          SExecTaskInfo* pTaskInfo) {
   SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo));
   SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -3302,29 +3308,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock
 
   pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
                                          destroyExchangeOperatorInfo, NULL, NULL, NULL);
-
+  pInfo->pTransporter = pTransporter;
-#if 1
-  {  // todo refactor
-    SRpcInit rpcInit;
-    memset(&rpcInit, 0, sizeof(rpcInit));
-    rpcInit.localPort = 0;
-    rpcInit.label = "EX";
-    rpcInit.numOfThreads = 1;
-    rpcInit.cfp = qProcessFetchRsp;
-    rpcInit.sessions = tsMaxConnections;
-    rpcInit.connType = TAOS_CONN_CLIENT;
-    rpcInit.user = (char*)"root";
-    rpcInit.idleTime = tsShellActivityTimer * 1000;
-    rpcInit.ckey = "key";
-    rpcInit.spi = 1;
-    rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6";
-
-    pInfo->pTransporter = rpcOpen(&rpcInit);
-    if (pInfo->pTransporter == NULL) {
-      return NULL;  // todo
-    }
-  }
-#endif
 
   return pOperator;
 
@@ -3694,10 +3678,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
 
   SOptrBasicInfo* pInfo = &pAggInfo->binfo;
 
-  int32_t        order = TSDB_ORDER_ASC;
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
-  bool newgroup = true;
   while (1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@@ -3710,6 +3692,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
     //      setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfExprs);
     //    }
 
+    int32_t order = getTableScanOrder(pOperator);
+
     // there is an scalar expression that needs to be calculated before apply the group aggregation.
     if (pAggInfo->pScalarExprInfo != NULL) {
       int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx,
@@ -3742,8 +3726,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
   }
 
   closeAllResultRows(&pAggInfo->binfo.resultRowInfo);
-  finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf,
+  finalizeMultiTupleQueryResult(pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo,
-                                &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset);
+                                pAggInfo->binfo.rowCellInfoOffset);
 
   initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false);
   OPTR_SET_OPENED(pOperator);
@@ -3974,6 +3958,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
   SSDataBlock* pRes = pInfo->pRes;
   blockDataCleanup(pRes);
 
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
   }
@@ -4025,7 +4010,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       }
     }
 
-    // todo dynamic set tags
+    // todo set tags
+
     //    STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
     //    if (pTableQueryInfo != NULL) {
     //      setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs);
@@ -4037,8 +4023,11 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, false);
     blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
 
-    projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs,
+    pTaskInfo->code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs,
                           pProjectInfo->pPseudoColInfo);
+    if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, pTaskInfo->code);
+    }
 
     int32_t status = handleLimitOffset(pOperator, pBlock);
     if (status == PROJECT_RETRIEVE_CONTINUE) {
@@ -4099,8 +4088,8 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
   }
 
   // todo handle different group data interpolation
-  bool n = false;
+  bool  n = false;
-  bool *newgroup = &n;
+  bool* newgroup = &n;
   doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo);
   if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) {
     return pResBlock;
@@ -4206,7 +4195,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
     }
   }
 
-  taosMemoryFree(pOperator->pExpr);
+  taosMemoryFreeClear(pOperator->pExpr);
   taosMemoryFreeClear(pOperator->info);
   taosMemoryFreeClear(pOperator);
 }
@@ -4224,7 +4213,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   }
 
   uint32_t defaultPgsz = 4096;
-  while(defaultPgsz < pAggSup->resultRowSize*4) {
+  while (defaultPgsz < pAggSup->resultRowSize * 4) {
     defaultPgsz <<= 1u;
   }
 
@@ -4286,11 +4275,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf
     SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i);
     for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) {
       STableKeyInfo* pk = taosArrayGet(pa, j);
-
       STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++];
-      //      pTQueryInfo->uid = pk->uid;
       pTQueryInfo->lastKey = pk->lastKey;
-      //      pTQueryInfo->groupIndex = i;
     }
   }
 
@@ -4309,7 +4295,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
     goto _error;
   }
 
-  int32_t numOfRows = 1;
+  int32_t numOfRows = 10;
   size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
 
   initResultSizeInfo(pOperator, numOfRows);
@@ -4320,9 +4306,6 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
     goto _error;
   }
 
-  pOperator->resultInfo.capacity = 4096;
-  pOperator->resultInfo.threshold = 4096 * 0.75;
-
   int32_t numOfGroup = 10;  // todo replaced with true value
   pInfo->groupId = INT32_MIN;
   initResultRowInfo(&pInfo->binfo.resultRowInfo, numOfGroup);
@@ -4375,7 +4358,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
 }
 
 void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
-  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*) param;
+  SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
 }
 
 void destroyAggOperatorInfo(void* param, int32_t numOfOutput) {
@@ -4391,6 +4374,9 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
 }
 
 static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
+  if (NULL == param) {
+    return;
+  }
   SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param;
   doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
   cleanupAggSup(&pInfo->aggSup);
@@ -4487,8 +4473,9 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
 }
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
-                                      SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode,
+                                      SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock,
-                                      bool multigroupResult, SExecTaskInfo* pTaskInfo) {
+                                      int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult,
+                                      SExecTaskInfo* pTaskInfo) {
   SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -4522,8 +4509,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp
   SResultInfo* pResultInfo = &pOperator->resultInfo;
   initResultSizeInfo(pOperator, 4096);
 
-  int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity,
+  int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, pTaskInfo->id.str,
-                              pTaskInfo->id.str, pInterval, type);
+                              pInterval, type);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4548,42 +4535,6 @@ _error:
   return NULL;
 }
 
-static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
-  int32_t j = 0;
-
-  if (TSDB_COL_IS_TAG(pExpr->pParam[0].pCol->type)) {
-    if (pExpr->pParam[0].pCol->colId == TSDB_TBNAME_COLUMN_INDEX) {
-      return TSDB_TBNAME_COLUMN_INDEX;
-    }
-
-    while (j < pTableInfo->numOfTags) {
-      if (pExpr->pParam[0].pCol->colId == pTagCols[j].colId) {
-        return j;
-      }
-
-      j += 1;
-    }
-
-  } /*else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) {  // user specified column data
-    return TSDB_UD_COLUMN_INDEX;
-  } else {
-    while (j < pTableInfo->numOfCols) {
-      if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) {
-        return j;
-      }
-
-      j += 1;
-    }
-  }*/
-
-  return INT32_MIN;  // return a less than TSDB_TBNAME_COLUMN_INDEX value
-}
-
-bool validateExprColumnInfo(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
-  int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols);
-  return j != INT32_MIN;
-}
-
 static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision,
                                   const char* name) {
   SResSchema s = {0};
@@ -4736,12 +4687,13 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t
                                   uint64_t queryId, uint64_t taskId);
 static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo);
 static SArray* extractColumnInfo(SNodeList* pNodeList);
-static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type);
+static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
+                                   int32_t type);
 
 static SArray* createSortInfo(SNodeList* pNodeList);
 static SArray* extractPartitionColInfo(SNodeList* pNodeList);
 static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
-static void    setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode);
+static void    setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
 
 static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) {
   SInterval interval = {
@@ -4776,33 +4728,34 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SSDataBlock* pResBlock = createResDataBlock(pDescNode);
 
       SQueryTableDataCond cond = {0};
-      int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
+      int32_t             code = initQueryTableDataCond(&cond, pTableScanNode);
       if (code != TSDB_CODE_SUCCESS) {
         return NULL;
       }
 
-      SInterval interval = extractIntervalInfo(pTableScanNode);
+      SInterval      interval = extractIntervalInfo(pTableScanNode);
-      SOperatorInfo* pOperator = createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired,
+      SOperatorInfo* pOperator = createTableScanOperatorInfo(
-                                         pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions,
+          pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock,
-                                         &interval, pTableScanNode->ratio, pTaskInfo);
+          pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
       SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
       SSDataBlock*        pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
-      return createExchangeOperatorInfo(pExchange->pSrcEndPoints, pResBlock, pTaskInfo);
+      return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
       SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
 
-      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
+      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
+                                        queryId, taskId);
       SArray* tableIdList = extractTableIdList(pTableGroupInfo);
 
       SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
-      SSDataBlock* pResBlock = createResDataBlock(pDescNode);
+      SSDataBlock*        pResBlock = createResDataBlock(pDescNode);
 
-      int32_t numOfCols = 0;
+      int32_t        numOfCols = 0;
-      SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
+      SArray*        pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
       SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo,
                                                               pScanPhyNode->node.pConditions);
       taosArrayDestroy(tableIdList);
@@ -4822,25 +4775,27 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
           pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
-      STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode;
+      STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
 
       SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
 
       SSDataBlock* pResBlock = createResDataBlock(pDescNode);
 
-      int32_t  code =
+      int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
-          doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
+                                        queryId, taskId);
       if (code != TSDB_CODE_SUCCESS) {
         return NULL;
       }
 
-      int32_t num = 0;
+      int32_t    num = 0;
       SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num);
 
       int32_t numOfOutputCols = 0;
-      SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID);
+      SArray* colList =
+          extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID);
 
-      SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo);
+      SOperatorInfo* pOperator =
+          createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo);
       return pOperator;
     } else {
       ASSERT(0);
@@ -4887,7 +4842,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr,
                                           pTaskInfo, pTableGroupInfo);
     }
-  } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type) {
+  } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
 
     SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -4925,13 +4880,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
   } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) {
     SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
 
-    STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType};
+    STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark,
+                             .calTrigger = pSessionNode->window.triggerType};
 
     SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num);
     SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
     int32_t      tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
 
-    pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo);
+    pOptr =
+        createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) {
     SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode;
     SArray*              pColList = extractPartitionColInfo(pPartNode->pPartitionKeys);
@@ -4957,11 +4914,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createMergeJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
     SFillPhysiNode* pFillNode = (SFillPhysiNode*)pPhyNode;
-    SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+    SSDataBlock*    pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
-    SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num);
+    SExprInfo*      pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num);
 
     SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval;
-    pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo);
+    pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode,
+                                   (SNodeListNode*)pFillNode->pValues, false, pTaskInfo);
   } else {
     ASSERT(0);
   }
@@ -5101,7 +5059,8 @@ SArray* createSortInfo(SNodeList* pNodeList) {
   return pList;
 }
 
-SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type) {
+SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
+                            int32_t type) {
   size_t  numOfCols = LIST_LENGTH(pNodeList);
   SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo));
   if (pList == NULL) {
@@ -5114,10 +5073,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
     SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
 
     SColMatchInfo c = {0};
-    c.output       = true;
+    c.output = true;
-    c.colId        = pColNode->colId;
+    c.colId = pColNode->colId;
-    c.srcSlotId    = pColNode->slotId;
+    c.srcSlotId = pColNode->slotId;
-    c.matchType    = type;
+    c.matchType = type;
     c.targetSlotId = pNode->slotId;
     taosArrayPush(pList, &c);
   }
@@ -5235,28 +5194,6 @@ _complete:
   return code;
 }
 
-static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs,
-                                             int32_t numOfOutput, int32_t tagLen, bool superTable) {
-  for (int32_t i = 0; i < numOfOutput; ++i) {
-    int16_t functId = getExprFunctionId(&pExprs[i]);
-
-    if (functId == FUNCTION_TOP || functId == FUNCTION_BOTTOM) {
-      int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols);
-      if (j < 0 || j >= pTableInfo->numOfCols) {
-        return TSDB_CODE_QRY_INVALID_MSG;
-      } else {
-        SColumnInfo* pCol = &pTableInfo->colList[j];
-        //        int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i,
-        //                                        &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes,
-        //                                        &pExprs[i].base.interBytes, tagLen, superTable, NULL);
-        //        assert(ret == TSDB_CODE_SUCCESS);
-      }
-    }
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) {
   const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512);
 
@@ -5527,8 +5464,8 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
 }
 
 SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo,
-                                      int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
+                                           int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition,
-                                      SExecTaskInfo* pTaskInfo) {
+                                           SExecTaskInfo* pTaskInfo) {
   SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pOperator == NULL || pInfo == NULL) {
@@ -5537,14 +5474,14 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
 
   initResultSizeInfo(pOperator, 4096);
 
-  pInfo->pRes          = pResBlock;
+  pInfo->pRes = pResBlock;
-  pOperator->name      = "MergeJoinOperator";
+  pOperator->name = "MergeJoinOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
-  pOperator->blocking  = false;
+  pOperator->blocking = false;
-  pOperator->status    = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->pExpr     = pExprInfo;
+  pOperator->pExpr = pExprInfo;
   pOperator->numOfExprs = numOfCols;
-  pOperator->info      = pInfo;
+  pOperator->info = pInfo;
   pOperator->pTaskInfo = pTaskInfo;
 
   SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
@@ -5568,9 +5505,9 @@ _error:
 }
 
 void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
-  pColumn->slotId    = pColumnNode->slotId;
+  pColumn->slotId = pColumnNode->slotId;
-  pColumn->type      = pColumnNode->node.resType.type;
+  pColumn->type = pColumnNode->node.resType.type;
-  pColumn->bytes     = pColumnNode->node.resType.bytes;
+  pColumn->bytes = pColumnNode->node.resType.bytes;
   pColumn->precision = pColumnNode->node.resType.precision;
-  pColumn->scale     = pColumnNode->node.resType.scale;
+  pColumn->scale = pColumnNode->node.resType.scale;
 }

source/libs/executor/src/groupoperator.c

@@ -262,6 +262,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
     return NULL;
   }
 
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
   SGroupbyOperatorInfo* pInfo = pOperator->info;
   SSDataBlock* pRes = pInfo->binfo.pRes;
 
@@ -289,7 +291,10 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
 
     // there is an scalar expression that needs to be calculated right before apply the group aggregation.
     if (pInfo->pScalarExprInfo != NULL) {
-      projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL);
+      pTaskInfo->code = projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL);
+      if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+        longjmp(pTaskInfo->env, pTaskInfo->code);
+      }
     }
 
     //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfExprs);
@@ -299,8 +304,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   pOperator->status = OP_RES_TO_RETURN;
   closeAllResultRows(&pInfo->binfo.resultRowInfo);
 
-  finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf,
+  finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo,
-                                &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
+                                pInfo->binfo.rowCellInfoOffset);
   //  if (!stableQuery) { // finalize include the update of result rows
   //    finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs);
   //  } else {

source/libs/executor/src/scanoperator.c

@@ -1057,7 +1057,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
       pMsgSendInfo->fp = loadSysTableCallback;
 
       int64_t transporterId = 0;
-      int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo);
+      int32_t code =
+          asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo);
       tsem_wait(&pInfo->ready);
 
       if (pTaskInfo->code) {
@@ -1182,29 +1183,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe
   } else {
     tsem_init(&pInfo->ready, 0, 0);
     pInfo->epSet = epset;
-
+    pInfo->readHandle = *(SReadHandle*)readHandle;
-#if 1
-    {  // todo refactor
-      SRpcInit rpcInit;
-      memset(&rpcInit, 0, sizeof(rpcInit));
-      rpcInit.localPort = 0;
-      rpcInit.label = "DB-META";
-      rpcInit.numOfThreads = 1;
-      rpcInit.cfp = qProcessFetchRsp;
-      rpcInit.sessions = tsMaxConnections;
-      rpcInit.connType = TAOS_CONN_CLIENT;
-      rpcInit.user = (char*)"root";
-      rpcInit.idleTime = tsShellActivityTimer * 1000;
-      rpcInit.ckey = "key";
-      rpcInit.spi = 1;
-      rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6";
-
-      pInfo->pTransporter = rpcOpen(&rpcInit);
-      if (pInfo->pTransporter == NULL) {
-        return NULL;  // todo
-      }
-    }
-#endif
   }
 
   pOperator->name = "SysTableScanOperator";

source/libs/executor/src/sortoperator.c

@@ -114,7 +114,10 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) {
   SOperatorInfo* pOperator = param;
   SSortOperatorInfo* pSort = pOperator->info;
   if (pOperator->pExpr != NULL) {
-    projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL);
+    int32_t code = projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL);
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pOperator->pTaskInfo->env, code);
+    }
   }
 }
 

source/libs/executor/src/timewindowoperator.c

@@ -798,8 +798,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
   }
 
   closeAllResultRows(&pInfo->binfo.resultRowInfo);
-  finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf,
+  finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo,
-                                &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
+                                pInfo->binfo.rowCellInfoOffset);
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
   OPTR_SET_OPENED(pOperator);
@@ -916,7 +916,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
 
   pOperator->status = OP_RES_TO_RETURN;
   closeAllResultRows(&pBInfo->resultRowInfo);
-  finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
+  finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
                                 pBInfo->rowCellInfoOffset);
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
@@ -1293,7 +1293,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
   // restore the value
   pOperator->status = OP_RES_TO_RETURN;
   closeAllResultRows(&pBInfo->resultRowInfo);
-  finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
+  finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
                                 pBInfo->rowCellInfoOffset);
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);

source/libs/executor/test/executorTests.cpp

@@ -23,34 +23,34 @@
 #pragma GCC diagnostic ignored "-Wsign-compare"
 #include "os.h"
 
-#include "tglobal.h"
+#include "executor.h"
 #include "executorimpl.h"
 #include "function.h"
-#include "taos.h"
-#include "tdef.h"
-#include "tvariant.h"
-#include "tdatablock.h"
-#include "trpc.h"
 #include "stub.h"
-#include "executor.h"
+#include "taos.h"
+#include "tdatablock.h"
+#include "tdef.h"
+#include "tglobal.h"
 #include "tmsg.h"
 #include "tname.h"
+#include "trpc.h"
+#include "tvariant.h"
 
 namespace {
 
 enum {
   data_rand = 0x1,
-  data_asc  = 0x2,
+  data_asc = 0x2,
   data_desc = 0x3,
 };
 
 typedef struct SDummyInputInfo {
-  int32_t      totalPages;     // numOfPages
+  int32_t      totalPages;  // numOfPages
   int32_t      current;
   int32_t      startVal;
   int32_t      type;
   int32_t      numOfRowsPerPage;
-  int32_t      numOfCols;      // number of columns
+  int32_t      numOfCols;  // number of columns
   int64_t      tsStart;
   SSDataBlock* pBlock;
 } SDummyInputInfo;
@@ -75,26 +75,26 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
 
     taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
 
-//    SColumnInfoData colInfo1 = {0};
+    //    SColumnInfoData colInfo1 = {0};
-//    colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
+    //    colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
-//    colInfo1.info.bytes = 40;
+    //    colInfo1.info.bytes = 40;
-//    colInfo1.info.colId = 2;
+    //    colInfo1.info.colId = 2;
-//
+    //
-//    colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t);
+    //    colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t);
-//    colInfo1.varmeta.length = 0;
+    //    colInfo1.varmeta.length = 0;
-//    colInfo1.varmeta.offset = static_cast<int32_t*>(taosMemoryCalloc(1, numOfRows * sizeof(int32_t)));
+    //    colInfo1.varmeta.offset = static_cast<int32_t*>(taosMemoryCalloc(1, numOfRows * sizeof(int32_t)));
-//
+    //
-//    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
+    //    taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
   } else {
     blockDataCleanup(pInfo->pBlock);
   }
 
   SSDataBlock* pBlock = pInfo->pBlock;
 
-  char buf[128] = {0};
+  char    buf[128] = {0};
-  char b1[128] = {0};
+  char    b1[128] = {0};
   int32_t v = 0;
-  for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) {
+  for (int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) {
     SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
 
     if (pInfo->type == data_desc) {
@@ -107,11 +107,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
 
     colDataAppend(pColInfo, i, reinterpret_cast<const char*>(&v), false);
 
-//    sprintf(buf, "this is %d row", i);
+    //    sprintf(buf, "this is %d row", i);
-//    STR_TO_VARSTR(b1, buf);
+    //    STR_TO_VARSTR(b1, buf);
-//
+    //
-//    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
+    //    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
-//    colDataAppend(pColInfo2, i, b1, false);
+    //    colDataAppend(pColInfo2, i, b1, false);
   }
 
   pBlock->info.rows = pInfo->numOfRowsPerPage;
@@ -137,7 +137,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
     colInfo.info.bytes = sizeof(int64_t);
     colInfo.info.colId = 1;
     colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t)));
-//    colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
+    //    colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
 
     taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
 
@@ -156,11 +156,11 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
 
   SSDataBlock* pBlock = pInfo->pBlock;
 
-  char buf[128] = {0};
+  char    buf[128] = {0};
-  char b1[128] = {0};
+  char    b1[128] = {0};
   int64_t ts = 0;
-  int32_t v  = 0;
+  int32_t v = 0;
-  for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) {
+  for (int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) {
     SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
 
     ts = (++pInfo->tsStart);
@@ -177,11 +177,11 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
 
     colDataAppend(pColInfo1, i, reinterpret_cast<const char*>(&v), false);
 
-//    sprintf(buf, "this is %d row", i);
+    //    sprintf(buf, "this is %d row", i);
-//    STR_TO_VARSTR(b1, buf);
+    //    STR_TO_VARSTR(b1, buf);
-//
+    //
-//    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
+    //    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
-//    colDataAppend(pColInfo2, i, b1, false);
+    //    colDataAppend(pColInfo2, i, b1, false);
   }
 
   pBlock->info.rows = pInfo->numOfRowsPerPage;
@@ -191,10 +191,10 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
 
   blockDataUpdateTsWindow(pBlock);
   return pBlock;
-
 }
 
-SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) {
+SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type,
+                                   int32_t numOfCols) {
   SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(taosMemoryCalloc(1, sizeof(SOperatorInfo)));
   pOperator->name = "dummyInputOpertor4Test";
 
@@ -204,24 +204,25 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_
     pOperator->fpSet.getNextFn = get2ColsDummyBlock;
   }
 
-  SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo));
+  SDummyInputInfo* pInfo = (SDummyInputInfo*)taosMemoryCalloc(1, sizeof(SDummyInputInfo));
   pInfo->totalPages = numOfBlocks;
-  pInfo->startVal   = startVal;
+  pInfo->startVal = startVal;
   pInfo->numOfRowsPerPage = rowsPerPage;
-  pInfo->type       = type;
+  pInfo->type = type;
-  pInfo->tsStart    = 1620000000000;
+  pInfo->tsStart = 1620000000000;
 
   pOperator->info = pInfo;
   return pOperator;
 }
-}
+}  // namespace
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
 
 TEST(testCase, build_executor_tree_Test) {
-  const char* msg = "{\n"
+  const char* msg =
+      "{\n"
       "    \"NodeType\": \"48\",\n"
       "    \"Name\": \"PhysiSubplan\",\n"
       "    \"PhysiSubplan\": {\n"
@@ -938,16 +939,19 @@ TEST(testCase, build_executor_tree_Test) {
 
   SExecTaskInfo* pTaskInfo = nullptr;
   DataSinkHandle sinkHandle = nullptr;
-  SReadHandle handle = { reinterpret_cast<void*>(0x1), reinterpret_cast<void*>(0x1), NULL };
+  SReadHandle    handle = {reinterpret_cast<void*>(0x1), reinterpret_cast<void*>(0x1), NULL};
 
-  struct SSubplan *plan = NULL;
+  struct SSubplan* plan = NULL;
-  int32_t code = qStringToSubplan(msg, &plan);
+  int32_t          code = qStringToSubplan(msg, &plan);
   ASSERT_EQ(code, 0);
 
-  code = qCreateExecTask(&handle, 2, 1, plan, (void**) &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH);
+  code = qCreateExecTask(&handle, 2, 1, plan, (void**)&pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH);
   ASSERT_EQ(code, 0);
 }
-
+TEST(testCase, index_plan_test) {
+  // add later
+  EXPECT_EQ(0, 0);
+}
 #if 0
 
 TEST(testCase, inMem_sort_Test) {
@@ -983,19 +987,19 @@ TEST(testCase, inMem_sort_Test) {
 
 typedef struct su {
   int32_t v;
-  char   *c;
+  char*   c;
 } su;
 
 int32_t cmp(const void* p1, const void* p2) {
-  su* v1 = (su*) p1;
+  su* v1 = (su*)p1;
-  su* v2 = (su*) p2;
+  su* v2 = (su*)p2;
 
-  int32_t x1 = *(int32_t*) v1->c;
+  int32_t x1 = *(int32_t*)v1->c;
-  int32_t x2 = *(int32_t*) v2->c;
+  int32_t x2 = *(int32_t*)v2->c;
   if (x1 == x2) {
     return 0;
   } else {
-    return x1 < x2? -1:1;
+    return x1 < x2 ? -1 : 1;
   }
 }
 
@@ -1228,4 +1232,4 @@ TEST(testCase, time_interval_Operator_Test) {
 }
 #endif
 
-#pragma GCC diagnosti
+#pragma GCC diagnosti

source/libs/function/inc/builtinsimpl.h

@@ -25,6 +25,7 @@ extern "C" {
 
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
 int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
 
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
@@ -37,11 +38,11 @@ bool getSumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t sumFunction(SqlFunctionCtx *pCtx);
 int32_t sumInvertFunction(SqlFunctionCtx *pCtx);
 
-bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+bool minmaxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
-bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t minFunction(SqlFunctionCtx* pCtx);
 int32_t maxFunction(SqlFunctionCtx *pCtx);
+int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
 bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -55,6 +56,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx);
 int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t stddevInvertFunction(SqlFunctionCtx* pCtx);
 
+bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t leastSQRFunction(SqlFunctionCtx* pCtx);
+int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx);
+
 bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t percentileFunction(SqlFunctionCtx *pCtx);
@@ -70,6 +77,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx);
 
 bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 int32_t topFunction(SqlFunctionCtx *pCtx);
+int32_t bottomFunction(SqlFunctionCtx *pCtx);
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
 bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
@@ -82,6 +90,12 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultIn
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
+bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t stateCountFunction(SqlFunctionCtx* pCtx);
+
+bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/function/inc/functionMgtInt.h

@@ -40,6 +40,7 @@ extern "C" {
 #define FUNC_MGT_MULTI_RES_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(11)
 #define FUNC_MGT_SCAN_PC_FUNC           FUNC_MGT_FUNC_CLASSIFICATION_MASK(12)
 #define FUNC_MGT_SELECT_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(13)
+#define FUNC_MGT_REPEAT_SCAN_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(14)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/inc/taggfunction.h

@@ -67,7 +67,7 @@ bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const cha
  */
 static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32_t bufLen) {
   pResInfo->initialized = true;  // the this struct has been initialized flag
-  
+
   pResInfo->complete  = false;
   pResInfo->numOfRes  = 0;
   memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);

source/libs/function/src/builtins.c

@@ -207,7 +207,8 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
 }
 
 static int32_t translateBottom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  // todo
+  SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
+  pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -225,6 +226,23 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (3 != numOfParams) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  for (int32_t i = 0; i < numOfParams; ++i) {
+    uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
+    if (!IS_NUMERIC_TYPE(colType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+  }
+
+  pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY };
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (4 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -241,7 +259,27 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  pFunc->node.resType = (SDataType) { .bytes = 512, .type = TSDB_DATA_TYPE_BINARY };
+  pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (3 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (!IS_NUMERIC_TYPE(colType)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+      (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BIGINT &&
+       ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_DOUBLE)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT };
   return TSDB_CODE_SUCCESS;
 }
 
@@ -273,7 +311,8 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   }
 
   SExprNode* p1 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
-  if (!IS_NUMERIC_TYPE(p1->resType.type)) {
+  if (!IS_SIGNED_NUMERIC_TYPE(p1->resType.type) && !IS_FLOAT_TYPE(p1->resType.type) &&
+      TSDB_DATA_TYPE_BOOL != p1->resType.type) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
   pFunc->node.resType = p1->resType;
@@ -509,9 +548,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateInOutNum,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
-    .initFunc     = minFunctionSetup,
+    .initFunc     = minmaxFunctionSetup,
     .processFunc  = minFunction,
-    .finalizeFunc = functionFinalize
+    .finalizeFunc = minmaxFunctionFinalize
   },
   {
     .name = "max",
@@ -520,9 +559,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateInOutNum,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
-    .initFunc     = maxFunctionSetup,
+    .initFunc     = minmaxFunctionSetup,
     .processFunc  = maxFunction,
-    .finalizeFunc = functionFinalize
+    .finalizeFunc = minmaxFunctionFinalize
   },
   {
     .name = "stddev",
@@ -535,6 +574,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .finalizeFunc = stddevFinalize,
     .invertFunc   = stddevInvertFunction
   },
+  {
+    .name = "leastsquares",
+    .type = FUNCTION_TYPE_LEASTSQUARES,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateLeastSQR,
+    .getEnvFunc   = getLeastSQRFuncEnv,
+    .initFunc     = leastSQRFunctionSetup,
+    .processFunc  = leastSQRFunction,
+    .finalizeFunc = leastSQRFinalize,
+    .invertFunc   = leastSQRInvertFunction
+  },
   {
     .name = "avg",
     .type = FUNCTION_TYPE_AVG,
@@ -549,7 +599,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "percentile",
     .type = FUNCTION_TYPE_PERCENTILE,
-    .classification = FUNC_MGT_AGG_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC,
     .translateFunc = translatePercentile,
     .getEnvFunc   = getPercentileFuncEnv,
     .initFunc     = percentileFunctionSetup,
@@ -562,14 +612,14 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .classification = FUNC_MGT_AGG_FUNC,
     .translateFunc = translateApercentile,
     .getEnvFunc   = getMinmaxFuncEnv,
-    .initFunc     = maxFunctionSetup,
+    .initFunc     = minmaxFunctionSetup,
     .processFunc  = maxFunction,
     .finalizeFunc = functionFinalize
   },
   {
     .name = "top",
     .type = FUNCTION_TYPE_TOP,
-    .classification = FUNC_MGT_AGG_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
     .translateFunc = translateTop,
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = functionSetup,
@@ -579,12 +629,12 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "bottom",
     .type = FUNCTION_TYPE_BOTTOM,
-    .classification = FUNC_MGT_AGG_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
     .translateFunc = translateBottom,
-    .getEnvFunc   = getMinmaxFuncEnv,
+    .getEnvFunc   = getTopBotFuncEnv,
-    .initFunc     = maxFunctionSetup,
+    .initFunc     = functionSetup,
-    .processFunc  = maxFunction,
+    .processFunc  = bottomFunction,
-    .finalizeFunc = functionFinalize
+    .finalizeFunc = topBotFinalize
   },
   {
     .name = "spread",
@@ -603,7 +653,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC,
     .translateFunc = translateLastRow,
     .getEnvFunc   = getMinmaxFuncEnv,
-    .initFunc     = maxFunctionSetup,
+    .initFunc     = minmaxFunctionSetup,
     .processFunc  = maxFunction,
     .finalizeFunc = functionFinalize
   },
@@ -647,6 +697,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = histogramFunction,
     .finalizeFunc = histogramFinalize
   },
+  {
+    .name = "state_count",
+    .type = FUNCTION_TYPE_STATE_COUNT,
+    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC,
+    .translateFunc = translateState,
+    .getEnvFunc   = getStateFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = stateCountFunction,
+    .finalizeFunc = NULL
+  },
   {
     .name = "abs",
     .type = FUNCTION_TYPE_ABS,
@@ -957,6 +1017,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = NULL,
     .finalizeFunc = NULL
   },
+  {
+    .name = "_c0",
+    .type = FUNCTION_TYPE_ROWTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .translateFunc = translateTimePseudoColumn,
+    .getEnvFunc   = getTimePseudoFuncEnv,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
   {
     .name = "tbname",
     .type = FUNCTION_TYPE_TBNAME,
@@ -1032,9 +1102,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .type = FUNCTION_TYPE_SELECT_VALUE,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
     .translateFunc = translateSelectValue,
-    .getEnvFunc   = NULL,
+    .getEnvFunc   = getSelectivityFuncEnv,  // todo remove this function later.
-    .initFunc     = NULL,
+    .initFunc     = functionSetup,
-    .sprocessFunc = NULL,
+    .processFunc  = NULL,
     .finalizeFunc = NULL
   }
 };

source/libs/function/src/functionMgt.c

@@ -169,6 +169,8 @@ bool fmIsDynamicScanOptimizedFunc(int32_t funcId) {
 
 bool fmIsMultiResFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); }
 
+bool fmIsRepeatScanFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_REPEAT_SCAN_FUNC); }
+
 bool fmIsUserDefinedFunc(int32_t funcId) { return funcId > FUNC_UDF_ID_START; }
 
 void fmFuncMgtDestroy() {
@@ -197,15 +199,14 @@ int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet) {
 bool fmIsInvertible(int32_t funcId) {
   bool res = false;
   switch (funcMgtBuiltins[funcId].type) {
-  case FUNCTION_TYPE_COUNT:
+    case FUNCTION_TYPE_COUNT:
-  case FUNCTION_TYPE_SUM:
+    case FUNCTION_TYPE_SUM:
-  case FUNCTION_TYPE_STDDEV:
+    case FUNCTION_TYPE_STDDEV:
-  case FUNCTION_TYPE_AVG:
+    case FUNCTION_TYPE_AVG:
-    res = true;
+      res = true;
-    break;
+      break;
-  default:
+    default:
-    break;
+      break;
   }
   return res;
 }
-

source/libs/function/src/tudf.c

@@ -18,15 +18,175 @@
 #include "tudf.h"
 #include "tudfInt.h"
 #include "tarray.h"
+#include "tglobal.h"
 #include "tdatablock.h"
 #include "querynodes.h"
 #include "builtinsimpl.h"
 #include "functionMgt.h"
 
-//TODO: network error processing.
 //TODO: add unit test
 //TODO: include all global variable under context struct
 
+typedef struct SUdfdData {
+  bool          startCalled;
+  bool          needCleanUp;
+  uv_loop_t     loop;
+  uv_thread_t   thread;
+  uv_barrier_t  barrier;
+  uv_process_t  process;
+  int           spawnErr;
+  uv_pipe_t     ctrlPipe;
+  uv_async_t    stopAsync;
+  int32_t        stopCalled;
+
+  int32_t         dnodeId;
+} SUdfdData;
+
+SUdfdData udfdGlobal = {0};
+
+static int32_t udfSpawnUdfd(SUdfdData *pData);
+
+void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
+  fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
+  SUdfdData *pData = process->data;
+  if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pData->stopCalled)) {
+    fnInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
+  } else {
+    fnInfo("udfd process restart");
+    udfSpawnUdfd(pData);
+  }
+}
+
+static int32_t udfSpawnUdfd(SUdfdData* pData) {
+  fnInfo("dnode start spawning udfd");
+  uv_process_options_t options = {0};
+
+  char path[PATH_MAX] = {0};
+  if (tsProcPath == NULL) {
+    path[0] = '.';
+  } else {
+    strncpy(path, tsProcPath, strlen(tsProcPath));
+    taosDirName(path);
+  }
+#ifdef WINDOWS
+  strcat(path, "udfd.exe");
+#else
+  strcat(path, "/udfd");
+#endif
+  char* argsUdfd[] = {path, "-c", configDir, NULL};
+  options.args = argsUdfd;
+  options.file = path;
+
+  options.exit_cb = udfUdfdExit;
+
+  uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
+
+  uv_stdio_container_t child_stdio[3];
+  child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
+  child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
+  child_stdio[1].flags = UV_IGNORE;
+  child_stdio[2].flags = UV_INHERIT_FD;
+  child_stdio[2].data.fd = 2;
+  options.stdio_count = 3;
+  options.stdio = child_stdio;
+
+  options.flags = UV_PROCESS_DETACHED;
+
+  char dnodeIdEnvItem[32] = {0};
+  char thrdPoolSizeEnvItem[32] = {0};
+  snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId);
+  float numCpuCores = 4;
+  taosGetCpuCores(&numCpuCores);
+  snprintf(thrdPoolSizeEnvItem,32,  "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
+  char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
+  options.env = envUdfd;
+
+  int err = uv_spawn(&pData->loop, &pData->process, &options);
+  pData->process.data = (void*)pData;
+
+  if (err != 0) {
+    fnError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
+  }
+  return err;
+}
+
+static void udfUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
+  if (!uv_is_closing(handle)) {
+    uv_close(handle, NULL);
+  }
+}
+
+static void udfUdfdStopAsyncCb(uv_async_t *async) {
+  SUdfdData *pData = async->data;
+  uv_stop(&pData->loop);
+}
+
+static void udfWatchUdfd(void *args) {
+  SUdfdData *pData = args;
+  uv_loop_init(&pData->loop);
+  uv_async_init(&pData->loop, &pData->stopAsync, udfUdfdStopAsyncCb);
+  pData->stopAsync.data = pData;
+  int32_t err = udfSpawnUdfd(pData);
+  atomic_store_32(&pData->spawnErr, err);
+  uv_barrier_wait(&pData->barrier);
+  uv_run(&pData->loop, UV_RUN_DEFAULT);
+  uv_loop_close(&pData->loop);
+
+  uv_walk(&pData->loop, udfUdfdCloseWalkCb, NULL);
+  uv_run(&pData->loop, UV_RUN_DEFAULT);
+  uv_loop_close(&pData->loop);
+  return;
+}
+
+int32_t udfStartUdfd(int32_t startDnodeId) {
+  if (!tsStartUdfd) {
+    fnInfo("start udfd is disabled.")
+    return 0;
+  }
+  SUdfdData *pData = &udfdGlobal;
+  if (pData->startCalled) {
+    fnInfo("dnode-mgmt start udfd already called");
+    return 0;
+  }
+  pData->startCalled = true;
+  char dnodeId[8] = {0};
+  snprintf(dnodeId, sizeof(dnodeId), "%d", startDnodeId);
+  uv_os_setenv("DNODE_ID", dnodeId);
+  pData->dnodeId = startDnodeId;
+
+  uv_barrier_init(&pData->barrier, 2);
+  uv_thread_create(&pData->thread, udfWatchUdfd, pData);
+  uv_barrier_wait(&pData->barrier);
+  int32_t err = atomic_load_32(&pData->spawnErr);
+  if (err != 0) {
+    uv_barrier_destroy(&pData->barrier);
+    uv_async_send(&pData->stopAsync);
+    uv_thread_join(&pData->thread);
+    pData->needCleanUp = false;
+    fnInfo("dnode-mgmt udfd cleaned up after spawn err");
+  } else {
+    pData->needCleanUp = true;
+  }
+  return err;
+}
+
+int32_t udfStopUdfd() {
+  SUdfdData *pData = &udfdGlobal;
+  fnInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
+        pData->needCleanUp, pData->spawnErr);
+  if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
+    return 0;
+  }
+  atomic_store_32(&pData->stopCalled, 1);
+  pData->needCleanUp = false;
+  uv_barrier_destroy(&pData->barrier);
+  uv_async_send(&pData->stopAsync);
+  uv_thread_join(&pData->thread);
+  fnInfo("dnode-mgmt udfd cleaned up");
+  return 0;
+}
+
+//==============================================================================================
 /* Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
  * The QUEUE is copied from queue.h under libuv
  * */
@@ -1259,7 +1419,9 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
     return false;
   }
   UdfcFuncHandle handle;
-  if (setupUdf((char*)pCtx->udfName, &handle) != 0) {
+  int32_t udfCode = 0;
+  if ((udfCode = setupUdf((char*)pCtx->udfName, &handle)) != 0) {
+    fnError("udfAggInit error. step setupUdf. udf code: %d", udfCode);
     return false;
   }
   SClientUdfUvSession *session = (SClientUdfUvSession *)handle;
@@ -1272,7 +1434,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
 
   udfRes->session = (SClientUdfUvSession *)handle;
   SUdfInterBuf buf = {0};
-  if (callUdfAggInit(handle, &buf) != 0) {
+  if ((udfCode = callUdfAggInit(handle, &buf)) != 0) {
+    fnError("udfAggInit error. step callUdfAggInit. udf code: %d", udfCode);
     return false;
   }
   udfRes->interResNum = buf.numOfResult;
@@ -1316,21 +1479,23 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
                         .numOfResult = udfRes->interResNum};
   SUdfInterBuf newState = {0};
 
-  callUdfAggProcess(session, inputBlock, &state, &newState);
+  int32_t udfCode = callUdfAggProcess(session, inputBlock, &state, &newState);
-
+  if (udfCode != 0) {
-  udfRes->interResNum = newState.numOfResult;
+    fnError("udfAggProcess error. code: %d", udfCode);
-  memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
+    newState.numOfResult = 0;
-
+  } else {
+    udfRes->interResNum = newState.numOfResult;
+    memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
+  }
   if (newState.numOfResult == 1 || state.numOfResult == 1) {
     GET_RES_INFO(pCtx)->numOfRes = 1;
   }
 
   blockDataDestroy(inputBlock);
-
   taosArrayDestroy(tempBlock.pDataBlock);
 
   taosMemoryFree(newState.buf);
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
@@ -1344,15 +1509,22 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
   SUdfInterBuf state = {.buf = udfRes->interResBuf,
                         .bufLen = session->bufSize,
                         .numOfResult = udfRes->interResNum};
-  callUdfAggFinalize(session, &state, &resultBuf);
+  int32_t udfCallCode= 0;
-
+  udfCallCode= callUdfAggFinalize(session, &state, &resultBuf);
-  udfRes->finalResBuf = resultBuf.buf;
+  if (udfCallCode!= 0) {
-  udfRes->finalResNum = resultBuf.numOfResult;
+    fnError("udfAggFinalize error. callUdfAggFinalize step. udf code:%d", udfCallCode);
-
+    GET_RES_INFO(pCtx)->numOfRes = 0;
-  teardownUdf(session);
+  } else {
-
+    memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
-  if (resultBuf.numOfResult == 1) {
+    udfRes->finalResNum = resultBuf.numOfResult;
-    GET_RES_INFO(pCtx)->numOfRes = 1;
+    GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
   }
+
+  int32_t code = teardownUdf(session);
+  if (code != 0) {
+    fnError("udfAggFinalize error. teardownUdf step. udf code: %d", code);
+  }
+
   return functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
+
 }

source/libs/function/src/udfd.c

@@ -409,7 +409,7 @@ void udfdPipeCloseCb(uv_handle_t *pipe) {
 void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); }
 
 void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
-  fnDebug("udf read %zu bytes from client", nread);
+  fnDebug("udf read %zd bytes from client", nread);
   if (nread == 0) return;
 
   SUdfdUvConn *conn = client->data;

source/libs/index/inc/indexComm.h

@@ -20,11 +20,23 @@
 extern "C" {
 #endif
 
+#include "indexInt.h"
+#include "tcompare.h"
+
 extern char JSON_COLUMN[];
 extern char JSON_VALUE_DELIM;
 
 char* indexPackJsonData(SIndexTerm* itm);
 char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
+
+typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
+
+typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
+
+TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b);
+
+_cache_range_compare indexGetCompare(RangeType ty);
+
 #ifdef __cplusplus
 }
 #endif