Merge branch '3.0' into feature/TD-14761

cmake/cmake.platform

@@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "")
   IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
     MESSAGE(STATUS "The current platform is amd64")
     SET(PLATFORM_ARCH_STR "amd64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
     MESSAGE(STATUS "The current platform is x86")
     SET(PLATFORM_ARCH_STR "i386")
+    SET(TD_INTEL_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
     MESSAGE(STATUS "The current platform is aarch32")
     SET(PLATFORM_ARCH_STR "arm")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
     MESSAGE(STATUS "The current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
+    SET(TD_ARM_64 TRUE)
   ENDIF ()
 ELSE ()
   # if generate ARM version:
@@ -92,18 +96,23 @@ ELSE ()
   IF (${CPUTYPE} MATCHES "aarch32")
     SET(PLATFORM_ARCH_STR "arm")
     MESSAGE(STATUS "input cpuType: aarch32")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     MESSAGE(STATUS "input cpuType: aarch64")
+    SET(TD_ARM_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "mips64")
     SET(PLATFORM_ARCH_STR "mips")
     MESSAGE(STATUS "input cpuType: mips64")
+    SET(TD_MIPS_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x64")
     SET(PLATFORM_ARCH_STR "amd64")
     MESSAGE(STATUS "input cpuType: x64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x86")
     SET(PLATFORM_ARCH_STR "i386")
     MESSAGE(STATUS "input cpuType: x86")
+    SET(TD_INTEL_32 TRUE)
   ELSE ()
     MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
   ENDIF ()

docs/zh/12-taos-sql/12-interval.md

@@ -14,6 +14,7 @@ TDengine提供的特色查询包括标签切分查询和窗口切分查询。
 ```sql
 PARTITION BY tag_list
 ```
+
 其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。
 
 TDengine 按如下方式处理标签切分子句：
@@ -42,11 +43,13 @@ SELECT function_list FROM tb_name
 在上述语法中的具体限制如下
 
 ### 窗口切分查询中使用函数的限制
+
 - 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。
 - 此外 LAST_ROW 查询也不能与窗口聚合同时出现。
 - 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。
 
 ### 窗口子句的规则
+
 - 窗口子句位于标签切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。
 - 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含：
   - 常量。
@@ -56,7 +59,9 @@ SELECT function_list FROM tb_name
 - WHERE 语句可以指定查询的起止时间和其他过滤条件。
 
 ### FILL 子句
+
 FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种：
+
 1. 不进行填充：NONE（默认填充模式）。
 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。
 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。
@@ -93,6 +98,7 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m);
 ```
 
 使用时间窗口需要注意：
+
 - 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。
 - 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。
 - 返回的结果中时间序列严格单调递增。

docs/zh/12-taos-sql/22-information.md

@@ -0,0 +1,5 @@
+---
+sidebar_label: Information内置数据库
+title: Information内置数据库
+---
+

include/common/tcommon.h

@@ -80,8 +80,6 @@ typedef struct {
   SArray*   pTableList;
   SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
   bool      needSortTableByGroupId;
-  void*     pTagCond;
-  void*     pTagIndexCond;
   uint64_t  suid;
 } STableListInfo;
 

include/common/tmsg.h

@@ -2074,6 +2074,7 @@ typedef struct {
   char*   tbName;
   int8_t  action;
   char*   colName;
+  int32_t colId;
   // TSDB_ALTER_TABLE_ADD_COLUMN
   int8_t  type;
   int8_t  flags;

include/common/ttime.h

@@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
                       : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
                                                                  : 1000000000;
   time_t     t = taosTime(NULL);
-  struct tm* tm = taosLocalTime(&t, NULL);
+  struct tm  tm;
-  tm->tm_hour = 0;
+  taosLocalTime(&t, &tm);
-  tm->tm_min = 0;
+  tm.tm_hour = 0;
-  tm->tm_sec = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
 
-  return (int64_t)taosMktime(tm) * factor;
+  return (int64_t)taosMktime(&tm) * factor;
 }
 
 int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);

include/libs/nodes/plannodes.h

@@ -26,6 +26,12 @@ extern "C" {
 
 #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
 
+typedef enum EDataOrderLevel {
+  DATA_ORDER_LEVEL_NONE = 1,
+  DATA_ORDER_LEVEL_IN_BLOCK,
+  DATA_ORDER_LEVEL_IN_GROUP
+} EDataOrderLevel;
+
 typedef struct SLogicNode {
   ENodeType          type;
   SNodeList*         pTargets;  // SColumnNode
@@ -36,6 +42,8 @@ typedef struct SLogicNode {
   uint8_t            precision;
   SNode*             pLimit;
   SNode*             pSlimit;
+  EDataOrderLevel    requireDataOrder;  // requirements for input data
+  EDataOrderLevel    resultDataOrder;   // properties of the output data
 } SLogicNode;
 
 typedef enum EScanType {
@@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
+  bool       mergeDataBlock;
 } SProjectPhysiNode;
 
 typedef struct SIndefRowsFuncPhysiNode {

include/libs/scalar/scalar.h

@@ -117,6 +117,12 @@ int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
 int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }

packaging/deb/makedeb.sh

@@ -73,7 +73,7 @@ cp ${compile_dir}/../include/client/taos.h          ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../include/common/taosdef.h       ${pkg_dir}${install_home_path}/include
 cp ${compile_dir}/../include/util/taoserror.h       ${pkg_dir}${install_home_path}/include
 cp ${compile_dir}/../include/libs/function/taosudf.h       ${pkg_dir}${install_home_path}/include
-cp ${compile_dir}/../src/inc/taosws.h               ${pkg_dir}${install_home_path}/include ||:
+cp ${compile_dir}/build/include/taosws.h            ${pkg_dir}${install_home_path}/include ||:
 cp -r ${top_dir}/examples/*                         ${pkg_dir}${install_home_path}/examples
 #cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
 #cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector

packaging/rpm/tdengine.spec

@@ -80,7 +80,7 @@ cp %{_compiledir}/../include/client/taos.h          %{buildroot}%{homepath}/incl
 cp %{_compiledir}/../include/common/taosdef.h       %{buildroot}%{homepath}/include
 cp %{_compiledir}/../include/util/taoserror.h       %{buildroot}%{homepath}/include
 cp %{_compiledir}/../include/libs/function/taosudf.h       %{buildroot}%{homepath}/include
-cp %{_compiledir}/../src/inc/taosws.h               %{buildroot}%{homepath}/include ||:
+cp %{_compiledir}/build/include/taosws.h            %{buildroot}%{homepath}/include ||:
 #cp -r %{_compiledir}/../src/connector/python        %{buildroot}%{homepath}/connector
 #cp -r %{_compiledir}/../src/connector/go            %{buildroot}%{homepath}/connector
 #cp -r %{_compiledir}/../src/connector/nodejs        %{buildroot}%{homepath}/connector

packaging/tools/install.sh

@@ -229,13 +229,13 @@ function install_lib() {
   ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
   ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
-  ${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+  [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
 
   if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
     ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
     ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
 
-    ${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+    ${csudo}ln -s ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
   fi
 
   ${csudo}ldconfig

packaging/tools/install_client.sh

@@ -116,6 +116,7 @@ function install_bin() {
 
 function clean_lib() {
     sudo rm -f /usr/lib/libtaos.* || :
+    sudo rm -f /usr/lib/libtaosws.* || :
     sudo rm -rf ${lib_dir} || :
 }
 
@@ -123,6 +124,9 @@ function install_lib() {
     # Remove links
     ${csudo}rm -f ${lib_link_dir}/libtaos.*         || :
     ${csudo}rm -f ${lib64_link_dir}/libtaos.*       || :
+
+    ${csudo}rm -f ${lib_link_dir}/libtaosws.*         || :
+    ${csudo}rm -f ${lib64_link_dir}/libtaosws.*       || :
     #${csudo}rm -rf ${v15_java_app_dir}              || :
 
     ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
@@ -131,13 +135,19 @@ function install_lib() {
         ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
         ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
+        [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so
+
         if [ -d "${lib64_link_dir}" ]; then
             ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1       || :
             ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so               || :
+
+            [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so       || :
         fi
     else
         ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib
         ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib
+
+        [ -f ${install_main_dir}/driver/libtaosws.dylib ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib
     fi
 
     if [ "$osType" != "Darwin" ]; then
@@ -154,6 +164,8 @@ function install_header() {
     ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
     ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
     ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h    
+
+    [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taos.h
 }
 
 function install_jemalloc() {

packaging/tools/makeclient.sh

@@ -57,12 +57,16 @@ if [ "$osType" != "Darwin" ]; then
         ${script_dir}/get_client.sh"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
+  wslib_files="${build_dir}/lib/libtaosws.so"
 else
   bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh"
   lib_files="${build_dir}/lib/libtaos.${version}.dylib"
+  wslib_files="${build_dir}/lib/libtaosws.dylib"
 fi
 
 header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
+wsheader_files="${build_dir}/include/taosws.h"
+
 if [ "$dbName" != "taos" ]; then
   cfg_dir="${top_dir}/../enterprise/packaging/cfg"
 else
@@ -74,6 +78,8 @@ install_files="${script_dir}/install_client.sh"
 # make directories.
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
+[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc
+
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
 mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/*
 
@@ -187,6 +193,7 @@ cp ${lib_files} ${install_dir}/driver
 # Copy connector
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
+[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver
 
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then

packaging/tools/makepkg.sh

@@ -96,7 +96,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}"
 wslib_files="${build_dir}/lib/libtaosws.so."
 header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
 
-wsheader_files="${code_dir}/inc/taosws.h"
+wsheader_files="${build_dir}/include/taosws.h"
 
 if [ "$dbName" != "taos" ]; then
   cfg_dir="${top_dir}/../enterprise/packaging/cfg"
@@ -114,7 +114,7 @@ init_file_rpm=${script_dir}/../rpm/taosd
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 
-${wsheader_files} ${install_dir}/inc || :
+[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc || :
 
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
 

packaging/tools/post.sh

@@ -82,22 +82,33 @@ function kill_taosd() {
 
 function install_include() {
     ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
+    ${csudo}rm -f ${inc_link_dir}/taosws.h
+
     ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h
     ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h
     ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h
     ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h    
+
+    [ -f ${inc_dir}/taosws.h ] && ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ||:
 }
 
 function install_lib() {
     ${csudo}rm -f ${lib_link_dir}/libtaos* || :
     ${csudo}rm -f ${lib64_link_dir}/libtaos* || :
 
+    ${csudo}rm -f ${lib_link_dir}/libtaosws* || :
+    ${csudo}rm -f ${lib64_link_dir}/libtaosws* || :
+
     ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
     ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
+    [ -f ${lib_dir}/libtaosws.so ]${csudo}ln -s ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so
+
     if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
       ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1           || :
       ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so   || :
+
+      [ -ff ${lib_dir}/libtaosws.so ] && ${csudo}ln -s ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so           || :
     fi
 
     ${csudo}ldconfig

source/client/src/tmq.c

@@ -357,7 +357,11 @@ tmq_list_t* tmq_list_new() {
 
 int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
+  if (src == NULL || src[0] == 0) return -1;
   char* topic = strdup(src);
+  if (topic[0] != '`') {
+    strtolower(topic, src);
+  }
   if (taosArrayPush(container, &topic) == NULL) return -1;
   return 0;
 }

source/common/src/tdatablock.c

@@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
   uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     // Handle the bitmap
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
       if (p == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
     memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
     pColumnInfoData->varmeta.length = len + oldLen;
   } else {
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
       if (tmp == NULL) {
@@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
     }
     */
 
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
       ms += 1000;
     }
   }
-
+  struct tm ptm = {0};
-  struct tm* ptm = taosLocalTime(&tt, NULL);
+  taosLocalTime(&tt, &ptm);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);

source/common/src/ttime.c

@@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) {
 
 void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
   char       ts[40] = {0};
-  struct tm* ptm;
+  struct tm  ptm;
 
   int32_t fractionLen;
   char*   format = NULL;
@@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
       assert(false);
   }
 
-  ptm = taosLocalTime(&quot, NULL);
+  taosLocalTime(&quot, &ptm);
-  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
+  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
   length += snprintf(ts + length, fractionLen, format, mod);
-  length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
+  length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
 
   tstrncpy(buf, ts, bufLen);
 }

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -49,9 +49,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
         dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
       }
     }
-    rpcFreeCont(pRsp->pCont);
     tFreeSStatusRsp(&statusRsp);
   }
+  rpcFreeCont(pRsp->pCont);
 }
 
 void dmSendStatusReq(SDnodeMgmt *pMgmt) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       } else {
         dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
         taosWriteQitem(pVnode->pWriteQ, pMsg);
+#if 0  // tests for batch writes
+        if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
+          SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
+          memcpy(pDup, pMsg, sizeof(SRpcMsg));
+          pDup->pCont = rpcMallocCont(pMsg->contLen);
+          memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
+          pDup->info.handle = NULL;
+          taosWriteQitem(pVnode->pWriteQ, pDup);
+        }
+#endif
       }
       break;
     case SYNC_QUEUE:

source/dnode/mnode/impl/inc/mndDef.h

@@ -35,6 +35,46 @@
 extern "C" {
 #endif
 
+typedef enum {
+  MND_OPER_CONNECT = 1,
+  MND_OPER_CREATE_ACCT,
+  MND_OPER_DROP_ACCT,
+  MND_OPER_ALTER_ACCT,
+  MND_OPER_CREATE_USER,
+  MND_OPER_DROP_USER,
+  MND_OPER_ALTER_USER,
+  MND_OPER_CREATE_BNODE,
+  MND_OPER_DROP_BNODE,
+  MND_OPER_CREATE_DNODE,
+  MND_OPER_DROP_DNODE,
+  MND_OPER_CONFIG_DNODE,
+  MND_OPER_CREATE_MNODE,
+  MND_OPER_DROP_MNODE,
+  MND_OPER_CREATE_QNODE,
+  MND_OPER_DROP_QNODE,
+  MND_OPER_CREATE_SNODE,
+  MND_OPER_DROP_SNODE,
+  MND_OPER_REDISTRIBUTE_VGROUP,
+  MND_OPER_MERGE_VGROUP,
+  MND_OPER_SPLIT_VGROUP,
+  MND_OPER_BALANCE_VGROUP,
+  MND_OPER_CREATE_FUNC,
+  MND_OPER_DROP_FUNC,
+  MND_OPER_KILL_TRANS,
+  MND_OPER_KILL_CONN,
+  MND_OPER_KILL_QUERY,
+  MND_OPER_CREATE_DB,
+  MND_OPER_ALTER_DB,
+  MND_OPER_DROP_DB,
+  MND_OPER_COMPACT_DB,
+  MND_OPER_TRIM_DB,
+  MND_OPER_USE_DB,
+  MND_OPER_WRITE_DB,
+  MND_OPER_READ_DB,
+  MND_OPER_READ_OR_WRITE_DB,
+  MND_OPER_SHOW_VARIBALES,
+} EOperType;
+
 typedef enum {
   MND_AUTH_ACCT_START = 0,
   MND_AUTH_ACCT_USER,
@@ -109,9 +149,9 @@ typedef struct {
   ETrnPolicy     policy;
   ETrnConflct    conflict;
   ETrnExec       exec;
+  EOperType      oper;
   int32_t        code;
   int32_t        failedTimes;
-  SRpcHandleInfo rpcInfo;
   void*          rpcRsp;
   int32_t        rpcRspLen;
   int32_t        redoActionPos;
@@ -130,6 +170,7 @@ typedef struct {
   int32_t        stopFunc;
   int32_t        paramLen;
   void*          param;
+  SArray*        pRpcArray;
 } STrans;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndPrivilege.h

@@ -22,46 +22,6 @@
 extern "C" {
 #endif
 
-typedef enum {
-  MND_OPER_CONNECT = 1,
-  MND_OPER_CREATE_ACCT,
-  MND_OPER_DROP_ACCT,
-  MND_OPER_ALTER_ACCT,
-  MND_OPER_CREATE_USER,
-  MND_OPER_DROP_USER,
-  MND_OPER_ALTER_USER,
-  MND_OPER_CREATE_BNODE,
-  MND_OPER_DROP_BNODE,
-  MND_OPER_CREATE_DNODE,
-  MND_OPER_DROP_DNODE,
-  MND_OPER_CONFIG_DNODE,
-  MND_OPER_CREATE_MNODE,
-  MND_OPER_DROP_MNODE,
-  MND_OPER_CREATE_QNODE,
-  MND_OPER_DROP_QNODE,
-  MND_OPER_CREATE_SNODE,
-  MND_OPER_DROP_SNODE,
-  MND_OPER_REDISTRIBUTE_VGROUP,
-  MND_OPER_MERGE_VGROUP,
-  MND_OPER_SPLIT_VGROUP,
-  MND_OPER_BALANCE_VGROUP,
-  MND_OPER_CREATE_FUNC,
-  MND_OPER_DROP_FUNC,
-  MND_OPER_KILL_TRANS,
-  MND_OPER_KILL_CONN,
-  MND_OPER_KILL_QUERY,
-  MND_OPER_CREATE_DB,
-  MND_OPER_ALTER_DB,
-  MND_OPER_DROP_DB,
-  MND_OPER_COMPACT_DB,
-  MND_OPER_TRIM_DB,
-  MND_OPER_USE_DB,
-  MND_OPER_WRITE_DB,
-  MND_OPER_READ_DB,
-  MND_OPER_READ_OR_WRITE_DB,
-  MND_OPER_SHOW_VARIBALES,
-} EOperType;
-
 int32_t mndInitPrivilege(SMnode *pMnode);
 void    mndCleanupPrivilege(SMnode *pMnode);
 

source/dnode/mnode/impl/inc/mndTrans.h

@@ -73,12 +73,14 @@ void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
 void    mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
 void    mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2);
 void    mndTransSetSerial(STrans *pTrans);
+void    mndTransSetOper(STrans *pTrans, EOperType oper);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
 int32_t mndTransProcessRsp(SRpcMsg *pRsp);
 void    mndTransPullup(SMnode *pMnode);
 int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
 void    mndTransExecute(SMnode *pMnode, STrans *pTrans);
+int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -487,6 +487,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
   mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
 
   mndTransSetDbName(pTrans, dbObj.name, NULL);
+  mndTransSetOper(pTrans, MND_OPER_CREATE_DB);
   if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
@@ -534,6 +535,14 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
       terrno = TSDB_CODE_MND_DB_ALREADY_EXIST;
       goto _OVER;
     }
+  } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) {
+    if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
+      mDebug("db:%s, is creating and response after trans finished", createReq.db);
+      code = TSDB_CODE_ACTION_IN_PROGRESS;
+      goto _OVER;
+    } else {
+      goto _OVER;
+    }
   } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) {
     goto _OVER;
   }

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -298,7 +298,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                 .pVgEp = pVgEp,
             };
             taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
+                  pConsumerEp->consumerId);
           }
           imbCnt++;
         }
@@ -312,7 +313,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
               .pVgEp = pVgEp,
           };
           taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
+                pConsumerEp->consumerId);
         }
       }
     }
@@ -354,7 +356,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+            pConsumerEp->consumerId);
     }
   }
 
@@ -371,8 +374,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       ASSERT(pConsumerEp->consumerId > 0);
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
+      if (pRebVg->newConsumerId == pRebVg->oldConsumerId) {
+        mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+              pConsumerEp->consumerId);
+        continue;
+      }
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+            pConsumerEp->consumerId);
     }
   } else {
     // if all consumer is removed, put all vg into unassigned

source/dnode/mnode/impl/src/mndTrans.c

@@ -122,6 +122,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
   SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@@ -269,15 +273,22 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   int8_t policy = 0;
   int8_t conflict = 0;
   int8_t exec = 0;
+  int8_t oper = 0;
+  int8_t reserved = 0;
   int8_t actionType = 0;
   SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &policy, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &exec, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &oper, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
   pTrans->stage = stage;
   pTrans->policy = policy;
   pTrans->conflict = conflict;
   pTrans->exec = exec;
+  pTrans->oper = oper;
   SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@@ -495,6 +506,10 @@ static void mndTransDropData(STrans *pTrans) {
     mndTransDropActions(pTrans->commitActions);
     pTrans->commitActions = NULL;
   }
+  if (pTrans->pRpcArray != NULL) {
+    taosArrayDestroy(pTrans->pRpcArray);
+    pTrans->pRpcArray = NULL;
+  }
   if (pTrans->rpcRsp != NULL) {
     taosMemoryFree(pTrans->rpcRsp);
     pTrans->rpcRsp = NULL;
@@ -585,14 +600,18 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
   pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
+  pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
 
-  if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL) {
+  if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL ||
+      pTrans->pRpcArray == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("failed to create transaction since %s", terrstr());
     return NULL;
   }
 
-  if (pReq != NULL) pTrans->rpcInfo = pReq->info;
+  if (pReq != NULL) {
+    taosArrayPush(pTrans->pRpcArray, &pReq->info);
+  }
   mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
@@ -677,6 +696,31 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *
   pTrans->paramLen = paramLen;
 }
 
+int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname) {
+  STrans *pTrans = NULL;
+  void   *pIter = NULL;
+  int32_t code = -1;
+
+  while (1) {
+    pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
+    if (pIter == NULL) break;
+
+    if (pTrans->oper == oper) {
+      if (strcasecmp(dbname, pTrans->dbname1) == 0) {
+        mDebug("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper);
+        if (taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) {
+          code = 0;
+        }
+        sdbRelease(pMnode->pSdb, pTrans);
+        break;
+      }
+    }
+
+    sdbRelease(pMnode->pSdb, pTrans);
+  }
+  return code;
+}
+
 void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) {
   if (dbname1 != NULL) {
     memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN);
@@ -688,6 +732,8 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2)
 
 void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
 
+void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; }
+
 static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
   SSdbRaw *pRaw = mndTransActionEncode(pTrans);
   if (pRaw == NULL) {
@@ -711,7 +757,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
 
 static bool mndCheckDbConflict(const char *db, STrans *pTrans) {
   if (db[0] == 0) return false;
-  if (strcmp(db, pTrans->dbname1) == 0 || strcmp(db, pTrans->dbname2) == 0) return true;
+  if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true;
   return false;
 }
 
@@ -784,9 +830,10 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
     return -1;
   }
 
-  pNew->rpcInfo = pTrans->rpcInfo;
+  pNew->pRpcArray = pTrans->pRpcArray;
   pNew->rpcRsp = pTrans->rpcRsp;
   pNew->rpcRspLen = pTrans->rpcRspLen;
+  pTrans->pRpcArray = NULL;
   pTrans->rpcRsp = NULL;
   pTrans->rpcRspLen = 0;
 
@@ -835,13 +882,20 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
     }
   }
 
-  if (sendRsp && pTrans->rpcInfo.handle != NULL) {
+  if (!sendRsp) return;
+
+  int32_t size = taosArrayGetSize(pTrans->pRpcArray);
+  if (size <= 0) return;
+
+  for (int32_t i = 0; i < size; ++i) {
+    SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i);
+    if (pInfo->handle != NULL) {
       mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
-           pTrans->rpcInfo.ahandle);
+             pInfo->ahandle);
       if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
         code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
       }
-    SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo};
+      SRpcMsg rspMsg = {.code = code, .info = *pInfo};
 
       if (pTrans->rpcRspLen != 0) {
         void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
@@ -850,15 +904,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
           rspMsg.pCont = rpcCont;
           rspMsg.contLen = pTrans->rpcRspLen;
         }
-      taosMemoryFree(pTrans->rpcRsp);
       }
 
       tmsgSendRsp(&rspMsg);
-    pTrans->rpcInfo.handle = NULL;
-    pTrans->rpcRsp = NULL;
-    pTrans->rpcRspLen = 0;
     }
   }
+  taosArrayClear(pTrans->pRpcArray);
+}
 
 int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
   SMnode *pMnode = pRsp->info.node;

source/dnode/vnode/src/inc/sma.h

@@ -105,8 +105,13 @@ struct SRSmaInfoItem {
 struct SRSmaInfo {
   STSchema *pTSchema;
   int64_t   suid;
+  int8_t    delFlag;
+  T_REF_DECLARE()
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
+#define RSMA_INFO_HEAD_LEN   24
+#define RSMA_INFO_IS_DEL(r)  ((r)->delFlag == 1)
+#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1)
 
 enum {
   TASK_TRIGGER_STAT_INIT = 0,
@@ -120,8 +125,8 @@ enum {
 enum {
   RSMA_ROLE_CREATE = 0,
   RSMA_ROLE_DROP = 1,
-  RSMA_ROLE_FETCH = 2,
+  RSMA_ROLE_SUBMIT = 2,
-  RSMA_ROLE_SUBMIT = 3,
+  RSMA_ROLE_FETCH = 3,
   RSMA_ROLE_ITERATE = 4,
 };
 
@@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
 
 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
 
 void   *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
 int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
@@ -193,6 +200,7 @@ void           tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
+void           tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
@@ -258,7 +266,8 @@ void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
 void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
+                      char *outputName);
 void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName);
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/tsdb.h

@@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
 typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
 void    tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
 bool    tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
+bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
 int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
 int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);

source/dnode/vnode/src/sma/smaCommit.c

@@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
   if ((pDir = taosOpenDir(dir)) == NULL) {
     regfree(&regex);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
+    smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
     return TSDB_CODE_FAILED;
   }
 
@@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
   // step 2: cleanup outdated qtaskinfo files
   tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
 
+  atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
+
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaEnv.c

@@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
   return 0;
 }
 
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_INC(pRSmaInfo);
+  smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+  return 0;
+}
+
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_DEC(pRSmaInfo);
+  smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+
+  if (ref == 0) {
+    tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid);
+  }
+  return 0;
+}
+
 static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
   ASSERT(pSmaStat != NULL);
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = {
 };
 
 #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
+#define TD_RSMAINFO_DEL_FILE      "rsmainfo.del"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
 static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
-
-
 static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
   // adapt accordingly if definition of SRSmaInfo update
   SRSmaInfo *pResult = NULL;
-  int32_t    rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
   ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2);
-  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen));
+  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN));
   ASSERT(pResult->pTSchema->numOfCols > 1);
   return pResult;
 }
@@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
         if (isDeepFree && pItem->tmrId) {
-          smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
+          smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid,
-                   i + 1);
+                   pItem->tmrId, i + 1);
           taosTmrStopA(&pItem->tmrId);
         }
         tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1);
@@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
   }
   pRSmaInfo->pTSchema = pTSchema;
   pRSmaInfo->suid = suid;
+  T_REF_INIT_VAL(pRSmaInfo, 1);
 
   if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
     goto _err;
@@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) {
 int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) {
   SVnode *pVnode = pSma->pVnode;
   if (!VND_IS_RSMA(pVnode)) {
-    smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
+    smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
              pReq->suid);
     return TSDB_CODE_SUCCESS;
   }
 
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid);
+
+  if (!pRSmaInfo) {
+    smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name,
+            pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set del flag for data in mem
+  RSMA_INFO_SET_DEL(pRSmaInfo);
+  tdUnRefRSmaInfo(pSma, pRSmaInfo);
+
+  // save to file
+
   smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid);
   return TSDB_CODE_SUCCESS;
 }
@@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!pEnv) {
-    // only applicable when rsma env exists
     return NULL;
   }
 
@@ -683,6 +703,7 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pCowRSmaInfo = NULL;
   // lock
   taosWLockLatch(SMA_ENV_LOCK(pEnv));
+  if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) {  // 2-phase lock
     void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
     if (iRSmaInfo) {
       SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
@@ -692,32 +713,69 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
           smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
           return NULL;
         }
+        smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid);
         if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
           taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
           return NULL;
         }
       }
     }
+  }
   // unlock
   taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
   return pCowRSmaInfo;
 }
 
+/**
+ * @brief During the drop procedure, only need to delete the object in rsmaInfoHash.
+ *
+ * @param pSma
+ * @param suid
+ * @return SRSmaInfo*
+ */
+void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  if (!pEnv) {
+    return;
+  }
+
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return;
+  }
+
+  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  if (pRSmaInfo) {
+    if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+      tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+    }
+    taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+    smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid);
+  }
+}
+
 static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
   if (!pRSmaInfo) {
-    smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (!pRSmaInfo->items[0].taskInfo) {
-    smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
+    tdRefRSmaInfo(pSma, pRSmaInfo);
+
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
+
+    tdUnRefRSmaInfo(pSma, pRSmaInfo);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1271,6 +1329,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   }
 
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+  if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+    goto _end;
+  }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
@@ -1279,13 +1340,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
                pItem->level, pRSmaInfo->suid);
 
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+      // sync procedure => async process
+      tdRefRSmaInfo(pSma, pRSmaInfo);
 
       SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
       qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
       tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
 
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+      tdUnRefRSmaInfo(pSma, pRSmaInfo);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",

source/dnode/vnode/src/tq/tqExec.c

@@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
       pRsp->rspOffset = *pOffset;
       return 0;
     } else {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       if (qStreamPrepareScan(task, pOffset) < 0) {
         pRsp->rspOffset = *pOffset;
         return 0;
@@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
     }
 
     if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       qStreamPrepareScan(task, pOffset);
       continue;
     }

source/dnode/vnode/src/tq/tqPush.c

@@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
     void* data = taosMemoryMalloc(msgLen);
     if (data == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      qError("failed to copy data for stream since out of memory");
+      tqError("failed to copy data for stream since out of memory");
       return -1;
     }
     memcpy(data, msg, msgLen);

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p
   char    fname[TSDB_FILENAME_LEN];
 
   if (pFrom && pTo) {
-    if (pFrom != pTo) {
+    if (!tsdbDelFileIsSame(pFrom, pTo)) {
       tsdbDelFileName(pFS->pTsdb, pFrom, fname);
       if (taosRemoveFile(fname) < 0) {
         code = TAOS_SYSTEM_ERROR(errno);

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty
   }
 }
 
+bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
+
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
   int32_t code = 0;
   int64_t n;

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
 
   tsdbDelFileName(pTsdb, pFile, fname);
   pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
-  if (pDelFReader == NULL) {
+  if (pDelFReader->pReadH == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     taosMemoryFree(pDelFReader);
     goto _err;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
         *(int64_t *)(dc.data + dc.pos) = uid;
         *(int64_t *)(dc.data + dc.pos + 8) = ctime;
 
+        vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid);
         tEndDecode(&dc);
       }
 
@@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
     goto end;
   }
 
-  vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
+  vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
   int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids);
   if (ret != 0) {
     goto end;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -17,35 +17,22 @@
 #include "vnd.h"
 
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
-  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) ||
+         (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL);
 }
 
 static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
 
-static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
+static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
-  if (!vnodeIsMsgBlock(type)) return;
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
-
+    vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
-  int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1);
-  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
-}
-
-static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
-  int32_t count = atomic_load_32(&pVnode->blockCount);
-  if (count <= 0) return;
-
-  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
     tsem_wait(&pVnode->syncSem);
   }
+}
 
-static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
+static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
-  if (!vnodeIsMsgBlock(type)) return;
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
-
+    vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
-  int32_t count = atomic_load_32(&pVnode->blockCount);
-  if (count <= 0) return;
-
-  count = atomic_sub_fetch_32(&pVnode->blockCount, 1);
-  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
-  if (count <= 0) {
     tsem_post(&pVnode->syncSem);
   }
 }
@@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
   int32_t  code = 0;
   SRpcMsg *pMsg = NULL;
 
+  vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
+
   for (int32_t m = 0; m < numOfMsgs; m++) {
     if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
     const STraceId *trace = &pMsg->info.traceId;
@@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           if (rsp.info.handle != NULL) {
             tmsgSendRsp(&rsp);
           }
+        } else if (code == 0) {
+          vnodeWaitBlockMsg(pVnode, pMsg);
+        } else {
         }
       }
     }
 
-    if (code == 0) {
+    if (code < 0) {
-      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
-    } else if (code < 0) {
       if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
         vnodeRedirectRpcMsg(pVnode, pMsg);
       } else {
@@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           tmsgSendRsp(&rsp);
         }
       }
-    } else {
     }
 
     vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
-
-  vnodeWaitBlockMsg(pVnode);
 }
 
 void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
@@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
       }
     }
 
-    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    vnodePostBlockMsg(pVnode, pMsg);
     if (rsp.info.handle != NULL) {
       tmsgSendRsp(&rsp);
     }
@@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
     tmsgSendRsp(&rpcMsg);
   }
 
-  vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA);
+  vnodePostBlockMsg(pVnode, pMsg);
 }
 
 static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {

source/libs/executor/inc/executil.h

@@ -15,6 +15,7 @@
 #ifndef TDENGINE_QUERYUTIL_H
 #define TDENGINE_QUERYUTIL_H
 
+#include "vnode.h"
 #include "function.h"
 #include "nodes.h"
 #include "plannodes.h"
@@ -106,7 +107,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
 
 EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
-int32_t  getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
+int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo);
 SArray*  createSortInfo(SNodeList* pNodeList);
 SArray*  extractPartitionColInfo(SNodeList* pNodeList);
 SArray*  extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
@@ -128,4 +129,6 @@ int32_t convertFillType(int32_t mode);
 
 int32_t resultrowComparAsc(const void* p1, const void* p2);
 
+int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
+
 #endif  // TDENGINE_QUERYUTIL_H

source/libs/executor/inc/executorimpl.h

@@ -311,17 +311,8 @@ typedef struct STableScanInfo {
   int32_t         dataBlockLoadFlag;
   SInterval       interval;     // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
   SSampleExecInfo sample;       // sample execution info
-
   int32_t         currentGroupId;
   int32_t         currentTable;
-
-#if 0
-  struct {
-    uint64_t uid;
-    int64_t ts;
-  } lastStatus;
-#endif
-
   int8_t          scanMode;
   int8_t          noTable;
 } STableScanInfo;
@@ -430,7 +421,8 @@ typedef struct SStreamScanInfo {
   // status for tmq
   // SSchemaWrapper schema;
   STqOffset              offset;
-
+  SNode*                 pTagCond;
+  SNode*                 pTagIndexCond;
 } SStreamScanInfo;
 
 typedef struct SSysTableScanInfo {
@@ -874,8 +866,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode,
                                                SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-    STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId);
+                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup);
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
 
@@ -966,12 +958,12 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
                                        SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
 int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
+                                STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr);
 
 SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
                                            SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 

source/libs/executor/src/executil.c

@@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
+int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, metaHandle, 0);
   metaGetTableEntryByUid(&mr, info->uid);
@@ -280,19 +280,22 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
   if (TSDB_CODE_SUCCESS != code) {
     terrno = code;
     nodesDestroyNode(pTagCondTmp);
-    return false;
+    *pQualified = false;
+
+    return code;
   }
 
   ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
   SValueNode* pValue = (SValueNode*)pNew;
 
   ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL);
-  bool result = pValue->datum.b;
+  *pQualified = pValue->datum.b;
+
   nodesDestroyNode(pNew);
-  return result;
+  return TSDB_CODE_SUCCESS;
 }
 
-int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
+int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
@@ -304,8 +307,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
 
   pListInfo->suid = pScanNode->suid;
 
-  SNode* pTagCond = (SNode*)pListInfo->pTagCond;
-  SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
   if (pScanNode->tableType == TSDB_SUPER_TABLE) {
     if (pTagIndexCond) {
       SIndexMetaArg metaArg = {
@@ -345,9 +346,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
     int32_t i = 0;
     while (i < taosArrayGetSize(pListInfo->pTableList)) {
       STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
-      bool           isOk = isTableOk(info, pTagCond, metaHandle);
+
-      if (terrno) return terrno;
+      bool qualified = true;
-      if (!isOk) {
+      code = isTableOk(info, pTagCond, metaHandle, &qualified);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      if (!qualified) {
         taosArrayRemove(pListInfo->pTableList, i);
         continue;
       }
@@ -362,7 +368,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
 
   // put into list as default group, remove it if grouping sorting is required later
   taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
-
   return code;
 }
 

source/libs/executor/src/executor.c

@@ -153,7 +153,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
   return pTaskInfo;
 }
 
-static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) {
+static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) {
   SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
 
   // let's discard the tables those are not created according to the queried super table.
@@ -164,7 +164,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons
 
     int32_t code = metaGetTableEntryByUid(&mr, *id);
     if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
+      qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr);
       continue;
     }
 
@@ -172,6 +172,21 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons
     if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
+
+    if (pScanInfo->pTagCond != NULL) {
+      bool          qualified = false;
+      STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid, .lastKey = 0};
+      code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
+      if (code != TSDB_CODE_SUCCESS) {
+        qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
+        continue;
+      }
+
+      if (!qualified) {
+        continue;
+      }
+    }
+
     /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
     // handle multiple partition
 
@@ -194,7 +209,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
   int32_t          code = 0;
   SStreamScanInfo* pScanInfo = pInfo->info;
   if (isAdd) {  // add new table id
-    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
+    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
 
     qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);

source/libs/executor/src/executorimpl.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <executorimpl.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@@ -4369,8 +4368,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC
 }
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
-                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo,
+                                  STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) {
-                                  const char* pUser) {
   int32_t type = nodeType(pPhyNode);
 
   if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
@@ -4378,7 +4376,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
       int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                             pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                             pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
       if (code) {
         pTaskInfo->code = code;
         return NULL;
@@ -4398,7 +4396,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                             pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                             pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
       if (code) {
         pTaskInfo->code = code;
         return NULL;
@@ -4411,7 +4409,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       }
 
       SOperatorInfo* pOperator =
-          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
+          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo);
 
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
@@ -4428,15 +4426,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       };
       if (pHandle) {
         int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                               pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                               pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
         if (code) {
           pTaskInfo->code = code;
           return NULL;
         }
       }
 
-      SOperatorInfo* pOperator =
+      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo, &twSup);
-          createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -4445,7 +4442,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
       STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
 
-      int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo);
+      int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo);
       if (code != TSDB_CODE_SUCCESS) {
         pTaskInfo->code = terrno;
         return NULL;
@@ -4481,8 +4478,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) {
       SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode;
 
-      int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo,
+      int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
-                                             queryId, taskId);
       if (code != TSDB_CODE_SUCCESS) {
         pTaskInfo->code = code;
         return NULL;
@@ -4506,7 +4502,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
   SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
   for (int32_t i = 0; i < size; ++i) {
     SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
-    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser);
+    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser);
     if (ops[i] == NULL) {
       return NULL;
     }
@@ -4689,6 +4685,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
   return pList;
 }
 
+#if 0
 STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, const char* idstr) {
   int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
@@ -4722,6 +4719,7 @@ _error:
   terrno = code;
   return NULL;
 }
+#endif
 
 static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) {
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@@ -4765,6 +4763,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) {
   return -1;
 }
 
+#if 0
 int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) {
   STableScanInfo* pTableScanInfo = NULL;
   if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) {
@@ -4788,6 +4787,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
   // TODO: set uid and ts to data reader
   return 0;
 }
+#endif
 
 int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
   int32_t code = TDB_CODE_SUCCESS;
@@ -4939,10 +4939,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   }
 
   (*pTaskInfo)->sql = sql;
-  (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
+  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user);
-  (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
-  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
-                                           &(*pTaskInfo)->tableqinfoList, pPlan->user);
 
   if (NULL == (*pTaskInfo)->pRoot) {
     code = (*pTaskInfo)->code;

source/libs/executor/src/scanoperator.c

@@ -1497,9 +1497,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) {
   taosMemoryFree(pStreamScan);
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
+                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) {
-                                            uint64_t taskId) {
   SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
   SOperatorInfo*   pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -1512,6 +1511,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
 
   SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
 
+  pInfo->pTagCond = pTagCond;
+
   int32_t numOfCols = 0;
   pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
 
@@ -2550,26 +2551,19 @@ typedef struct STableMergeScanInfo {
   int32_t         tableEndIndex;
   bool            hasGroupId;
   uint64_t        groupId;
-
   SArray*         dataReaders;  // array of tsdbReaderT*
   SReadHandle     readHandle;
-
   int32_t         bufPageSize;
   uint32_t        sortBufSize;  // max buffer size for in-memory sort
-
   SArray*         pSortInfo;
   SSortHandle*    pSortHandle;
 
   SSDataBlock*    pSortInputBlock;
   int64_t         startTs;  // sort start time
-
   SArray*         sortSourceParams;
-  uint64_t queryId;
-  uint64_t taskId;
 
   SFileBlockLoadRecorder readRecorder;
   int64_t         numOfRows;
-  //  int32_t         prevGroupId;  // previous table group id
   SScanInfo       scanInfo;
   int32_t         scanTimes;
   SNode*          pFilterNode;  // filter info, which is push down by optimizer
@@ -2584,26 +2578,25 @@ typedef struct STableMergeScanInfo {
   SExprInfo*      pPseudoExpr;
   int32_t         numOfPseudoExpr;
   SqlFunctionCtx* pPseudoCtx;
-  //  int32_t*        rowEntryInfoOffset;
 
   SQueryTableDataCond cond;
   int32_t         scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
   int32_t         dataBlockLoadFlag;
-  SInterval interval;  // if the upstream is an interval operator, the interval info is also kept here to get the time
+  // if the upstream is an interval operator, the interval info is also kept here to get the time
   // window to check if current data block needs to be loaded.
-
+  SInterval       interval;
   SSampleExecInfo sample;  // sample execution info
 } STableMergeScanInfo;
 
 int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
+                                STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) {
-  int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo);
+  int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
   if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
-    qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
+    qDebug("no table qualified for query, %s" PRIx64, idStr);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -3027,8 +3020,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
 }
 
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
-                                                uint64_t taskId) {
   STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
   SOperatorInfo*       pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -3067,9 +3059,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 
   pInfo->pResBlock = createResDataBlock(pDescNode);
   pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
-  pInfo->queryId = queryId;
-  pInfo->taskId = taskId;
-
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order);

source/libs/executor/src/timewindowoperator.c

@@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo
                                       SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                       int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
                                       SExecTaskInfo* pTaskInfo) {
-  assert(win->skey <= win->ekey);
   SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                   masterscan, tableGroupId, pTaskInfo, true, pAggSup);
 

source/libs/function/src/builtins.c

@@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
 void static addTimezoneParam(SNodeList* pList) {
   char       buf[6] = {0};
   time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
+  struct tm  tmInfo;
-  strftime(buf, sizeof(buf), "%z", tmInfo);
+  taosLocalTime(&t, &tmInfo);
+  strftime(buf, sizeof(buf), "%z", &tmInfo);
   int32_t len = (int32_t)strlen(buf);
 
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
@@ -1422,14 +1423,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 }
 
 static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
-  for (int32_t i = 0; i < numOfParams; ++i) {
-    SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
-    if (QUERY_NODE_VALUE == nodeType(pParamNode)) {
-      return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
-    }
-  }
-
   pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
   return TSDB_CODE_SUCCESS;
 }
@@ -2086,6 +2079,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = topBotFunctionSetup,
     .processFunc  = topFunction,
+    .sprocessFunc = topBotScalarFunction,
     .finalizeFunc = topBotFinalize,
     .combineFunc  = topCombine,
     .pPartialFunc = "top",
@@ -2100,6 +2094,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = topBotFunctionSetup,
     .processFunc  = bottomFunction,
+    .sprocessFunc = topBotScalarFunction,
     .finalizeFunc = topBotFinalize,
     .combineFunc  = bottomCombine,
     .pPartialFunc = "bottom",
@@ -2229,6 +2224,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = lastRowFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize
   },
   {
@@ -2249,6 +2245,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = firstFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize,
     .pPartialFunc = "_first_partial",
     .pMergeFunc   = "_first_merge",
@@ -2284,6 +2281,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = lastFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize,
     .pPartialFunc = "_last_partial",
     .pMergeFunc   = "_last_merge",
@@ -2464,6 +2462,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getSampleFuncEnv,
     .initFunc     = sampleFunctionSetup,
     .processFunc  = sampleFunction,
+    .sprocessFunc = sampleScalarFunction,
     .finalizeFunc = sampleFinalize
   },
   {
@@ -2475,6 +2474,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTailFuncEnv,
     .initFunc     = tailFunctionSetup,
     .processFunc  = tailFunction,
+    .sprocessFunc = tailScalarFunction,
     .finalizeFunc = NULL
   },
   {
@@ -2486,6 +2486,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getUniqueFuncEnv,
     .initFunc     = uniqueFunctionSetup,
     .processFunc  = uniqueFunction,
+    .sprocessFunc = uniqueScalarFunction,
     .finalizeFunc = NULL
   },
   {
@@ -2496,6 +2497,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getModeFuncEnv,
     .initFunc     = modeFunctionSetup,
     .processFunc  = modeFunction,
+    .sprocessFunc = modeScalarFunction,
     .finalizeFunc = modeFinalize,
   },
   {

source/libs/nodes/src/nodesCodeFuncs.c

@@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
 static const char* jkScanLogicPlanTableType = "TableType";
 static const char* jkScanLogicPlanTableId = "TableId";
 static const char* jkScanLogicPlanStableId = "StableId";
+static const char* jkScanLogicPlanScanType = "ScanType";
 static const char* jkScanLogicPlanScanCount = "ScanCount";
 static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount";
 static const char* jkScanLogicPlanTagCond = "TagCond";
 static const char* jkScanLogicPlanGroupTags = "GroupTags";
 
-// typedef struct SScanLogicNode {
-//   uint64_t      stableId;
-//   SVgroupsInfo* pVgroupList;
-//   EScanType     scanType;
-//   uint8_t       scanSeq[2];  // first is scan count, and second is reverse scan count
-//   STimeWindow   scanRange;
-//   SName         tableName;
-//   bool          showRewrite;
-//   double        ratio;
-//   SNodeList*    pDynamicScanFuncs;
-//   int32_t       dataRequired;
-//   int64_t       interval;
-//   int64_t       offset;
-//   int64_t       sliding;
-//   int8_t        intervalUnit;
-//   int8_t        slidingUnit;
-//   SNode*        pTagCond;
-//   SNode*        pTagIndexCond;
-//   int8_t        triggerType;
-//   int64_t       watermark;
-//   int8_t        igExpired;
-//   SArray*       pSmaIndexes;
-//   SNodeList*    pGroupTags;
-//   bool          groupSort;
-// } SScanLogicNode;
-
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
 
@@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]);
   }
@@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]);
   }
@@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectPhysiPlanProjections = "Projections";
+static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock";
 
 static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock);
+  }
 
   return code;
 }
@@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock);
+  }
 
   return code;
 }

source/libs/parser/src/parCalcConst.c

@@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) {
 }
 
 static bool isUselessCol(SExprNode* pProj) {
-  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) {
+  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) &&
+      !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) {
     return false;
   }
   return NULL == ((SExprNode*)pProj)->pAssociation;

source/libs/parser/src/parTranslater.c

@@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   if (NULL == pReq->tagName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
   if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
@@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/planner/src/planLogicCreater.c

@@ -20,6 +20,7 @@
 typedef struct SLogicPlanContext {
   SPlanContext* pPlanCxt;
   SLogicNode*   pCurrRoot;
+  bool          hasScan;
 } SLogicPlanContext;
 
 typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**);
@@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
     return SCAN_TYPE_STREAM;
   }
 
+  if (TSDB_SYSTEM_TABLE == tableType) {
+    return SCAN_TYPE_SYSTEM_TABLE;
+  }
+
   if (NULL == pScanCols) {
     return NULL == pScanPseudoCols
                ? SCAN_TYPE_TABLE
@@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
                       : SCAN_TYPE_TABLE);
   }
 
-  if (TSDB_SYSTEM_TABLE == tableType) {
-    return SCAN_TYPE_SYSTEM_TABLE;
-  }
-
-  SNode* pCol = NULL;
-  FOREACH(pCol, pScanCols) {
-    if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) {
-      return SCAN_TYPE_TABLE;
-    }
-  }
-
   return SCAN_TYPE_TABLE;
 }
 
@@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     nodesDestroyNode((SNode*)pScan);
   }
 
+  pCxt->hasScan = true;
+
   return code;
 }
 
@@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) {
 
 static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); }
 
-static void setLogicSubplanType(SLogicSubplan* pSubplan) {
+static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) {
   if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) {
-    pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
+    pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE;
   } else {
     SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode;
     pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren)
@@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) {
 }
 
 int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
-  SLogicPlanContext cxt = {.pPlanCxt = pCxt};
+  SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false};
 
   SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
   if (NULL == pSubplan) {
@@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
   int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode);
   if (TSDB_CODE_SUCCESS == code) {
     setLogicNodeParent(pSubplan->pNode);
-    setLogicSubplanType(pSubplan);
+    setLogicSubplanType(cxt.hasScan, pSubplan);
   }
 
   if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/src/planPhysiCreater.c

@@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext {
   int32_t       errCode;
   int16_t       nextDataBlockId;
   SArray*       pLocationHelper;
-  SArray*       pExecNodeList;  // SArray<SQueryNodeLoad>
+  bool          hasScan;
+  bool          hasSysScan;
 } SPhysiPlanContext;
 
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
@@ -255,7 +256,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo
 
 static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList,
                                            SDataBlockDescNode* pDataBlockDesc) {
-  return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false);
+  return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false);
 }
 
 static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) {
@@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-  SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
 }
 
@@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
     pSubplan->execNode.nodeId = MNODE_HANDLE;
     pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet;
   }
-  SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet;
   } else {
@@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
   }
   tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
 
+  pCxt->hasSysScan = true;
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
 }
 
@@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
 
 static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
                                    SPhysiNode** pPhyNode) {
+  pCxt->hasScan = true;
   switch (pScanLogicNode->scanType) {
     case SCAN_TYPE_TAG:
     case SCAN_TYPE_BLOCK_INFO:
@@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) {
   }
 }
 
+static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) {
+  if (NULL == pExecNodeList) {
+    return;
+  }
+  if (pCxt->hasSysScan || !pCxt->hasScan) {
+    SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
+    taosArrayPush(pExecNodeList, &node);
+  }
+}
+
 int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
   SPhysiPlanContext cxt = {.pPlanCxt = pCxt,
                            .errCode = TSDB_CODE_SUCCESS,
                            .nextDataBlockId = 0,
                            .pLocationHelper = taosArrayInit(32, POINTER_BYTES),
-                           .pExecNodeList = pExecNodeList};
+                           .hasScan = false,
+                           .hasSysScan = false};
   if (NULL == cxt.pLocationHelper) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (QUERY_POLICY_VNODE == tsQueryPolicy) {
-    taosArrayClear(pExecNodeList);
-  }
-
   int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan);
   if (TSDB_CODE_SUCCESS == code) {
     setExplainInfo(pCxt, *pPlan);
+    setExecNodeList(&cxt, pExecNodeList);
   }
 
   destoryPhysiPlanContext(&cxt);

source/libs/scalar/src/sclfunc.c

@@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
       memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS);
     }
 
-    struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL);
+    struct tm tmInfo;
-    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo);
+    taosLocalTime((const time_t *)&timeVal, &tmInfo);
+    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo);
     int32_t len = (int32_t)strlen(buf);
 
     //add timezone string
@@ -2816,3 +2817,48 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
   pOutput->numOfRows = numOfBins;
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  int32_t type = GET_PARAM_TYPE(pInput);
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, 0);
+      continue;
+    }
+
+    char* data = colDataGetData(pInputData, i);
+    colDataAppend(pOutputData, i, data, false);
+  }
+
+
+  pOutput->numOfRows = 1;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}

source/libs/stream/src/streamExec.c

@@ -143,6 +143,7 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) {
 }
 
 static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
+  while (1) {
     int32_t cnt = 0;
     void*   data = NULL;
     while (1) {
@@ -153,10 +154,12 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
       }
       if (data == NULL) {
         data = qItem;
+        streamQueueProcessSuccess(pTask->inputQueue);
         if (qItem->type == STREAM_INPUT__DATA_BLOCK) {
           /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/
+        } else {
+          break;
         }
-      streamQueueProcessSuccess(pTask->inputQueue);
       } else {
         if (streamAppendQueueItem(data, qItem) < 0) {
           streamQueueProcessFail(pTask->inputQueue);
@@ -176,7 +179,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
       return NULL;
     }
 
-  if (data == NULL) return pRes;
+    if (data == NULL) break;
 
     if (pTask->execType == TASK_EXEC__NONE) {
       ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK);
@@ -213,6 +216,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
     }
 
     streamFreeQitem(data);
+  }
   return pRes;
 }
 

source/libs/sync/inc/syncInt.h

@@ -192,6 +192,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms);
 int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
+int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode);
 
 // utils --------------
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);

source/libs/sync/src/syncMain.c

@@ -1298,6 +1298,12 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
   return ret;
 }
 
+int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) {
+  syncNodeStopHeartbeatTimer(pSyncNode);
+  syncNodeStartHeartbeatTimer(pSyncNode);
+  return 0;
+}
+
 // utils --------------
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;

source/libs/sync/src/syncRaftLog.c

@@ -313,7 +313,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
       snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
                index, err, err, errStr, sysErr, sysErrStr);
       if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
-        syncNodeEventLog(pData->pSyncNode, logBuf);
+        // syncNodeEventLog(pData->pSyncNode, logBuf);
       } else {
         syncNodeErrorLog(pData->pSyncNode, logBuf);
       }
@@ -499,7 +499,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
         snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
                  index, err, err, errStr, sysErr, sysErrStr);
         if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
-          syncNodeEventLog(pData->pSyncNode, logBuf);
+          // syncNodeEventLog(pData->pSyncNode, logBuf);
         } else {
           syncNodeErrorLog(pData->pSyncNode, logBuf);
         }

source/libs/tdb/src/db/tdbBtree.c

@@ -103,10 +103,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
   // if pgno == 0 fetch new btree root leaf page
   if (pgno == 0) {
     // fetch page & insert into main db
-    // allocate a new child page
     SPage *pPage;
     TXN    txn;
-    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
+    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
 
     pPager->inTran = 1;
 
@@ -118,8 +117,6 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
       return -1;
     }
 
-    // TODO: Need to zero the page
-
     ret = tdbPagerWrite(pPager, pPage);
     if (ret < 0) {
       return -1;

source/libs/tdb/src/db/tdbPager.c

@@ -473,12 +473,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
     return -1;
   }
 
-  TXN txn;
-  tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  SBtreeInitPageArg iArg;
-  iArg.pBt = pBt;
-  iArg.flags = 0;
-
   for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) {
     // read pgno & the page from journal
     SPgno  pgno;
@@ -494,20 +488,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
       return -1;
     }
 
-    /*
-    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn);
-    if (ret < 0) {
-      return -1;
-    }
-
-    // write the page to db
-    ret = tdbPagerWritePageToDB(pPager, pPage);
-    if (ret < 0) {
-      return -1;
-    }
-
-    tdbPCacheRelease(pPager->pCache, pPage, &txn);
-    */
     i64 offset = pPager->pageSize * (pgno - 1);
     if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
       ASSERT(0);
@@ -523,8 +503,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
 
   tdbOsFSync(pPager->fd);
 
-  tdbTxnClose(&txn);
-
   tdbOsFree(pageBuf);
 
   tdbOsClose(jfd);

source/libs/wal/src/walRead.c

@@ -483,6 +483,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
            pRead->pHead->head.version, ver);
     pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+    ASSERT(0);
     return -1;
   }
 
@@ -491,6 +492,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
     wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver);
     pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+    ASSERT(0);
     return -1;
   }
   pRead->curVersion++;

source/os/src/osTime.c

@@ -357,14 +357,88 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) {
 
 time_t taosTime(time_t *t) { return time(t); }
 
-time_t taosMktime(struct tm *timep) { return mktime(timep); }
+time_t taosMktime(struct tm *timep) {
+#ifdef WINDOWS
+  struct tm tm1 = {0};
+  LARGE_INTEGER        t;
+  FILETIME             f;
+  SYSTEMTIME           s;
+  FILETIME             ff;
+  SYSTEMTIME           ss;
+  LARGE_INTEGER        offset;
+
+  time_t    tt = 0;
+  localtime_s(&tm1, &tt);
+  ss.wYear = tm1.tm_year + 1900;
+  ss.wMonth = tm1.tm_mon + 1;
+  ss.wDay = tm1.tm_wday;
+  ss.wHour = tm1.tm_hour;
+  ss.wMinute = tm1.tm_min;
+  ss.wSecond = tm1.tm_sec;
+  ss.wMilliseconds = 0;
+  SystemTimeToFileTime(&ss, &ff);
+  offset.QuadPart = ff.dwHighDateTime;
+  offset.QuadPart <<= 32;
+  offset.QuadPart |= ff.dwLowDateTime;
+
+  s.wYear = timep->tm_year + 1900;
+  s.wMonth = timep->tm_mon + 1;
+  s.wDay = timep->tm_wday;
+  s.wHour = timep->tm_hour;
+  s.wMinute = timep->tm_min;
+  s.wSecond = timep->tm_sec;
+  s.wMilliseconds = 0;
+  SystemTimeToFileTime(&s, &f);
+  t.QuadPart = f.dwHighDateTime;
+  t.QuadPart <<= 32;
+  t.QuadPart |= f.dwLowDateTime;
+
+  t.QuadPart -= offset.QuadPart;
+ return (time_t)(t.QuadPart / 10000000);
+#else
+  return mktime(timep);
+#endif
+ }
 
 struct tm *taosLocalTime(const time_t *timep, struct tm *result) {
   if (result == NULL) {
     return localtime(timep);
   }
 #ifdef WINDOWS
+  if (*timep < 0) {
+    SYSTEMTIME    ss,s;
+    FILETIME      ff,f;
+    LARGE_INTEGER offset;
+    struct tm     tm1;
+    time_t        tt = 0;
+    localtime_s(&tm1, &tt);
+    ss.wYear = tm1.tm_year + 1900;
+    ss.wMonth = tm1.tm_mon + 1;
+    ss.wDay = tm1.tm_mday;
+    ss.wHour = tm1.tm_hour;
+    ss.wMinute = tm1.tm_min;
+    ss.wSecond = tm1.tm_sec;
+    ss.wMilliseconds = 0;
+    SystemTimeToFileTime(&ss, &ff);
+    offset.QuadPart = ff.dwHighDateTime;
+    offset.QuadPart <<= 32;
+    offset.QuadPart |= ff.dwLowDateTime;
+    offset.QuadPart += *timep * 10000000;
+    f.dwLowDateTime = offset.QuadPart & 0xffffffff;
+    f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff;
+    FileTimeToSystemTime(&f, &s);
+    result->tm_sec = s.wSecond;
+    result->tm_min = s.wMinute;
+    result->tm_hour = s.wHour;
+    result->tm_mday = s.wDay;
+    result->tm_mon = s.wMonth-1;
+    result->tm_year = s.wYear-1900;
+    result->tm_wday = s.wDayOfWeek;
+    result->tm_yday = 0;
+    result->tm_isdst = 0;
+  } else {
     localtime_s(result, timep);
+  }
 #else
   localtime_r(timep, result);
 #endif

tests/pytest/crash_gen/crash_gen_main.py

@@ -30,6 +30,7 @@ import argparse
 import sys
 import os
 import io
+import datetime
 import signal
 import traceback
 import requests
@@ -1107,14 +1108,20 @@ class Database:
     # TODO: fix the error as result of above: "tsdb timestamp is out of range"
     @classmethod
     def setupLastTick(cls):
-        t1 = datetime.datetime(2020, 6, 1)
+        # start time will be auto generated , start at 10 years ago  local time 
+        local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16]
+        local_epoch_time = [int(i) for i in local_time.split("-")]
+        #local_epoch_time will be such as : [2022, 7, 18]
+
+        t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2])
         t2 = datetime.datetime.now()
         # maybe a very large number, takes 69 years to exceed Python int range
         elSec = int(t2.timestamp() - t1.timestamp())
         elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \
             500  # a number representing seconds within 10 years
         # print("elSec = {}".format(elSec))
-        t3 = datetime.datetime(2012, 1, 1)  # default "keep" is 10 years
+
+        t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2])  # default "keep" is 10 years
         t4 = datetime.datetime.fromtimestamp(
             t3.timestamp() + elSec2)  # see explanation above
         Logging.debug("Setting up TICKS to start from: {}".format(t4))

tests/script/jenkins/basic.txt

@@ -312,6 +312,7 @@
 ./test.sh -f tsim/valgrind/checkError3.sim
 ./test.sh -f tsim/valgrind/checkError4.sim
 ./test.sh -f tsim/valgrind/checkError5.sim
+./test.sh -f tsim/valgrind/checkError6.sim
 
 # --- vnode
 # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim

tests/script/tmp/data.sim

@@ -1,18 +1,125 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1
+system sh/deploy.sh -n dnode2 -i 2
-system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1
+system sh/deploy.sh -n dnode3 -i 3
-system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost
+system sh/deploy.sh -n dnode4 -i 4
-system sh/cfg.sh -n dnode1 -c telemetryPort -v 80
-
-return
 system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
 sql connect
 
-sql create database db 
+print =============== step1: create dnodes
-sql create table db.tb (ts timestamp, i int)
+sql create dnode $hostname port 7200
-sql insert into db.tb values(now, 1)
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
 
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> rows: $rows 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data24 $data35
+if $rows != 4 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create database
+sql create database db vgroups 1 replica 3
+sql show databases
+if $rows != 3 then 
+  return -1
+endi
+if $data(db)[4] != 3 then 
+  return -1
+endi
+
+sql show dnodes
+if $data(2)[2] != 1 then
+  return -1
+endi
+if $data(3)[2] != 1 then
+  return -1
+endi
+if $data(4)[2] != 1 then
+  return -1
+endi
+
+# vnodes
+sql show dnodes
+if $data(2)[2] != 1 then
+  return -1
+endi
+if $data(3)[2] != 1 then
+  return -1
+endi
+if $data(4)[2] != 1 then
+  return -1
+endi
+
+# v1_dnode
+$hasleader = 0
+$x = 0
+step2:
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show db.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(2)[5] != 3 then
+  return -1
+endi
+if $data(2)[7] != 4 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $hasleader = 1
+endi
+if $data(2)[6] == leader then
+  $hasleader = 1
+endi
+if $data(2)[8] == leader then
+  $hasleader = 1
+endi
+if $hasleader != 1 then
+  goto step2
+endi
+
+sql use db;
+sql create table stb (ts timestamp, c int) tags (t int);
+sql create table t0 using stb tags (0);
+sql insert into t0 values(now, 1);
+sql show db.stables;
+sql show db.tables;
+sql show db.vgroups;
+
+return
 print ======== start back
 run_back tmp/back.sim
 

tests/script/tmp/monitor.sim

@@ -6,32 +6,21 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1
 system sh/cfg.sh -n dnode1 -c monitorComp -v 1
 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 128
 
+#system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1
+#system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1
+#system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost
+#system sh/cfg.sh -n dnode1 -c telemetryPort -v 80
+
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
 print =============== show dnodes
-sleep 2000
 sql create database db vgroups 2;
 sql use db;
 sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd";
-sleep 2000
 
 print =============== create drop qnode 1
 sql create qnode on dnode 1
 sql create snode on dnode 1
 sql create bnode on dnode 1
 
-return
-print =============== restart
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode1 -s start
-
-
-return 
-system sh/deploy.sh -n dnode2 -i 2
-system sh/exec.sh -n dnode2 -s start
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s start
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT

tests/script/tsim/valgrind/checkError1.sim

@@ -152,7 +152,7 @@ endi
 
 system_content sh/checkValgrind.sh -n dnode2
 print cmd return result ----> [ $system_content ]
-if $system_content > 4 then
+if $system_content > 0 then
   return -1
 endi 
 

tests/script/tsim/valgrind/checkError5.sim

@@ -26,6 +26,7 @@ print =============== step2: create db
 sql create database db
 sql use db
 sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
+sql create table db.c1 using db.stb tags(101, 102, "103")
 
 print =============== step3: alter stb
 sql_error alter table db.stb add column ts int
@@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1
 sql alter table db.stb drop tag t5
 sql alter table db.stb MODIFY tag t3 binary(32)
 sql alter table db.stb rename tag t1 tx
-
 sql alter table db.stb comment 'abcde' ;
-goto _OVER
+sql drop table db.stb
 
 print =============== step4: alter tb
 sql create table tb (ts timestamp, a int)
@@ -66,6 +66,35 @@ sql alter table tb add column h binary(10)
 sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb
 sql select * from tb order by ts desc
 
+print =============== step5: alter stb and insert data
+sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd"
+sql show db.stables
+sql describe stb
+sql_error alter table stb add column ts int
+
+sql create table db.ctb using db.stb tags(101, 102, "103")
+sql insert into db.ctb values(now, 1, "2")
+sql show db.tables
+sql select * from db.stb
+sql select * from tb
+
+sql alter table stb add column c3 int
+sql describe stb
+sql select * from db.stb
+sql select * from tb
+sql insert into db.ctb values(now+1s, 1, 2, 3)
+sql select * from db.stb
+
+sql alter table db.stb add column c4 bigint
+sql select * from db.stb
+sql insert into db.ctb values(now+2s, 1, 2, 3, 4)
+
+sql alter table db.stb drop column c1
+sql reset query cache
+sql select * from tb
+sql insert into db.ctb values(now+3s, 2, 3, 4)
+sql select * from db.stb
+
 _OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 print =============== check

tests/script/tsim/valgrind/checkError6.sim

@@ -68,7 +68,7 @@ $null=
 
 system_content sh/checkValgrind.sh -n dnode1 
 print cmd return result ----> [ $system_content ]
-if $system_content > 0 then
+if $system_content > 3 then
   return -1
 endi 
 

tests/system-test/0-others/cachemodel.py

@@ -56,6 +56,16 @@ class TDTestCase:
         }
         return numbers.get(value, 'other')
 
+    def getCacheModelNum(self,str):
+        numbers = {
+            "none" : 0,
+            "last_row" : 1,
+            "last_value" : 2,
+            "both" : 3
+
+        }
+        return numbers.get(str, 'other')
+
     def prepare_datas(self):
         for i in range(4):
             str = self.getCacheModelStr(i)
@@ -69,7 +79,7 @@ class TDTestCase:
                 tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) )
                 tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) )
 
-    def check_cache_last_sets(self):
+    def check_cachemodel_sets(self):
         
         
         # check cache_last value for database 
@@ -84,52 +94,54 @@ class TDTestCase:
             # print(cache_last_value)
             if dbname in ["information_schema" , "performance_schema"]:
                 continue
-            cache_lasts[dbname]=cache_last_value
+            cache_lasts[dbname]=self.getCacheModelNum(cache_last_value)
 
         
         # cache_last_set value 
         for k , v in cache_lasts.items():
             
-            if k=="testdb_"+str(v):
+            if k=="testdb_"+str(self.getCacheModelStr(v)):
-                tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) )
+                tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) )
             else:
-                tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) )
+                tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) )
 
         # # check storage layer implementation
 
 
-        # buildPath = self.getBuildPath()
+        buildPath = self.getBuildPath()
-        # if (buildPath == ""):
+        if (buildPath == ""):
-        #     tdLog.exit("taosd not found!")
+            tdLog.exit("taosd not found!")
-        # else:
+        else:
-        #     tdLog.info("taosd found in %s" % buildPath)
+            tdLog.info("taosd found in %s" % buildPath)
-        # dataPath = buildPath + "/../sim/dnode1/data"
+        dataPath = buildPath + "/../sim/dnode1/data"
-        # abs_vnodePath = os.path.abspath(dataPath)+"/vnode/"
+        abs_vnodePath = os.path.abspath(dataPath)+"/vnode/"
-        # tdLog.info("abs_vnodePath: %s" % abs_vnodePath)
+        tdLog.info("abs_vnodePath: %s" % abs_vnodePath)
         
-        # tdSql.query(" show dnodes ")
+        tdSql.query(" show dnodes ")
-        # dnode_id  = tdSql.queryResult[0][0]
+        dnode_id  = tdSql.queryResult[0][0]
 
-        # for dbname in cache_lasts.keys():
+        for dbname in cache_lasts.keys():
             # print(dbname)
-        #     tdSql.execute(" use %s" % dbname)
+            tdSql.execute(" use %s" % dbname)
-        #     tdSql.query(" show vgroups ")
+            tdSql.query(" show vgroups ")
-        #     vgroups_infos = tdSql.queryResult
+            vgroups_infos = tdSql.queryResult
-        #     for vgroup_info in vgroups_infos:
+            for vgroup_info in vgroups_infos:
-        #         vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json"
+                vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json"
-        #         vnode_info_of_db = f"cat {vnode_json}" 
+                vnode_info_of_db = f"cat {vnode_json}" 
-        #         vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8")
+                vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8")
-        #         infoDict = json.loads(vnode_info)
+                infoDict = json.loads(vnode_info)
-        #         vnode_json_of_dbname = f"{dnode_id}."+ dbname
+                vnode_json_of_dbname = f"{dnode_id}."+ dbname
-        #         config = infoDict["config"]
+                config = infoDict["config"]
-        #         if infoDict["config"]["dbname"] == vnode_json_of_dbname:
+                if infoDict["config"]["dbname"] == vnode_json_of_dbname:
-        #             if "cachelast" in infoDict["config"]:
+                    if "cacheLast" in infoDict["config"]:
-        #                 if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]:
+                        if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]:
-        #                     tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0]))
+                            tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0]))
-        #             else:
+                        else:
-        #                 tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0]))
+                            tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0]))
+                    else:
+                        tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0]))
 
-    def restart_check_cache_last_sets(self):
+    def restart_check_cachemodel_sets(self):
         
         for i in range(3):
             tdSql.query("show dnodes")
@@ -137,14 +149,14 @@ class TDTestCase:
             tdDnodes.stop(index)
             tdDnodes.start(index)
             time.sleep(3)
-            self.check_cache_last_sets()
+            self.check_cachemodel_sets()
 
 
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         self.illegal_params()
         self.prepare_datas()
-        self.check_cache_last_sets()
+        self.check_cachemodel_sets()
-        self.restart_check_cache_last_sets()
+        self.restart_check_cachemodel_sets()
        
     def stop(self):
         tdSql.close()

tests/system-test/1-insert/delete_data.py

@@ -0,0 +1,190 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+
+from numpy import logspace
+from util import constant
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import TDSetSql
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.dbname = 'db_test'
+        self.setsql = TDSetSql()
+        self.ntbname = 'ntb'
+        self.rowNum = 10
+        self.tbnum = 20
+        self.ts = 1537146000000
+        self.binary_str = 'taosdata'
+        self.nchar_str = '涛思数据'
+        self.str_length = 20
+        self.column_dict = {
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': f'binary({self.str_length})',
+            'col13': f'nchar({self.str_length})',
+            
+        }
+        self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
+        self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
+        self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX)
+        self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
+        self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
+        self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
+        self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
+        self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
+        self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
+        self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
+        self.bool_val = random.randint(0,100)%2
+        self.binary_val = tdCom.getLongName(random.randint(0,self.str_length))
+        self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length))
+        self.base_data = {
+            'tinyint':self.tinyint_val,
+            'smallint':self.smallint_val,
+            'int':self.int_val,
+            'bigint':self.bigint_val,
+            'tinyint unsigned':self.untingint_val,
+            'smallint unsigned':self.unsmallint_val,
+            'int unsigned':self.unint_val,
+            'bigint unsigned':self.unbigint_val,
+            'bool':self.bool_val,
+            'float':self.float_val,
+            'double':self.double_val,
+            'binary':self.binary_val,
+            'nchar':self.nchar_val
+                    }
+    def insert_base_data(self,col_type,tbname,rows,base_data):
+        for i in range(rows):
+            if col_type.lower() == 'tinyint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})')
+            elif col_type.lower() == 'smallint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})')
+            elif col_type.lower() == 'int':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})')
+            elif col_type.lower() == 'bigint':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})')
+            elif col_type.lower() == 'tinyint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})')
+            elif col_type.lower() == 'smallint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})')
+            elif col_type.lower() == 'int unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})')
+            elif col_type.lower() == 'bigint unsigned':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})')
+            elif col_type.lower() == 'bool':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})')    
+            elif col_type.lower() == 'float':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})')      
+            elif col_type.lower() == 'double':
+                tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})')
+            elif 'binary' in col_type.lower():
+                tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''')
+            elif 'nchar' in col_type.lower():
+                tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''')        
+
+    def delete_all_data(self,tbname,col_type,row_num,base_data,dbname):
+        tdSql.execute(f'delete from {tbname}')
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select * from {tbname}')
+        tdSql.checkRows(0)
+        self.insert_base_data(col_type,tbname,row_num,base_data)
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select * from {tbname}')
+        tdSql.checkRows(row_num)
+    def delete_one_row(self,tbname,column_type,column_name,base_data,dbname):
+        tdSql.execute(f'delete from {tbname} where ts={self.ts}')
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        tdSql.query(f'select {column_name} from {tbname}')
+        tdSql.checkRows(self.rowNum-1)
+        tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
+        tdSql.checkRows(0)
+        if 'binary' in column_type.lower():
+            tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''')
+        elif 'nchar' in column_type.lower():
+            tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''')
+        else:
+            tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})')
+        tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
+        if column_type.lower() == 'float' or column_type.lower() == 'double':
+            if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001:
+                tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
+            else:
+                tdLog.exit(f'{column_type} data check failure')
+        elif 'binary' in column_type.lower():
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary'])
+        elif 'nchar' in column_type.lower():
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar'])
+        else:
+            tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type])
+    
+    def delete_rows(self):
+
+
+        pass
+    def delete_error(self,tbname,column_name,column_type,base_data):
+        for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']:
+            if 'binary' in column_type.lower():
+                tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''')
+            elif 'nchar' in column_type.lower():
+                tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''')
+            else:
+                tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}')
+                
+    def delete_data_ntb(self):
+        tdSql.execute(f'create database if not exists {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
+            self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
+            self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname)
+            self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname)
+            self.delete_error(self.ntbname,col_name,col_type,self.base_data)
+            for i in range(self.rowNum):
+                tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}')
+                tdSql.execute(f'flush database {self.dbname}')
+                tdSql.execute('reset query cache')
+                tdSql.query(f'select {col_name} from {self.ntbname}')
+                tdSql.checkRows(i+1)
+                self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
+            
+            tdSql.execute(f'drop table {self.ntbname}')
+        
+    def run(self):
+        self.delete_data_ntb()
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/insert_drop.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+import threading
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def genMultiThreadSeq(self, sql_list):
+        tlist = list()
+        for insert_sql in sql_list:
+            t = threading.Thread(target=tdSql.execute, args=(insert_sql,))
+            tlist.append(t)
+        return tlist
+
+    def multiThreadRun(self, tlist):
+        for t in tlist:
+            t.start()
+        for t in tlist:
+            t.join()
+
+    def run(self):
+        tdSql.prepare()
+        tdSql.execute('create database if not exists test;')
+        tdSql.execute('create table test.stb (ts timestamp, c11 int, c12 float ) TAGS(t11 int, t12 int );')
+        tdSql.execute('create table test.tb using test.stb TAGS (1, 1);')
+        sql_list = list()
+        for i in range(5):
+            sql = f'insert into test.tb values (now-{i}m, {i}, {i});'
+            sql_list.append(sql)
+        sql_list.append(f'drop database test;')
+        tlist = self.genMultiThreadSeq(sql_list)
+        self.multiThreadRun(tlist)
+        tdSql.query(f'show databases')
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/update_data.py

@@ -0,0 +1,229 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+from util import constant
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import TDSetSql
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.setsql = TDSetSql()
+        self.dbname = 'db_test'
+        self.ntbname = 'ntb'
+        self.stbname = 'stb'
+        self.ctbname = 'ctb'
+        self.ts = 1537146000000
+        self.str_length = 20
+        self.column_dict = {
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': f'binary({self.str_length})',
+            'col13': f'nchar({self.str_length})',
+            'col_ts'  : 'timestamp'
+        }
+           
+    def data_check(self,tbname,col_name,col_type,value):
+        tdSql.query(f'select {col_name} from {tbname}')
+        if col_type.lower() == 'float' or col_type.lower() == 'double':
+            if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001:
+                tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
+            else:
+                tdLog.exit(f'{col_name} data check failure')
+        elif col_type.lower() == 'timestamp':
+            tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f")))
+        else:
+            tdSql.checkEqual(tdSql.queryResult[0][0],value)
+    def update_and_check_data(self,tbname,col_name,col_type,value,dbname):
+        if 'binary' in col_type.lower() or 'nchar' in col_type.lower():
+            tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")')
+        else:
+            tdSql.execute(f'insert into {tbname} values({self.ts},{value})')
+        self.data_check(tbname,col_name,col_type,value)
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute('reset query cache')
+        self.data_check(tbname,col_name,col_type,value)
+        for func in ['first','last']:
+            tdSql.execute(f'select {func}({col_name}) from {tbname}')
+    def error_check(self,tbname,column_dict,tb_type=None,stbname=None):
+        str_length = self.str_length+1
+        for col_name,col_type in column_dict.items():
+            if tb_type == 'ntb':
+                tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
+            elif tb_type == 'ctb':
+                tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+                tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            if col_type.lower() == 'double':
+                for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'float':
+                for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif 'binary' in col_type.lower() or 'nchar' in col_type.lower():
+                for error_value in [tdCom.getLongName(str_length)]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")')
+            elif col_type.lower() == 'bool':
+                for error_value in [tdCom.getLongName(self.str_length)]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'tinyint':
+                for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'smallint':
+                for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'int':
+                for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'bigint':
+                for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'tinyint unsigned':
+                for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')     
+            elif col_type.lower() == 'smallint unsigned':
+                for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'int unsigned':
+                for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')
+            elif col_type.lower() == 'bigint unsigned':
+                for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]:
+                    tdSql.error(f'insert into {tbname} values({self.ts},{error_value})')           
+            tdSql.execute(f'drop table {tbname}')
+            if tb_type == 'ctb':
+                tdSql.execute(f'drop table {stbname}')
+    def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None):
+        up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
+        up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
+        up_int = random.randint(constant.INT_MIN,constant.INT_MAX)
+        up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
+        up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
+        up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
+        up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
+        up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
+        up_bool = random.randint(0,100)%2
+        up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
+        up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
+        binary_length = random.randint(0,self.str_length)
+        nchar_length = random.randint(0,self.str_length)
+        up_binary = tdCom.getLongName(binary_length)
+        up_nchar = tdCom.getLongName(nchar_length)
+        for col_name,col_type in column_dict.items():
+            if tb_type == 'ntb':
+                tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
+            elif tb_type == 'ctb':
+                tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+                tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            if col_type.lower() == 'tinyint':
+                self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname)
+            elif col_type.lower() == 'smallint':
+                self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname)
+            elif col_type.lower() == 'int':
+                self.update_and_check_data(tbname,col_name,col_type,up_int,dbname)
+            elif col_type.lower() == 'bigint':
+                self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname)
+            elif col_type.lower() == 'tinyint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname)
+            elif col_type.lower() == 'smallint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname)
+            elif col_type.lower() == 'int unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname)
+            elif col_type.lower() == 'bigint unsigned':
+                self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname)
+            elif col_type.lower() == 'bool':
+                self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname)    
+            elif col_type.lower() == 'float':
+                self.update_and_check_data(tbname,col_name,col_type,up_float,dbname)      
+            elif col_type.lower() == 'double':
+                self.update_and_check_data(tbname,col_name,col_type,up_double,dbname)
+            elif 'binary' in col_type.lower():
+                self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
+            elif 'nchar' in col_type.lower():
+                self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
+            elif col_type.lower() == 'timestamp':
+                self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname)
+            tdSql.execute(f'insert into {tbname} values({self.ts},null)')
+            tdSql.query(f'select {col_name} from {tbname}')
+            tdSql.checkEqual(tdSql.queryResult[0][0],None)
+            tdSql.execute(f'flush database {self.dbname}')
+            tdSql.execute('reset query cache')
+            tdSql.query(f'select {col_name} from {tbname}')
+            tdSql.checkEqual(tdSql.queryResult[0][0],None)
+            tdSql.execute(f'drop table {tbname}')
+            if tb_type == 'ctb':
+                tdSql.execute(f'drop table {stbname}')
+    def update_check(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+        tdSql.execute(f'create database {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb')
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
+            tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)')
+            if 'binary' in col_type.lower():
+                up_binary = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})')
+                self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname)
+            elif 'nchar' in col_type.lower():
+                up_nchar = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})')
+                self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname)
+            tdSql.execute(f'drop table {self.ntbname}')
+        self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname)
+        for col_name,col_type in self.column_dict.items():
+            tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)')
+            tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)')
+            tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)')
+            if 'binary' in col_type.lower():
+                up_binary = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})')
+                self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname)
+            elif 'nchar' in col_type.lower():
+                up_nchar = tdCom.getLongName(self.str_length+1)
+                tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})')
+                self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname)
+            tdSql.execute(f'drop table {self.stbname}')
+
+    def update_check_error(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+        tdSql.execute(f'create database {self.dbname}')
+        tdSql.execute(f'use {self.dbname}')
+        self.error_check(self.ntbname,self.column_dict,'ntb')
+        self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname)
+
+    def run(self):
+        self.update_check()
+        self.update_check_error()
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/2-query/abs.py

@@ -555,6 +555,9 @@ class TDTestCase:
         tdSql.checkRows(1)
         tdSql.checkData(0,0,0)
         
+        tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)")
+        tdSql.checkRows(25)
+
         tdSql.query(
             "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
 

tests/system-test/2-query/csum.py

@@ -435,8 +435,8 @@ class TDTestCase:
         tdSql.checkRows(40)
 
         # # bug need fix
-        # tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
+        tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ")
-        # tdSql.checkRows(4)
+        tdSql.checkRows(4)
         # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ")
 
 

tests/system-test/2-query/max_partition.py

@@ -162,10 +162,45 @@ class TDTestCase:
         tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)")
         tdSql.checkRows(self.row_nums*2)
 
+        tdSql.query("select unique(c1) from stb partition  by tbname order by tbname")
+
         tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)")
         tdSql.checkData(0,0,'sub_stb_1')
         tdSql.checkData(0,1,self.row_nums)
 
+        tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1")
+        tdSql.checkRows(72)
+
+        tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1")
+        tdSql.checkRows(72)
+
+        tdSql.query("select c1 , csum(c1) from stb partition by c1")
+        tdSql.checkRows(80)
+
+        tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1")
+        tdSql.checkRows(21)
+        # bug need fix 
+        # tdSql.checkData(0,1,None)
+
+        tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0,1,0.000000000)
+
+        tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1")
+        tdSql.checkRows(11)
+        tdSql.checkData(0,1,None)
+
+        tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1")
+        tdSql.checkRows(72)
+        # bug need fix 
+        # tdSql.checkData(0,1,None)
+
+
+
+
+
+
+
         # bug need fix 
         # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ")  
         # tdSql.checkRows(5)

tests/system-test/2-query/sample.py

@@ -870,7 +870,10 @@ class TDTestCase:
         tdSql.query("select sample(c1 ,1000) from st")
         tdSql.checkRows(1000)
 
-
+        # bug need fix 
+        tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ")
+        tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ")
+        # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ")
 
     def run(self):
         import traceback

tests/system-test/7-tmq/stbTagFilter.py

@@ -5,103 +5,250 @@ import time
 import socket
 import os
 import threading
+from enum import Enum
 
 from util.log import *
 from util.sql import *
 from util.cases import *
 from util.dnodes import *
-from util.common import *
 sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
+    def __init__(self):
+        self.snapshot   = 0
+        self.vgroups    = 4
+        self.ctbNum     = 1
+        self.rowsPerTbl = 10000
+        
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), False)
-        #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
-    def tmqCase1(self):
+    def prepareTestEnv(self):
-        tdLog.printNoPrefix("======== test case 1: ")
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
-        paraDict = {'dbName':     'db2',
+        paraDict = {'dbName':     'dbt',
                     'dropFlag':   1,
                     'event':      '',
-                    'vgroups':    1,
+                    'vgroups':    4,
                     'stbName':    'stb',
                     'colPrefix':  'c',
                     'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
                     'ctbPrefix':  'ctb',
                     'ctbStartIdx': 0,
-                    'ctbNum':     10,
+                    'ctbNum':     1,
-                    'rowsPerTbl': 1000,
+                    'rowsPerTbl': 100000,
-                    'batchNum':   10,
+                    'batchNum':   1200,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  10,
+                    'pollDelay':  3,
                     'showMsg':    1,
-                    'showRow':    1}
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
         
-        topicNameList = ['topic1']
-        expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1)
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
         tdLog.info("create stb")
-        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
         tdLog.info("create ctb")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
         tdLog.info("insert data")
-        tmqCom.asyncInsertData(paraDict)
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
+        #                                       ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                       startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
         
-        tdLog.info("create topics from stb with filter")
+        # tdLog.info("restart taosd to ensure that the data falls into the disk")        
-        # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
+        # tdSql.query("flush database %s"%(paraDict['dbName']))
-        queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
+        return
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+   
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 100000,
+                    'batchNum':   3000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        # update to half tables
+        # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2)
+        # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx",
+        #                                       ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                       startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])     
+        
+        tdLog.info("create topics from stb1")
+        topicFromStb1 = 'topic_stb1'                
+        queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
+        # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
         tdLog.info("create topic sql: %s"%sqlString)
         tdSql.execute(sqlString)        
         
-        # start tmq consume processor
+        # paraDict['ctbNum'] = self.ctbNum
-        tdLog.info("insert consume info to consume processor")
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
         consumerId     = 0
-        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
-        topicList    = topicNameList[0]
+        topicList      = topicFromStb1
-        ifcheckdata  = 0
+        ifcheckdata    = 1
         ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest'
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
         tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
 
         tdLog.info("start consume processor")
-        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
 
-        # tmqCom.getStartCommitNotifyFromTmqsim()
+        tdLog.info("insert process end, and start to check consume result")
-        tmqCom.getStartConsumeNotifyFromTmqsim()
-        tdLog.info("create some new ctb")
-        paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum']
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data into new ctb")
-        pThread = tmqCom.asyncInsertData(paraDict)
-
-        pThread.join()
-        tdLog.info("wait insert end") 
-        tdSql.query(queryString)        
-        expectRowsList.append(tdSql.getRows())
-
-        tdLog.info("wait the consume result") 
         expectRows = 1
         resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
 
-        if expectRowsList[0] != resultList[0]:
+        tdLog.info("run select sql from db")
-            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+        tdSql.query(queryString)
-            tdLog.exit("0 tmq consume rows error!")
+        expectrowcnt = tdSql.getRows()
                 
-        time.sleep(10)        
+        tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
-        for i in range(len(topicNameList)):
+        if totalConsumeRows != expectrowcnt:
-            tdSql.query("drop topic %s"%topicNameList[i])
+            tdLog.exit("tmq consume rows error!")
             
+        tmqCom.checkFileContent(consumerId, queryString) 
+
+        tdSql.query("drop topic %s"%topicFromStb1)
         tdLog.printNoPrefix("======== test case 1 end ...... ")
 
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")  
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   5000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  5,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+        
+        paraDict['snapshot'] = self.snapshot
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        
+        tdLog.info("restart taosd to ensure that the data falls into the disk")        
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        
+        # update to half tables
+        paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2)
+        paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2)
+        tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                              ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                              startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])     
+
+        tmqCom.initConsumerTable()
+        tdLog.info("create topics from stb1")
+        topicFromStb1 = 'topic_stb1'                
+        queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicFromStb1, queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        
+        # paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+        consumerId     = 1
+        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        topicList      = topicFromStb1
+        ifcheckdata    = 1
+        ifManualCommit = 1
+        keyList        = 'group.id:cgrp1,\
+                        enable.auto.commit:true,\
+                        auto.commit.interval.ms:1000,\
+                        auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        
+        tdLog.info("insert process end, and start to check consume result")
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        totalConsumeRows = 0
+        for i in range(expectRows):
+            totalConsumeRows += resultList[i]
+
+        tdSql.query(queryString)
+        totalRowsInserted = tdSql.getRows()
+        
+        tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt))
+        
+        if totalConsumeRows != expectrowcnt:
+            tdLog.exit("tmq consume rows error!")
+            
+        # tmqCom.checkFileContent(consumerId, queryString)   
+
+        tdSql.query("drop topic %s"%topicFromStb1)
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
     def run(self):
         tdSql.prepare()
+        self.prepareTestEnv()
+        tdLog.printNoPrefix("=============================================")
+        tdLog.printNoPrefix("======== snapshot is 0: only consume from wal")
         self.tmqCase1()
+        # self.tmqCase2()
+        
+        # self.prepareTestEnv()
+        # tdLog.printNoPrefix("====================================================================")
+        # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal")
+        # self.snapshot = 1
+        # self.tmqCase1()
+        # self.tmqCase2()
+        
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqCommon.py

@@ -205,6 +205,13 @@ class TMQCom:
         tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
         return    
 
+    def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0):
+        for _ in range(count):
+            create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};'
+            ctbStartIdx += 1
+            tdLog.info("drop ctb sql: %s"%create_ctable_sql)
+            tsql.execute(create_ctable_sql)
+
     # schema: (ts timestamp, c1 int, c2 binary(16))
     def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
         tdLog.debug("start to insert data ............")

tests/system-test/7-tmq/tmqUpdate-1ctb.py

@@ -116,7 +116,12 @@ class TDTestCase:
         # paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         consumerId     = 0
-        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2)
+        
+        if self.snapshot == 0:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2))
+        elif self.snapshot == 1:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
+            
         topicList      = topicFromStb1
         ifcheckdata    = 1
         ifManualCommit = 1
@@ -199,7 +204,11 @@ class TDTestCase:
         # paraDict['ctbNum'] = self.ctbNum
         paraDict['rowsPerTbl'] = self.rowsPerTbl
         consumerId     = 1
-        expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2)
+        if self.snapshot == 0:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2))
+        elif self.snapshot == 1:
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1))
+        
         topicList      = topicFromStb1
         ifcheckdata    = 1
         ifManualCommit = 1

tests/system-test/99-TDcase/TD-17255.py

@@ -320,7 +320,7 @@ class TDTestCase:
         tdSql.prepare()
 
         self.tmqCase1()
-        # self.tmqCase2() 
+        self.tmqCase2() 
         self.tmqCase3()
 
     def stop(self):

tests/system-test/fulltest.sh

@@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py
 python3 ./test.py -f 0-others/udfTest.py
 python3 ./test.py -f 0-others/udf_create.py
 python3 ./test.py -f 0-others/udf_restart_taosd.py
-python3 ./test.py -f 0-others/cachelast.py
+python3 ./test.py -f 0-others/cachemodel.py
 python3 ./test.py -f 0-others/udf_cfg1.py
 python3 ./test.py -f 0-others/udf_cfg2.py
 
@@ -32,7 +32,10 @@ python3 ./test.py -f 1-insert/block_wise.py
 python3 ./test.py -f 1-insert/create_retentions.py
 python3 ./test.py -f 1-insert/table_param_ttl.py
 
+python3 ./test.py -f 1-insert/update_data.py
+
 python3 ./test.py -f 2-query/db.py
+
 python3 ./test.py -f 2-query/between.py
 python3 ./test.py -f 2-query/distinct.py
 python3 ./test.py -f 2-query/varchar.py
@@ -126,7 +129,7 @@ python3 ./test.py -f 2-query/count_partition.py
 python3 ./test.py -f 2-query/function_null.py
 python3 ./test.py -f 2-query/queryQnode.py
 python3 ./test.py -f 2-query/max_partition.py
-
+python3 ./test.py -f 2-query/last_row.py
 
 python3 ./test.py -f 6-cluster/5dnode1mnode.py
 #BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py  -N 5 -M 3
@@ -182,7 +185,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
 python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
 #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
-#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
+python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
 python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py
 #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 
@@ -274,6 +277,7 @@ python3 ./test.py -f 2-query/irate.py  -Q 2
 python3 ./test.py -f 2-query/function_null.py  -Q 2
 python3 ./test.py -f 2-query/count_partition.py -Q 2
 python3 ./test.py -f 2-query/max_partition.py -Q 2
+python3 ./test.py -f 2-query/last_row.py -Q 2
 
 #------------querPolicy  3-----------
 
@@ -361,3 +365,4 @@ python3 ./test.py -f 2-query/irate.py -Q  3
 python3 ./test.py -f 2-query/function_null.py -Q  3
 python3 ./test.py -f 2-query/count_partition.py -Q 3
 python3 ./test.py -f 2-query/max_partition.py -Q 3
+python3 ./test.py -f 2-query/last_row.py -Q 3

tests/test/c/tmqDemo.c

@@ -596,7 +596,8 @@ void printParaIntoFile() {
   g_fp = pFile;
 
   time_t    tTime = taosGetTimestampSec();
-  struct tm tm = *taosLocalTime(&tTime, NULL);
+  struct tm tm;
+  taosLocalTime(&tTime, &tm);
 
   taosFprintfFile(pFile, "###################################################################\n");
   taosFprintfFile(pFile, "# configDir:                %s\n", configDir);

tests/test/c/tmqSim.c

@@ -171,7 +171,8 @@ static void printHelp() {
 
 char* getCurrentTimeString(char* timeString) {
   time_t    tTime = taosGetTimestampSec();
-  struct tm tm = *taosLocalTime(&tTime, NULL);
+  struct tm tm;
+  taosLocalTime(&tTime, &tm);
   sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
           tm.tm_min, tm.tm_sec);
 
@@ -420,18 +421,6 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) {
     ms = val % 1000;
   }
 
-  /*
-    comment out as it make testcases like select_with_tags.sim fail.
-    but in windows, this may cause the call to localtime crash if tt < 0,
-    need to find a better solution.
-    if (tt < 0) {
-      tt = 0;
-    }
-  */
-
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -443,8 +432,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) {
     }
   }
 
-  struct tm* ptm = taosLocalTime(&tt, NULL);
+  struct tm ptm;
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  taosLocalTime(&tt, &ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);

tests/tsim/src/simExe.c

@@ -635,7 +635,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
 bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
   char       timeStr[30] = {0};
   time_t     tt;
-  struct tm *tp;
+  struct tm  tp;
   SCmdLine  *line = &script->lines[script->linePos];
   int32_t    ret = -1;
 
@@ -768,20 +768,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
               } else {
                 tt = (*(int64_t *)row[i]) / 1000000000;
               }
-              /* comment out as it make testcases like select_with_tags.sim fail.
-                but in windows, this may cause the call to localtime crash if tt < 0,
-                need to find a better solution.
-              if (tt < 0) {
-                tt = 0;
-              }
-              */
 
-#ifdef WINDOWS
+              taosLocalTime(&tt, &tp);
-              if (tt < 0) tt = 0;
+              strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp);
-#endif
-
-              tp = taosLocalTime(&tt, NULL);
-              strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp);
               if (precision == TSDB_TIME_PRECISION_MILLI) {
                 sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000));
               } else if (precision == TSDB_TIME_PRECISION_MICRO) {

tests/tsim/src/simParse.c

@@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) {
 SScript *simParseScript(char *fileName) {
   TdFilePtr pFile;
   int32_t   tokenLen, lineNum = 0;
-  char     *buffer = NULL, name[128], *token, *rest;
+  char      buffer[10*1024], name[128], *token, *rest;
   SCommand *pCmd;
   SScript  *script;
 
@@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) {
   simResetParser();
 
   while (!taosEOFFile(pFile)) {
-    if (taosGetLineFile(pFile, &buffer) == -1) continue;
+    if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue;
 
     lineNum++;
     int32_t cmdlen = (int32_t)strlen(buffer);
@@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) {
       return NULL;
     }
   }
-  if(buffer != NULL) taosMemoryFree(buffer);
   taosCloseFile(&pFile);
 
   script = simBuildScriptObj(fileName);

tools/shell/src/shellEngine.c

@@ -231,18 +231,6 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
     ms = val % 1000;
   }
 
-  /*
-    comment out as it make testcases like select_with_tags.sim fail.
-    but in windows, this may cause the call to localtime crash if tt < 0,
-    need to find a better solution.
-    if (tt < 0) {
-      tt = 0;
-    }
-  */
-
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -254,8 +242,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
     }
   }
 
-  struct tm *ptm = taosLocalTime(&tt, NULL);
+  struct tm ptm = {0};
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  taosLocalTime(&tt, &ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);
@@ -748,13 +737,6 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver
 
 void shellReadHistory() {
   SShellHistory *pHistory = &shell.history;
-  int64_t file_size;
-  if (taosStatFile(pHistory->file, &file_size, NULL) != 0) {
-    return;
-  } else if (file_size > SHELL_MAX_COMMAND_SIZE) {
-    taosRemoveFile(pHistory->file);
-    return;
-  }
   TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM);
   if (pFile == NULL) return;
 
@@ -774,10 +756,29 @@ void shellReadHistory() {
 
   if (line != NULL) taosMemoryFree(line);
   taosCloseFile(&pFile);
+  int64_t file_size;
+  if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
+    fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout);
+    TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
+    if (pFile == NULL) return;
+    int32_t endIndex = pHistory->hstart;
+    if (endIndex != 0) {
+      endIndex = pHistory->hend;
+    }
+    for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) {
+      taosFprintfFile(pFile, "%s\n", pHistory->hist[i]);
+      i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE;
+    }
+    taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]);
+    taosFsyncFile(pFile);
+    taosCloseFile(&pFile);
+  }
+  pHistory->hend = pHistory->hstart;
 }
 
 void shellWriteHistory() {
   SShellHistory *pHistory = &shell.history;
+  if (pHistory->hend == pHistory->hstart) return;
   TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
   if (pFile == NULL) return;
 

tools/taos-tools

@@ -1 +1 @@
-Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72
+Subproject commit 0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a