Merge remote-tracking branch 'origin/3.0' into fix/3.0_bugfix_wxy

.gitignore

@@ -101,6 +101,7 @@ tests/examples/JDBC/JDBCDemo/.settings/
 source/libs/parser/inc/sql.*
 tests/script/tmqResult.txt
 tests/tmqResult.txt
+tests/script/jenkins/basic.txt
 
 # Emacs
 # -*- mode: gitignore; -*-

Jenkinsfile2

@@ -303,7 +303,7 @@ def pre_test_build_win() {
         set CL=/MP8
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
         time /t
-        cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true || exit 7
+        cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
         time /t
         jom -j 6 || exit 8
@@ -361,7 +361,7 @@ pipeline {
             }
             parallel {
                 stage('check docs') {
-                    agent{label " worker03 || slave215 || slave217 || slave219 || Mac_catalina "}
+                    agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 || Mac_catalina "}
                     steps {
                         check_docs()
                     }
@@ -407,7 +407,7 @@ pipeline {
                     }
                 }
                 stage('linux test') {
-                    agent{label " worker03 || slave215 || slave217 || slave219 "}
+                    agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
                     options { skipDefaultCheckout() } 
                     when {
                         changeRequest()

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG e00ebd9
+        GIT_TAG fab042d
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/05-get-started/index.md

@@ -31,7 +31,7 @@ The TDengine Knowledge Map covers the various knowledge points of TDengine, reve
 </center>
 </figure>
 
-### Join TDengine Community
+## Join TDengine Community
 
 <table width="100%">
 <tr align="center" style={{border:0}}>

docs/en/12-taos-sql/06-select.md

@@ -301,7 +301,7 @@ SELECT TIMEZONE();
 ### Syntax
 
 ```txt
-WHERE (column|tbname) **match/MATCH/nmatch/NMATCH** _regex_
+WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_
 ```
 
 ### Specification

docs/examples/python/tmq_example.py

@@ -4,6 +4,7 @@ from taos.tmq import *
 conn = taos.connect()
 
 print("init")
+conn.execute("drop topic if exists topic_ctb_column")
 conn.execute("drop database if exists py_tmq")
 conn.execute("create database if not exists py_tmq vgroups 2")
 conn.select_db("py_tmq")
@@ -15,7 +16,6 @@ conn.execute("create table if not exists tb2 using stb1 tags(2)")
 conn.execute("create table if not exists tb3 using stb1 tags(3)")
 
 print("create topic")
-conn.execute("drop topic if exists topic_ctb_column")
 conn.execute(
     "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1"
 )

docs/zh/08-connector/10-cpp.mdx

@@ -72,6 +72,94 @@ TDengine 客户端驱动的安装请参考 [安装指南](../#安装步骤)
 
 ```c
 {{#include examples/c/demo.c}}
+```
+格式化输出不同类型字段函数 taos_print_row
+```c
+int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
+  int32_t len = 0;
+  for (int i = 0; i < num_fields; ++i) {
+    if (i > 0) {
+      str[len++] = ' ';
+    }
+
+    if (row[i] == NULL) {
+      len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR);
+      continue;
+    }
+
+    switch (fields[i].type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        len += sprintf(str + len, "%d", *((int8_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_UTINYINT:
+        len += sprintf(str + len, "%u", *((uint8_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_SMALLINT:
+        len += sprintf(str + len, "%d", *((int16_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_USMALLINT:
+        len += sprintf(str + len, "%u", *((uint16_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_INT:
+        len += sprintf(str + len, "%d", *((int32_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_UINT:
+        len += sprintf(str + len, "%u", *((uint32_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_BIGINT:
+        len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_UBIGINT:
+        len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_FLOAT: {
+        float fv = 0;
+        fv = GET_FLOAT_VAL(row[i]);
+        len += sprintf(str + len, "%f", fv);
+      } break;
+
+      case TSDB_DATA_TYPE_DOUBLE: {
+        double dv = 0;
+        dv = GET_DOUBLE_VAL(row[i]);
+        len += sprintf(str + len, "%lf", dv);
+      } break;
+
+      case TSDB_DATA_TYPE_BINARY:
+      case TSDB_DATA_TYPE_NCHAR: {
+        int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
+        if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
+          assert(charLen <= fields[i].bytes && charLen >= 0);
+        } else {
+          assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0);
+        }
+
+        memcpy(str + len, row[i], charLen);
+        len += charLen;
+      } break;
+
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
+        break;
+
+      case TSDB_DATA_TYPE_BOOL:
+        len += sprintf(str + len, "%d", *((int8_t *)row[i]));
+      default:
+        break;
+    }
+  }
+  str[len] = 0;
+
+  return len;
+}
+  
 ```  
   
 </details>

docs/zh/12-taos-sql/06-select.md

@@ -302,7 +302,7 @@ SELECT TIMEZONE();
 ### 语法
 
 ```txt
-WHERE (column|tbname) **match/MATCH/nmatch/NMATCH** _regex_
+WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_
 ```
 
 ### 正则表达式规范

include/common/tdatablock.h

@@ -90,6 +90,33 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData,
   }
 }
 
+static FORCE_INLINE bool colDataIsNNull_s(const SColumnInfoData* pColumnInfoData, int32_t startIndex,
+                                          uint32_t nRows) {
+  if (!pColumnInfoData->hasNull) {
+    return false;
+  }
+
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    for (int32_t i = startIndex; i < nRows; ++i) {
+      if (!colDataIsNull_var(pColumnInfoData, i)) {
+        return false;
+      }
+    }
+  } else {
+    if (pColumnInfoData->nullbitmap == NULL) {
+      return false;
+    }
+
+    for (int32_t i = startIndex; i < nRows; ++i) {
+      if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
 static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row,
                                        SColumnDataAgg* pColAgg) {
   if (!pColumnInfoData->hasNull) {

include/common/tdataformat.h

@@ -27,17 +27,17 @@
 extern "C" {
 #endif
 
-typedef struct SBuffer       SBuffer;
+typedef struct SBuffer  SBuffer;
-typedef struct SSchema       SSchema;
+typedef struct SSchema  SSchema;
-typedef struct STColumn      STColumn;
+typedef struct STColumn STColumn;
-typedef struct STSchema      STSchema;
+typedef struct STSchema STSchema;
-typedef struct SValue        SValue;
+typedef struct SValue   SValue;
-typedef struct SColVal       SColVal;
+typedef struct SColVal  SColVal;
-typedef struct STSRow2       STSRow2;
+typedef struct SRow     SRow;
-typedef struct STSRowBuilder STSRowBuilder;
+typedef struct SRowIter SRowIter;
-typedef struct STagVal       STagVal;
+typedef struct STagVal  STagVal;
-typedef struct STag          STag;
+typedef struct STag     STag;
-typedef struct SColData      SColData;
+typedef struct SColData SColData;
 
 #define HAS_NONE  ((uint8_t)0x1)
 #define HAS_NULL  ((uint8_t)0x2)
@@ -68,13 +68,10 @@ struct SBuffer {
 void    tBufferDestroy(SBuffer *pBuffer);
 int32_t tBufferInit(SBuffer *pBuffer, int64_t size);
 int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData);
+int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData);
 
 // STSchema ================================
-int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
+void tDestroyTSchema(STSchema *pTSchema);
-void    tTSchemaDestroy(STSchema *pTSchema);
-
-// SValue ================================
-static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
 
 // SColVal ================================
 #define CV_FLAG_VALUE ((int8_t)0x0)
@@ -89,26 +86,14 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
 #define COL_VAL_IS_NULL(CV)  ((CV)->flag == CV_FLAG_NULL)
 #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)
 
-// STSRow2 ================================
+// SRow ================================
-#define TSROW_LEN(PROW, V)  tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL)
+int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer);
-#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL)
+void    tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 
-int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow);
+// SRowIter ================================
-int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow);
+int32_t  tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
-void    tTSRowFree(STSRow2 *pRow);
+void     tRowIterClose(SRowIter **ppIter);
-void    tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+SColVal *tRowIterNext(SRowIter *pIter);
-int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray);
-int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
-int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
-
-// STSRowBuilder ================================
-#define tsRowBuilderInit() ((STSRowBuilder){0})
-#define tsRowBuilderClear(B)     \
-  do {                           \
-    if ((B)->pBuf) {             \
-      taosMemoryFree((B)->pBuf); \
-    }                            \
-  } while (0)
 
 // STag ================================
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
@@ -147,29 +132,17 @@ struct STSchema {
   int32_t  numOfCols;
   int32_t  version;
   int32_t  flen;
-  int32_t  vlen;
   int32_t  tlen;
   STColumn columns[];
 };
 
-#define TSROW_HAS_NONE ((uint8_t)0x1)
+struct SRow {
-#define TSROW_HAS_NULL ((uint8_t)0x2U)
+  uint8_t  flag;
-#define TSROW_HAS_VAL  ((uint8_t)0x4U)
+  uint8_t  rsv;
-#define TSROW_KV_SMALL ((uint8_t)0x10U)
+  uint16_t sver;
-#define TSROW_KV_MID   ((uint8_t)0x20U)
+  uint32_t len;
-#define TSROW_KV_BIG   ((uint8_t)0x40U)
+  TSKEY    ts;
-#pragma pack(push, 1)
+  uint8_t  data[];
-struct STSRow2 {
-  TSKEY   ts;
-  uint8_t flags;
-  uint8_t data[];
-};
-#pragma pack(pop)
-
-struct STSRowBuilder {
-  // STSRow2  tsRow;
-  int32_t  szBuf;
-  uint8_t *pBuf;
 };
 
 struct SValue {
@@ -258,37 +231,17 @@ typedef struct {
   int32_t      nCols;
   schema_ver_t version;
   uint16_t     flen;
-  int32_t      vlen;
   int32_t      tlen;
   STColumn    *columns;
 } STSchemaBuilder;
 
-// use 2 bits for bitmap(default: STSRow/sub block)
-#define TD_VTYPE_BITS        2
-#define TD_VTYPE_PARTS       4  // PARTITIONS: 1 byte / 2 bits
-#define TD_VTYPE_OPTR        3  // OPERATOR: 4 - 1, utilize to get remainder
-#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2)
-
-// use 1 bit for bitmap(super block)
-#define TD_VTYPE_BITS_I        1
-#define TD_VTYPE_PARTS_I       8  // PARTITIONS: 1 byte / 1 bit
-#define TD_VTYPE_OPTR_I        7  // OPERATOR: 8 - 1, utilize to get remainder
-#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3)
-
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
-static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
+STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version);
-  if (IS_VAR_DATA_TYPE(type)) {
-    return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL);
-  } else {
-    memcpy(&pValue->val, p, tDataTypes[type].bytes);
-    return tDataTypes[type].bytes;
-  }
-}
 
 #endif
 

include/common/tmsg.h

@@ -1629,7 +1629,6 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq);
 int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq);
 void tFreeSSubQueryMsg(SSubQueryMsg *pReq);
 
-
 typedef struct {
   SMsgHead header;
   uint64_t sId;
@@ -1667,6 +1666,10 @@ typedef struct {
   int32_t  execId;
 } SResFetchReq;
 
+int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq);
+int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq);
+
+
 typedef struct {
   SMsgHead header;
   uint64_t sId;
@@ -2948,6 +2951,10 @@ typedef struct {
   STqOffsetVal reqOffset;
 } SMqPollReq;
 
+int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq);
+int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq);
+
+
 typedef struct {
   int32_t vgId;
   int64_t offset;

include/common/trow.h

@@ -55,6 +55,14 @@ typedef struct STSRow {
 #define TD_ROW_TP 0x0U  // default
 #define TD_ROW_KV 0x01U
 
+#define TD_VTYPE_PARTS       4  // PARTITIONS: 1 byte / 2 bits
+#define TD_VTYPE_OPTR        3  // OPERATOR: 4 - 1, utilize to get remainder
+#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2)
+
+#define TD_VTYPE_PARTS_I       8  // PARTITIONS: 1 byte / 1 bit
+#define TD_VTYPE_OPTR_I        7  // OPERATOR: 8 - 1, utilize to get remainder
+#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3)
+
 /**
  * @brief value type
  *  - for data from client input and STSRow in memory, 3 types of value none/null/norm available
@@ -244,7 +252,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal
  */
 static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) {
   // The primary TS key is stored separatedly.
-  return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY));
+  return POINTER_SHIFT(TD_ROW_DATA(pRow), flen);
   // return POINTER_SHIFT(pRow->ts, flen);
 }
 

include/libs/sync/sync.h

@@ -41,6 +41,7 @@ extern "C" {
 #define SNAPSHOT_WAIT_MS             1000 * 30
 
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
+#define SYNC_HEART_TIMEOUT_MS          1000 * 8
 
 #define SYNC_MAX_BATCH_SIZE 1
 #define SYNC_INDEX_BEGIN    0

packaging/release.sh

@@ -221,12 +221,12 @@ if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" =
     # community-version compile
     cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
   elif [ "$verMode" == "cloud" ]; then
-    cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
+    cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
   elif [ "$verMode" == "cluster" ]; then
     if [[ "$dbName" != "taos" ]]; then
       replace_enterprise_$dbName
     fi
-    cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
+    cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro}
   fi
 else
   echo "input cpuType=${cpuType} error!!!"

packaging/tools/install.sh

@@ -213,7 +213,7 @@ function install_bin() {
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
   [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
-  [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || :
+  [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || :
   [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
   [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

source/client/src/clientMain.c

@@ -1106,6 +1106,8 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId
     return terrno;
   }
 
+  pRequest->syncQuery = true;
+
   STscObj  *pTscObj = pRequest->pTscObj;
   SCatalog *pCtg = NULL;
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);

source/client/src/clientRawBlockWrite.c

@@ -1275,6 +1275,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
       nVar++;
     }
   }
+  fLen -= sizeof(TSKEY);
 
   int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
                             (int32_t)TD_BITMAP_BYTES(numOfCols - 1);
@@ -1333,7 +1334,9 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
         }
       }
 
-      offset += TYPE_BYTES[pColumn->type];
+      if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
+        offset += TYPE_BYTES[pColumn->type];
+      }
     }
     tdSRowEnd(&rb);
     int32_t rowLen = TD_ROW_LEN(rowData);
@@ -1503,6 +1506,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
         nVar++;
       }
     }
+    fLen -= sizeof(TSKEY);
 
     int32_t rows = rspObj.resInfo.numOfRows;
     int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
@@ -1585,8 +1589,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
             tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k);
           }
         }
-
+        if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-        offset += TYPE_BYTES[pColumn->type];
+          offset += TYPE_BYTES[pColumn->type];
+        }
       }
       tdSRowEnd(&rb);
       int32_t rowLen = TD_ROW_LEN(rowData);
@@ -1803,6 +1808,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
         nVar++;
       }
     }
+    fLen -= sizeof(TSKEY);
 
     int32_t rows = rspObj.resInfo.numOfRows;
     int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
@@ -1888,8 +1894,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
             tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k);
           }
         }
-
+        if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-        offset += TYPE_BYTES[pColumn->type];
+          offset += TYPE_BYTES[pColumn->type];
+        }
       }
       tdSRowEnd(&rb);
       int32_t rowLen = TD_ROW_LEN(rowData);

source/client/src/clientTmq.c

@@ -1461,12 +1461,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) {
   return code;
 }
 
-SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+void tmqBuildConsumeReqImpl(SMqPollReq *pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
-  SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq));
-  if (pReq == NULL) {
-    return NULL;
-  }
-
   /*strcpy(pReq->topic, pTopic->topicName);*/
   /*strcpy(pReq->cgroup, tmq->groupId);*/
 
@@ -1485,9 +1480,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
 
   pReq->useSnapshot = tmq->useSnapshot;
 
-  pReq->head.vgId = htonl(pVg->vgId);
+  pReq->head.vgId = pVg->vgId;
-  pReq->head.contLen = htonl(sizeof(SMqPollReq));
-  return pReq;
 }
 
 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
@@ -1559,15 +1552,32 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 #endif
       }
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, timeout, pTopic, pVg);
+
-      if (pReq == NULL) {
+      SMqPollReq req = {0};
+      tmqBuildConsumeReqImpl(&req, tmq, timeout, pTopic, pVg);
+      int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
+      if (msgSize < 0) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         tsem_post(&tmq->rspSem);
         return -1;
       }
+      char *msg = taosMemoryCalloc(1, msgSize);
+      if (NULL == msg) {
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        tsem_post(&tmq->rspSem);
+        return -1;
+      }
+      
+      if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
+        taosMemoryFree(msg);
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        tsem_post(&tmq->rspSem);
+        return -1;
+      }
+      
       SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
       if (pParam == NULL) {
-        taosMemoryFree(pReq);
+        taosMemoryFree(msg);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         tsem_post(&tmq->rspSem);
         return -1;
@@ -1581,7 +1591,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 
       SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
       if (sendInfo == NULL) {
-        taosMemoryFree(pReq);
+        taosMemoryFree(msg);
         taosMemoryFree(pParam);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         tsem_post(&tmq->rspSem);
@@ -1589,11 +1599,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       }
 
       sendInfo->msgInfo = (SDataBuf){
-          .pData = pReq,
+          .pData = msg,
-          .len = sizeof(SMqPollReq),
+          .len = msgSize,
           .handle = NULL,
       };
-      sendInfo->requestId = pReq->reqId;
+      sendInfo->requestId = req.reqId;
       sendInfo->requestObjRefId = 0;
       sendInfo->param = pParam;
       sendInfo->fp = tmqPollCb;
@@ -1605,7 +1615,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       char offsetFormatBuf[80];
       tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset);
       tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64,
-               tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
+               tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
       /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;

source/client/test/clientTests.cpp

@@ -692,7 +692,6 @@ TEST(testCase, insert_test) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
-#endif
 
 TEST(testCase, projection_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -752,9 +751,6 @@ TEST(testCase, projection_query_tables) {
   taos_close(pConn);
 }
 
-
-#if 0
-
 TEST(testCase, tsbs_perf_test) {
   TdThread qid[20] = {0};
 
@@ -764,15 +760,16 @@ TEST(testCase, tsbs_perf_test) {
   getchar();
 }
 
+#endif
 
 TEST(testCase, projection_query_stables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+  TAOS_RES* pRes = taos_query(pConn, "use test");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select ts from st1");
+  pRes = taos_query(pConn, "select * from meters limit 50000000");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);
@@ -785,14 +782,15 @@ TEST(testCase, projection_query_stables) {
 
   char str[512] = {0};
   while ((pRow = taos_fetch_row(pRes)) != NULL) {
-    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-    printf("%s\n", str);
+//    printf("%s\n", str);
   }
 
   taos_free_result(pRes);
   taos_close(pConn);
 }
 
+#if 0
 TEST(testCase, agg_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);

source/common/src/tdatablock.c

@@ -36,7 +36,7 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows;
   } else {
-    return pColumnInfoData->info.bytes * numOfRows + BitmapLen(numOfRows);
+    return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows);
   }
 }
 
@@ -652,7 +652,10 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
       ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen);
     }
 
-    memcpy(pCol->pData, pStart, colLength);
+    if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) {
+      memcpy(pCol->pData, pStart, colLength);
+    }
+
     pStart += pCol->info.bytes * capacity;
   }
 
@@ -1084,8 +1087,6 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) {
   return 0;
 }
 
-int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; }
-
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) {
   // Allocate the additional buffer.
   int64_t p0 = taosGetTimestampUs();
@@ -1962,6 +1963,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
           memset(pBuf, 0, sizeof(pBuf));
           char*   pData = colDataGetVarData(pColInfoData, j);
           int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData));
+          dataSize = TMIN(dataSize, 50);
           memcpy(pBuf, varDataVal(pData), dataSize);
           len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf);
           if (len >= size - 1) return dumpBuf;
@@ -2053,6 +2055,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB
               isStartKey = true;
               tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true,
                                   offset, k);
+              continue; // offset should keep 0 for next column
 
             } else if (colDataIsNull_s(pColInfoData, j)) {
               tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL,

source/common/src/tmsg.c

@@ -4723,6 +4723,139 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) {
   taosMemoryFreeClear(pReq->msg);
 }
 
+
+int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+  if (buf != NULL) {
+    buf = (char *)buf + headLen;
+    bufLen -= headLen;
+  }
+
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeU64(&encoder, pReq->sId) < 0) return -1;
+  if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1;
+  if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->execId) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+
+  if (buf != NULL) {
+    SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen);
+    pHead->vgId = htonl(pReq->header.vgId);
+    pHead->contLen = htonl(tlen + headLen);
+  }
+
+  return tlen + headLen;
+}
+
+int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SMsgHead *pHead = buf;
+  pHead->vgId = pReq->header.vgId;
+  pHead->contLen = pReq->header.contLen;
+
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1;
+  if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1;
+  if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1;
+  
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+
+int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) {
+  if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1;
+  if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1;
+  if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1;
+
+  return 0;
+}
+
+int32_t tDerializeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffset) {
+  if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1;
+
+  return 0;
+}
+
+int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+  if (buf != NULL) {
+    buf = (char *)buf + headLen;
+    bufLen -= headLen;
+  }
+
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeCStr(&encoder, pReq->subKey) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->useSnapshot) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->epoch) < 0) return -1;
+  if (tEncodeU64(&encoder, pReq->reqId) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1;
+  if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+
+  if (buf != NULL) {
+    SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen);
+    pHead->vgId = htonl(pReq->head.vgId);
+    pHead->contLen = htonl(tlen + headLen);
+  }
+
+  return tlen + headLen;
+}
+
+int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SMsgHead *pHead = buf;
+  pHead->vgId = pReq->head.vgId;
+  pHead->contLen = pReq->head.contLen;
+
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeCStrTo(&decoder, pReq->subKey) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->useSnapshot) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1;
+  if (tDecodeU64(&decoder, &pReq->reqId) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1;
+  if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1;
+  
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+
 int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
   if (buf != NULL) {

source/common/src/trow.c

@@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl
     return true;
   }
   void *pBitmap = tdGetBitmapAddrTp(pRow, flen);
-  tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx);
+  tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx);
   return true;
 }
 
@@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC
         return false;
       }
     }
-    tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal);
     ++pIter->colIdx;
   } else if (TD_IS_KV_ROW(pIter->pRow)) {
     return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal);
@@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) {
   }
 
   if (TD_IS_TP_ROW(pIter->pRow)) {
-    tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal);
   } else if (TD_IS_KV_ROW(pIter->pRow)) {
     tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal);
   } else {
@@ -412,7 +412,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
         valType = TD_VTYPE_NULL;
       } else if (IS_VAR_DATA_TYPE(pTColumn->type)) {
         varDataSetLen(varBuf, pColVal->value.nData);
-        memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
+        if (pColVal->value.nData != 0) {
+          memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
+        }
         val = varBuf;
       } else {
         val = (const void *)&pColVal->value.val;
@@ -467,7 +469,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell
 #ifdef TD_SUPPORT_BITMAP
     colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn);
 #endif
-    tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1);
+    tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1);
   } else if (TD_IS_KV_ROW(pRow)) {
     SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx),
                                                compareKvRowColId, TD_EQ);
@@ -755,11 +757,10 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
 
 int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
                               int8_t colType, int16_t colIdx, int32_t offset) {
-  if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) {
+  if (colIdx < 1) {
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
-  offset -= sizeof(TSKEY);
   --colIdx;
 
 #ifdef TD_SUPPORT_BITMAP
@@ -851,7 +852,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
       memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
 #endif
       // the primary TS key is stored separatedly
-      len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps;
+      len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps;
       TD_ROW_SET_LEN(pBuilder->pBuf, len);
       TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
       break;

source/common/src/ttypes.c

@@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
 static float  floatMin = -FLT_MAX, floatMax = FLT_MAX;
 static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;
 
-FORCE_INLINE void *getDataMin(int32_t type, void* value) {
+FORCE_INLINE void *getDataMin(int32_t type, void *value) {
   switch (type) {
     case TSDB_DATA_TYPE_FLOAT:
       *(float *)value = floatMin;
@@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void* value) {
   return value;
 }
 
-FORCE_INLINE void *getDataMax(int32_t type, void* value) {
+FORCE_INLINE void *getDataMax(int32_t type, void *value) {
   switch (type) {
     case TSDB_DATA_TYPE_FLOAT:
       *(float *)value = floatMax;

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -196,7 +196,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
 
   code = 0;
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -464,7 +464,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
 
   code = 0;
 

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -148,8 +148,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pHead->vgId, pMsg, terrstr(),
+    dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(),
-            TMSG_INFO(pMsg->msgType), qtype);
+            TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
     return terrno != 0 ? terrno : -1;
   }
 

source/dnode/vnode/src/inc/tsdb.h

@@ -56,7 +56,7 @@ typedef struct SDataFWriter     SDataFWriter;
 typedef struct SDataFReader     SDataFReader;
 typedef struct SDelFWriter      SDelFWriter;
 typedef struct SDelFReader      SDelFReader;
-typedef struct SRowIter         SRowIter;
+typedef struct STSDBRowIter     STSDBRowIter;
 typedef struct STsdbFS          STsdbFS;
 typedef struct SRowMerger       SRowMerger;
 typedef struct STsdbReadSnap    STsdbReadSnap;
@@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
 void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
 int32_t tsdbRowCmprFn(const void *p1, const void *p2);
-// SRowIter
+// STSDBRowIter
-void     tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
+void     tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
-SColVal *tRowIterNext(SRowIter *pIter);
+SColVal *tsdbRowIterNext(STSDBRowIter *pIter);
 // SRowMerger
 int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
 int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
@@ -562,7 +562,7 @@ struct SDFileSet {
   SSttFile  *aSttF[TSDB_MAX_STT_TRIGGER];
 };
 
-struct SRowIter {
+struct STSDBRowIter {
   TSDBROW  *pRow;
   STSchema *pTSchema;
   SColVal   colVal;

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -203,10 +203,13 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
     goto _err;
   }
 
-  SBatchDeleteReq deleteReq;
+  SBatchDeleteReq deleteReq = {0};
   SSubmitReq     *pSubmitReq =
       tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true,
                       pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq);
+  // TODO deleteReq
+  taosArrayDestroy(deleteReq.deleteReqs);
+  
 
   if (!pSubmitReq) {
     smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),

source/dnode/vnode/src/tq/tq.c

@@ -458,20 +458,26 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
 }
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
-  SMqPollReq*  pReq = pMsg->pCont;
+  SMqPollReq   req = {0};
-  int64_t      consumerId = pReq->consumerId;
-  int32_t      reqEpoch = pReq->epoch;
   int32_t      code = 0;
-  STqOffsetVal reqOffset = pReq->reqOffset;
   STqOffsetVal fetchOffsetNew;
   SWalCkHead*  pCkHead = NULL;
 
+  if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
+    tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
+    return -1;
+  }
+
+  int64_t      consumerId = req.consumerId;
+  int32_t      reqEpoch = req.epoch;
+  STqOffsetVal reqOffset = req.reqOffset;
+
   // 1.find handle
-  STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey));
+  STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
   /*ASSERT(pHandle);*/
   if (pHandle == NULL) {
     tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId,
-            TD_VID(pTq->pVnode), pReq->subKey);
+            TD_VID(pTq->pVnode), req.subKey);
     return -1;
   }
 
@@ -479,7 +485,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   if (pHandle->consumerId != consumerId) {
     tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64
             ", in vgId:%d, subkey %s, handle consumer id %" PRId64,
-            consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
+            consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
     terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
     return -1;
   }
@@ -493,13 +499,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   char buf[80];
   tFormatOffset(buf, 80, &reqOffset);
   tqDebug("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
-          pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
+          req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
 
   // 2.reset offset if needed
   if (reqOffset.type > 0) {
     fetchOffsetNew = reqOffset;
   } else {
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
     if (pOffset != NULL) {
       fetchOffsetNew = pOffset->val;
       char formatBuf[80];
@@ -508,7 +514,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
               TD_VID(pTq->pVnode), formatBuf);
     } else {
       if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
-        if (pReq->useSnapshot) {
+        if (req.useSnapshot) {
           if (pHandle->fetchMeta) {
             tqOffsetResetToMeta(&fetchOffsetNew, 0);
           } else {
@@ -520,21 +526,21 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
         if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
           SMqDataRsp dataRsp = {0};
-          tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+          tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType);
 
           tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
           tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId,
                   pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
-          if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+          if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) {
             code = -1;
           }
           tDeleteSMqDataRsp(&dataRsp);
           return code;
         } else {
           STaosxRsp taosxRsp = {0};
-          tqInitTaosxRsp(&taosxRsp, pReq);
+          tqInitTaosxRsp(&taosxRsp, &req);
           tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
-          if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
+          if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
             code = -1;
           }
           tDeleteSTaosxRsp(&taosxRsp);
@@ -543,7 +549,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
         tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
                 " in vg %d, subkey %s, reset none failed",
-                pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
+                pHandle->subKey, consumerId, TD_VID(pTq->pVnode), req.subKey);
         terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
         return -1;
       }
@@ -552,7 +558,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
 
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
     SMqDataRsp dataRsp = {0};
-    tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+    tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType);
     // lock
     taosWLockLatch(&pTq->pushLock);
     tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew);
@@ -580,7 +586,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
 #endif
     taosWUnLockLatch(&pTq->pushLock);
 
-    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+    if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) {
       code = -1;
     }
 
@@ -599,13 +605,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   SMqMetaRsp metaRsp = {0};
 
   STaosxRsp taosxRsp = {0};
-  tqInitTaosxRsp(&taosxRsp, pReq);
+  tqInitTaosxRsp(&taosxRsp, &req);
 
   if (fetchOffsetNew.type != TMQ_OFFSET__LOG) {
     tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew);
 
     if (metaRsp.metaRspLen > 0) {
-      if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
+      if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) {
         code = -1;
       }
       tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, send meta offset type:%d,uid:%" PRId64
@@ -618,7 +624,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
     }
 
     if (taosxRsp.blockNum > 0) {
-      if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
+      if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
         code = -1;
       }
       tDeleteSTaosxRsp(&taosxRsp);
@@ -648,13 +654,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       if (consumerEpoch > reqEpoch) {
         tqWarn("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, vg %d offset %" PRId64
                ", found new consumer epoch %d, discard req epoch %d",
-               consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
+               consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
         break;
       }
 
       if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) {
         tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-        if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
+        if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
           code = -1;
         }
         tDeleteSTaosxRsp(&taosxRsp);
@@ -665,7 +671,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       SWalCont* pHead = &pCkHead->head;
 
       tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
-              pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
+              req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
 
       if (pHead->msgType == TDMT_VND_SUBMIT) {
         SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
@@ -674,7 +680,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         }
         if (taosxRsp.blockNum > 0 /* threshold */) {
           tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-          if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
+          if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
             code = -1;
           }
           tDeleteSTaosxRsp(&taosxRsp);
@@ -692,7 +698,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         metaRsp.resMsgType = pHead->msgType;
         metaRsp.metaRspLen = pHead->bodyLen;
         metaRsp.metaRsp = pHead->body;
-        if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
+        if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) {
           code = -1;
           taosMemoryFreeClear(pCkHead);
           tDeleteSTaosxRsp(&taosxRsp);
@@ -1269,6 +1275,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
         qError("stream task input del failed, task id %d", pTask->taskId);
 
+        atomic_sub_fetch_32(pRef, 1);
         taosFreeQitem(pRefBlock);
         continue;
       }
@@ -1286,7 +1293,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   int32_t ref = atomic_sub_fetch_32(pRef, 1);
   ASSERT(ref >= 0);
   if (ref == 0) {
-    taosMemoryFree(pDelBlock);
+    blockDataDestroy(pDelBlock);
     taosMemoryFree(pRef);
   }
 

source/dnode/vnode/src/tq/tqPush.c

@@ -308,9 +308,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   }
 
   if (vnodeIsRoleLeader(pTq->pVnode)) {
+    if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0;
     if (msgType == TDMT_VND_SUBMIT) {
-      if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0;
-
       void* data = taosMemoryMalloc(msgLen);
       if (data == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/src/tq/tqSink.c

@@ -236,6 +236,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
     SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
     if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
       pDeleteReq->suid = suid;
+      pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
       tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq);
       continue;
     }

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -287,12 +287,17 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
               hasRes = true;
               p->ts = pColVal->ts;
 
-              uint8_t* px = p->colVal.value.pData;
+              if (!IS_VAR_DATA_TYPE(pColVal->colVal.type)) {
-              p->colVal = pColVal->colVal;
+                p->colVal = pColVal->colVal;
+              } else {
+                if (COL_VAL_IS_VALUE(&pColVal->colVal)) {
+                  memcpy(p->colVal.value.pData, pColVal->colVal.value.pData, pColVal->colVal.value.nData);
+                }
 
-              if (COL_VAL_IS_VALUE(&pColVal->colVal) && IS_VAR_DATA_TYPE(pColVal->colVal.type)) {
+                p->colVal.value.nData = pColVal->colVal.value.nData;
-                p->colVal.value.pData = px;
+                p->colVal.type = pColVal->colVal.type;
-                memcpy(px, pColVal->colVal.value.pData, pColVal->colVal.value.nData);
+                p->colVal.flag = pColVal->colVal.flag;
+                p->colVal.cid = pColVal->colVal.cid;
               }
             }
           }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo
 
   pSkmInfo->suid = suid;
   pSkmInfo->uid = uid;
-  tTSchemaDestroy(pSkmInfo->pTSchema);
+  tDestroyTSchema(pSkmInfo->pTSchema);
   code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema);
   TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid
 
   pCommitter->skmRow.suid = suid;
   pCommitter->skmRow.uid = uid;
-  tTSchemaDestroy(pCommitter->skmRow.pTSchema);
+  tDestroyTSchema(pCommitter->skmRow.pTSchema);
   code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema);
   TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -623,7 +623,8 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa
 
 _exit:
   if (code) {
-    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino,
+              tstrerror(code));
   }
   return code;
 }
@@ -666,7 +667,8 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray
 
 _exit:
   if (code) {
-    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino,
+              tstrerror(code));
   }
   return code;
 }
@@ -706,7 +708,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde
 
 _exit:
   if (code) {
-    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino,
+              tstrerror(code));
   }
   return code;
 }
@@ -919,8 +922,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
 #else
   tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
 #endif
-  tTSchemaDestroy(pCommitter->skmTable.pTSchema);
+  tDestroyTSchema(pCommitter->skmTable.pTSchema);
-  tTSchemaDestroy(pCommitter->skmRow.pTSchema);
+  tDestroyTSchema(pCommitter->skmRow.pTSchema);
 }
 
 static int32_t tsdbCommitData(SCommitter *pCommitter) {

source/dnode/vnode/src/tsdb/tsdbDiskData.c

@@ -595,21 +595,21 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS
   if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts;
   if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts;
 
-  SRowIter iter = {0};
+  STSDBRowIter iter = {0};
-  tRowIterInit(&iter, pRow, pTSchema);
+  tsdbRowIterInit(&iter, pRow, pTSchema);
 
-  SColVal *pColVal = tRowIterNext(&iter);
+  SColVal *pColVal = tsdbRowIterNext(&iter);
   for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
     SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
 
     while (pColVal && pColVal->cid < pDCBuilder->cid) {
-      pColVal = tRowIterNext(&iter);
+      pColVal = tsdbRowIterNext(&iter);
     }
 
     if (pColVal && pColVal->cid == pDCBuilder->cid) {
       code = tDiskColAddVal(pDCBuilder, pColVal);
       if (code) return code;
-      pColVal = tRowIterNext(&iter);
+      pColVal = tsdbRowIterNext(&iter);
     } else {
       code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type));
       if (code) return code;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -305,7 +305,7 @@ static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) {
 }
 
 // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
-static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableKeyInfo* idList, int32_t numOfTables) {
+static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t numOfTables) {
   // allocate buffer in order to load data blocks from file
   // todo use simple hash instead, optimize the memory consumption
   SHashObj* pTableMap =
@@ -315,10 +315,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
   }
 
   int64_t st = taosGetTimestampUs();
-  initBlockScanInfoBuf(&pTsdbReader->blockInfoBuf, numOfTables);
+  initBlockScanInfoBuf(pBuf, numOfTables);
 
   for (int32_t j = 0; j < numOfTables; ++j) {
-    STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(&pTsdbReader->blockInfoBuf, j);
+    STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j);
     pScanInfo->uid = idList[j].uid;
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
       int64_t skey = pTsdbReader->window.skey;
@@ -2432,7 +2432,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
     TSDBKEY   keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
 
     // it is a clean block, load it directly
-    if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
+    if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && 
+        pBlock->nRow <= pReader->capacity) {
       if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) {
         copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
 
@@ -3785,9 +3786,9 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
   }
 
   STsdbReader* p = (pReader->innerReader[0] != NULL)? pReader->innerReader[0]:pReader;
-  pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables);
+  pReader->status.pTableMap = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, numOfTables);
   if (pReader->status.pTableMap == NULL) {
-    tsdbReaderClose(pReader);
+    tsdbReaderClose(p);
     *ppReader = NULL;
 
     code = TSDB_CODE_TDB_OUT_OF_MEMORY;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   }
 
   tBlockDataDestroy(&pReader->bData, 1);
-  tTSchemaDestroy(pReader->skmTable.pTSchema);
+  tDestroyTSchema(pReader->skmTable.pTSchema);
 
   // del
   if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader);
@@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
   taosArrayDestroy(pWriter->dReader.aBlockIdx);
 
   tBlockDataDestroy(&pWriter->bData, 1);
-  tTSchemaDestroy(pWriter->skmTable.pTSchema);
+  tDestroyTSchema(pWriter->skmTable.pTSchema);
 
   for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {
     tFree(pWriter->aBuf[iBuf]);

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) {
   return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2));
 }
 
-// SRowIter ======================================================
+// STSDBRowIter ======================================================
-void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
+void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
   pIter->pRow = pRow;
   if (pRow->type == 0) {
     ASSERT(pTSchema);
@@ -594,7 +594,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
   }
 }
 
-SColVal *tRowIterNext(SRowIter *pIter) {
+SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
   if (pIter->pRow->type == 0) {
     if (pIter->i < pIter->pTSchema->numOfCols) {
       tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal);
@@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch
           cv.flag = CV_FLAG_VALUE;
 
           if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-            void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY));
+            void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset);
             cv.value.nData = varDataLen(pData);
             cv.value.pData = varDataVal(pData);
           } else {
-            memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes);
+            memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes);
           }
 
           code = tColDataAppendValue(pColData, &cv);
@@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch
         cv.flag = CV_FLAG_VALUE;
 
         if (IS_VAR_DATA_TYPE(pTColumn->type)) {
-          void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY));
+          void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset);
           cv.value.nData = varDataLen(pData);
           cv.value.pData = varDataVal(pData);
         } else {
-          memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes);
+          memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes);
         }
 
         code = tColDataAppendValue(pColData, &cv);

source/libs/executor/CMakeLists.txt

@@ -2,10 +2,6 @@ aux_source_directory(src EXECUTOR_SRC)
 #add_library(executor ${EXECUTOR_SRC})
 
 add_library(executor STATIC ${EXECUTOR_SRC})
-#set_target_properties(executor PROPERTIES
-#        IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a"
-#        INTERFACE_INCLUDE_DIRECTORIES "${TD_SOURCE_DIR}/include/libs/executor"
-#        )
 
 target_link_libraries(executor
         PRIVATE os util common function parser planner qcom vnode scalar nodes index stream

source/libs/executor/inc/executorimpl.h

@@ -235,16 +235,6 @@ typedef enum {
 #define COL_MATCH_FROM_COL_ID  0x1
 #define COL_MATCH_FROM_SLOT_ID 0x2
 
-typedef struct SSourceDataInfo {
-  int32_t            index;
-  SRetrieveTableRsp* pRsp;
-  uint64_t           totalRows;
-  int64_t            startTime;
-  int32_t            code;
-  EX_SOURCE_STATUS   status;
-  const char*        taskId;
-} SSourceDataInfo;
-
 typedef struct SLoadRemoteDataInfo {
   uint64_t totalSize;     // total load bytes from remote
   uint64_t totalRows;     // total number of rows
@@ -371,23 +361,8 @@ typedef struct STagScanInfo {
   SColMatchInfo   matchInfo;
   int32_t         curPos;
   SReadHandle     readHandle;
-  STableListInfo* pTableList;
 } STagScanInfo;
 
-typedef struct SLastrowScanInfo {
-  SSDataBlock*    pRes;
-  SReadHandle     readHandle;
-  void*           pLastrowReader;
-  SColMatchInfo   matchInfo;
-  int32_t*        pSlotIds;
-  SExprSupp       pseudoExprSup;
-  int32_t         retrieveType;
-  int32_t         currentGroupIndex;
-  SSDataBlock*    pBufferredRes;
-  SArray*         pUidList;
-  int32_t         indexOfBufferedRes;
-} SLastrowScanInfo;
-
 typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_READERHANDLE = 1,
   STREAM_SCAN_FROM_RES,
@@ -504,40 +479,6 @@ typedef struct {
   SSnapContext* sContext;
 } SStreamRawScanInfo;
 
-typedef struct SSysTableIndex {
-  int8_t  init;
-  SArray* uids;
-  int32_t lastIdx;
-} SSysTableIndex;
-
-typedef struct SSysTableScanInfo {
-  SRetrieveMetaTableRsp* pRsp;
-  SRetrieveTableReq      req;
-  SEpSet                 epSet;
-  tsem_t                 ready;
-  SReadHandle            readHandle;
-  int32_t                accountId;
-  const char*            pUser;
-  bool                   sysInfo;
-  bool                   showRewrite;
-  SNode*                 pCondition;  // db_name filter condition, to discard data that are not in current database
-  SMTbCursor*            pCur;        // cursor for iterate the local table meta store.
-  SSysTableIndex*        pIdx;        // idx for local table meta
-  SColMatchInfo          matchInfo;
-  SName                  name;
-  SSDataBlock*           pRes;
-  int64_t                numOfBlocks;  // extract basic running information.
-  SLoadRemoteDataInfo    loadInfo;
-} SSysTableScanInfo;
-
-typedef struct SBlockDistInfo {
-  SSDataBlock* pResBlock;
-  STsdbReader* pHandle;
-  SReadHandle  readHandle;
-  uint64_t     uid;  // table uid
-} SBlockDistInfo;
-
-// todo remove this
 typedef struct SOptrBasicInfo {
   SResultRowInfo resultRowInfo;
   SSDataBlock*   pRes;
@@ -603,24 +544,6 @@ typedef struct SAggOperatorInfo {
   SExprSupp        scalarExprSup;
 } SAggOperatorInfo;
 
-typedef struct SProjectOperatorInfo {
-  SOptrBasicInfo binfo;
-  SAggSupporter  aggSup;
-  SArray*        pPseudoColInfo;
-  SLimitInfo     limitInfo;
-  bool           mergeDataBlocks;
-  SSDataBlock*   pFinalRes;
-} SProjectOperatorInfo;
-
-typedef struct SIndefOperatorInfo {
-  SOptrBasicInfo binfo;
-  SAggSupporter  aggSup;
-  SArray*        pPseudoColInfo;
-  SExprSupp      scalarSup;
-  uint64_t       groupId;
-  SSDataBlock*   pNextGroupRes;
-} SIndefOperatorInfo;
-
 typedef struct SFillOperatorInfo {
   struct SFillInfo* pFillInfo;
   SSDataBlock*      pRes;
@@ -638,42 +561,12 @@ typedef struct SFillOperatorInfo {
   SExprSupp         noFillExprSupp;
 } SFillOperatorInfo;
 
-typedef struct SGroupbyOperatorInfo {
-  SOptrBasicInfo binfo;
-  SAggSupporter  aggSup;
-  SArray*        pGroupCols;     // group by columns, SArray<SColumn>
-  SArray*        pGroupColVals;  // current group column values, SArray<SGroupKeys>
-  bool           isInit;         // denote if current val is initialized or not
-  char*          keyBuf;         // group by keys for hash
-  int32_t        groupKeyLen;    // total group by column width
-  SGroupResInfo  groupResInfo;
-  SExprSupp      scalarSup;
-} SGroupbyOperatorInfo;
-
 typedef struct SDataGroupInfo {
   uint64_t groupId;
   int64_t  numOfRows;
   SArray*  pPageList;
 } SDataGroupInfo;
 
-// The sort in partition may be needed later.
-typedef struct SPartitionOperatorInfo {
-  SOptrBasicInfo binfo;
-  SArray*        pGroupCols;
-  SArray*        pGroupColVals;  // current group column values, SArray<SGroupKeys>
-  char*          keyBuf;         // group by keys for hash
-  int32_t        groupKeyLen;    // total group by column width
-  SHashObj*      pGroupSet;      // quick locate the window object for each result
-
-  SDiskbasedBuf* pBuf;              // query result buffer based on blocked-wised disk file
-  int32_t        rowCapacity;       // maximum number of rows for each buffer page
-  int32_t*       columnOffset;      // start position for each column data
-  SArray*        sortedGroupArray;  // SDataGroupInfo sorted by group id
-  int32_t        groupIndex;        // group index
-  int32_t        pageIndex;         // page index of current group
-  SExprSupp      scalarSup;
-} SPartitionOperatorInfo;
-
 typedef struct SWindowRowsSup {
   STimeWindow win;
   TSKEY       prevTs;
@@ -754,6 +647,23 @@ typedef struct SStreamPartitionOperatorInfo {
   SSDataBlock*          pDelRes;
 } SStreamPartitionOperatorInfo;
 
+typedef struct SStreamFillSupporter {
+  int32_t        type;  // fill type
+  SInterval      interval;
+  SResultRowData prev;
+  SResultRowData cur;
+  SResultRowData next;
+  SResultRowData nextNext;
+  SFillColInfo*  pAllColInfo;  // fill exprs and not fill exprs
+  SExprSupp      notFillExprSup;
+  int32_t        numOfAllCols;  // number of all exprs, including the tags columns
+  int32_t        numOfFillCols;
+  int32_t        numOfNotFillCols;
+  int32_t        rowSize;
+  SSHashObj*     pResMap;
+  bool           hasDelete;
+} SStreamFillSupporter;
+
 typedef struct SStreamFillOperatorInfo {
   SStreamFillSupporter* pFillSup;
   SSDataBlock*          pRes;
@@ -800,33 +710,6 @@ typedef struct SStateWindowOperatorInfo {
   STimeWindowAggSupp twAggSup;
 } SStateWindowOperatorInfo;
 
-typedef struct SSortOperatorInfo {
-  SOptrBasicInfo binfo;
-  uint32_t       sortBufSize;  // max buffer size for in-memory sort
-  SArray*        pSortInfo;
-  SSortHandle*   pSortHandle;
-  SColMatchInfo  matchInfo;
-  int32_t        bufPageSize;
-  int64_t        startTs;      // sort start time
-  uint64_t       sortElapsed;  // sort elapsed time, time to flush to disk not included.
-  SLimitInfo     limitInfo;
-} SSortOperatorInfo;
-
-typedef struct SJoinOperatorInfo {
-  SSDataBlock* pRes;
-  int32_t      joinType;
-  int32_t      inputOrder;
-
-  SSDataBlock* pLeft;
-  int32_t      leftPos;
-  SColumnInfo  leftCol;
-
-  SSDataBlock* pRight;
-  int32_t      rightPos;
-  SColumnInfo  rightCol;
-  SNode*       pCondAfterMerge;
-} SJoinOperatorInfo;
-
 #define OPTR_IS_OPENED(_optr)  (((_optr)->status & OP_OPENED) == OP_OPENED)
 #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
 
@@ -850,7 +733,6 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
 void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                             SDiskbasedBuf* pBuf);
 
-int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
 bool    hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
 void    initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
 void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
@@ -880,9 +762,6 @@ void cleanupAggSup(SAggSupporter* pAggSup);
 void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
 void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name);
 
-int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts);
-int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
-
 SSDataBlock* loadNextDataBlock(void* param);
 
 void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
@@ -965,9 +844,8 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order,
 bool    isTaskKilled(SExecTaskInfo* pTaskInfo);
 int32_t checkForQueryBuf(size_t numOfTables);
 
-void setTaskKilled(SExecTaskInfo* pTaskInfo);
+void    setTaskKilled(SExecTaskInfo* pTaskInfo);
-void queryCostStatis(SExecTaskInfo* pTaskInfo);
+void    queryCostStatis(SExecTaskInfo* pTaskInfo);
-
 void    doDestroyTask(SExecTaskInfo* pTaskInfo);
 void    destroyOperatorInfo(SOperatorInfo* pOperator);
 int32_t getMaximumIdleDurationSec();
@@ -995,9 +873,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
 int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
 int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
 
-int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result);
-int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length);
-
 STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
                                 int32_t order);
 int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,

source/libs/executor/inc/tfill.h

@@ -111,22 +111,6 @@ typedef struct SStreamFillInfo {
   int32_t                delIndex;
 } SStreamFillInfo;
 
-typedef struct SStreamFillSupporter {
-  int32_t        type;  // fill type
-  SInterval      interval;
-  SResultRowData prev;
-  SResultRowData cur;
-  SResultRowData next;
-  SResultRowData nextNext;
-  SFillColInfo*  pAllColInfo;   // fill exprs and not fill exprs
-  int32_t        numOfAllCols;  // number of all exprs, including the tags columns
-  int32_t        numOfFillCols;
-  int32_t        numOfNotFillCols;
-  int32_t        rowSize;
-  SSHashObj*     pResMap;
-  bool           hasDelete;
-} SStreamFillSupporter;
-
 int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
 
 void                 taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);

source/libs/executor/inc/tsort.h

@@ -36,12 +36,13 @@ typedef struct SMultiMergeSource {
 
 typedef struct SSortSource {
   SMultiMergeSource src;
-  union {
+  struct {
-    struct {
+    SArray* pageIdList;
-      SArray* pageIdList;
+    int32_t pageIndex;
-      int32_t pageIndex;
+  };
-    };
+  struct {
     void* param;
+    bool  onlyRef;
   };
 
 } SSortSource;

source/libs/executor/src/cachescanoperator.c

@@ -25,15 +25,29 @@
 #include "thash.h"
 #include "ttypes.h"
 
+typedef struct SCacheRowsScanInfo {
+  SSDataBlock*    pRes;
+  SReadHandle     readHandle;
+  void*           pLastrowReader;
+  SColMatchInfo   matchInfo;
+  int32_t*        pSlotIds;
+  SExprSupp       pseudoExprSup;
+  int32_t         retrieveType;
+  int32_t         currentGroupIndex;
+  SSDataBlock*    pBufferredRes;
+  SArray*         pUidList;
+  int32_t         indexOfBufferedRes;
+} SCacheRowsScanInfo;
+
 static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
-static void         destroyLastrowScanOperator(void* param);
+static void         destroyCacheScanOperator(void* param);
 static int32_t      extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
 static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo);
 
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                            SExecTaskInfo* pTaskInfo) {
   int32_t           code = TSDB_CODE_SUCCESS;
-  SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo));
+  SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo));
   SOperatorInfo*    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -97,14 +111,14 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
 
   pOperator->fpSet =
-      createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL);
+      createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL);
 
   pOperator->cost.openCost = 0;
   return pOperator;
 
   _error:
   pTaskInfo->code = code;
-  destroyLastrowScanOperator(pInfo);
+  destroyCacheScanOperator(pInfo);
   taosMemoryFree(pOperator);
   return NULL;
 }
@@ -114,7 +128,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-  SLastrowScanInfo* pInfo = pOperator->info;
+  SCacheRowsScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*    pTaskInfo = pOperator->pTaskInfo;
   STableListInfo*   pTableList = pTaskInfo->pTableInfoList;
 
@@ -157,9 +171,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
         SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId);
         SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId);
 
-        char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
+        if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) {
-        bool  isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes);
+          colDataAppendNULL(pDst, 0);
-        colDataAppend(pDst, 0, p, isNull);
+        } else {
+          char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
+          colDataAppend(pDst, 0, p, false);
+        }
       }
 
       pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes);
@@ -226,6 +243,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
 
         pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
         return pInfo->pRes;
+      } else {
+        pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
       }
     }
 
@@ -234,8 +253,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
   }
 }
 
-void destroyLastrowScanOperator(void* param) {
+void destroyCacheScanOperator(void* param) {
-  SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param;
+  SCacheRowsScanInfo* pInfo = (SCacheRowsScanInfo*)param;
   blockDataDestroy(pInfo->pRes);
   blockDataDestroy(pInfo->pBufferredRes);
   taosMemoryFree(pInfo->pSlotIds);

source/libs/executor/src/dataDispatcher.c

@@ -138,7 +138,9 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
   
   toDataCacheEntry(pDispatcher, pInput, pBuf);
   taosWriteQitem(pDispatcher->pDataBlocks, pBuf);
-  *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false);
+
+  int32_t status = updateStatus(pDispatcher);
+  *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY);
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/executor/src/dataInserter.c

@@ -250,6 +250,8 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
     return code;
   }
 
+  taosArrayClear(pInserter->pDataBlocks);
+  
   code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet);
   if (code) {
     return code;

source/libs/executor/src/exchangeoperator.c

@@ -41,6 +41,16 @@ typedef struct SFetchRspHandleWrapper {
   int32_t  sourceIndex;
 } SFetchRspHandleWrapper;
 
+typedef struct SSourceDataInfo {
+  int32_t            index;
+  SRetrieveTableRsp* pRsp;
+  uint64_t           totalRows;
+  int64_t            startTime;
+  int32_t            code;
+  EX_SOURCE_STATUS   status;
+  const char*        taskId;
+} SSourceDataInfo;
+
 static void destroyExchangeOperatorInfo(void* param);
 static void freeBlock(void* pParam);
 static void freeSourceDataInfo(void* param);
@@ -52,6 +62,8 @@ static int32_t getCompletedSources(const SArray* pArray);
 static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator);
 static int32_t seqLoadRemoteData(SOperatorInfo* pOperator);
 static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator);
+static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
+static int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo);
 
 static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo,
                                            SExecTaskInfo* pTaskInfo) {
@@ -64,7 +76,9 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
   }
 
   while (1) {
+    qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo));
     tsem_wait(&pExchangeInfo->ready);
+
     if (isTaskKilled(pTaskInfo)) {
       longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
     }
@@ -99,20 +113,12 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
         break;
       }
 
-      SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
+      code = doExtractResultBlocks(pExchangeInfo, pDataInfo);
-      int32_t            index = 0;
+      if (code != TSDB_CODE_SUCCESS) {
-      char*              pStart = pRetrieveRsp->data;
+        goto _error;
-      while (index++ < pRetrieveRsp->numOfBlocks) {
-        SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
-        code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
-        if (code != 0) {
-          taosMemoryFreeClear(pDataInfo->pRsp);
-          goto _error;
-        }
-
-        taosArrayPush(pExchangeInfo->pResultBlockList, &pb);
       }
 
+      SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
       updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pDataInfo->startTime, pOperator);
       pDataInfo->totalRows += pRetrieveRsp->numOfRows;
 
@@ -349,7 +355,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
 
   SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId);
   if (pExchangeInfo == NULL) {
-    qWarn("failed to acquire exchange operator, since it may have been released");
+    qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo);
     taosMemoryFree(pMsg->pData);
     return TSDB_CODE_SUCCESS;
   }
@@ -368,20 +374,23 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
     pRsp->numOfBlocks = htonl(pRsp->numOfBlocks);
 
     ASSERT(pRsp != NULL);
-    qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks,
+    qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks,
-           pRsp->numOfRows);
+           pRsp->numOfRows, pExchangeInfo);
   } else {
     taosMemoryFree(pMsg->pData);
     pSourceDataInfo->code = code;
-    qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code));
+    qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), pExchangeInfo);
   }
 
   pSourceDataInfo->status = EX_SOURCE_DATA_READY;
+  code = tsem_post(&pExchangeInfo->ready);
+  if (code != TSDB_CODE_SUCCESS) {
+    code = TAOS_SYSTEM_ERROR(code);
+    qError("failed to invoke post when fetch rsp is ready, code:%s, %p", tstrerror(code), pExchangeInfo);
+  }
 
-  tsem_post(&pExchangeInfo->ready);
   taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId);
-
+  return code;
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) {
@@ -405,27 +414,42 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
     loadRemoteDataCallback(pWrapper, &pBuf, code);
     taosMemoryFree(pWrapper);
   } else {
-    SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq));
+    SResFetchReq req = {0};
-    if (NULL == pMsg) {
+    req.header.vgId = pSource->addr.nodeId;
+    req.sId = pSource->schedId;
+    req.taskId = pSource->taskId;
+    req.queryId = pTaskInfo->id.queryId;
+    req.execId = pSource->execId;
+
+    int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req);
+    if (msgSize < 0) {
       pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
       taosMemoryFree(pWrapper);
       return pTaskInfo->code;
     }
     
-    qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
+    void* msg = taosMemoryCalloc(1, msgSize);
-           GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId,
+    if (NULL == msg) {
-           pSource->execId, sourceIndex, totalSources);
+      pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+      taosMemoryFree(pWrapper);
+      return pTaskInfo->code;
+    }
     
-    pMsg->header.vgId = htonl(pSource->addr.nodeId);
+    if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) {
-    pMsg->sId = htobe64(pSource->schedId);
+      pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
-    pMsg->taskId = htobe64(pSource->taskId);
+      taosMemoryFree(pWrapper);
-    pMsg->queryId = htobe64(pTaskInfo->id.queryId);
+      taosMemoryFree(msg);
-    pMsg->execId = htonl(pSource->execId);
+      return pTaskInfo->code;
+    }
+
+    qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu,
+           GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId,
+           pSource->execId, pExchangeInfo, sourceIndex, totalSources);
 
     // send the fetch remote task result reques
     SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
     if (NULL == pMsgSendInfo) {
-      taosMemoryFreeClear(pMsg);
+      taosMemoryFreeClear(msg);
       taosMemoryFree(pWrapper);
       qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
       pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -434,8 +458,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas
 
     pMsgSendInfo->param = pWrapper;
     pMsgSendInfo->paramFreeFp = taosMemoryFree;
-    pMsgSendInfo->msgInfo.pData = pMsg;
+    pMsgSendInfo->msgInfo.pData = msg;
-    pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq);
+    pMsgSendInfo->msgInfo.len = msgSize;
     pMsgSendInfo->msgType = pSource->fetchMsgType;
     pMsgSendInfo->fp = loadRemoteDataCallback;
 
@@ -555,10 +579,32 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) {
+  SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
+
+  char*   pStart = pRetrieveRsp->data;
+  int32_t index = 0;
+  int32_t code = 0;
+  while (index++ < pRetrieveRsp->numOfBlocks) {
+    SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
+
+    code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
+    if (code != 0) {
+      taosMemoryFreeClear(pDataInfo->pRsp);
+      return code;
+    }
+
+    taosArrayPush(pExchangeInfo->pResultBlockList, &pb);
+  }
+
+  return code;
+}
+
 int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
   SExchangeInfo* pExchangeInfo = pOperator->info;
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
+  int32_t code = 0;
   size_t  totalSources = taosArrayGetSize(pExchangeInfo->pSources);
   int64_t startTs = taosGetTimestampUs();
 
@@ -598,11 +644,12 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
       continue;
     }
 
+    code = doExtractResultBlocks(pExchangeInfo, pDataInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+
     SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
-
-    char*   pStart = pRetrieveRsp->data;
-    int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart);
-
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
                  ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
@@ -625,6 +672,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
     taosMemoryFreeClear(pDataInfo->pRsp);
     return TSDB_CODE_SUCCESS;
   }
+
+  _error:
+  pTaskInfo->code = code;
+  return code;
 }
 
 int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) {
@@ -647,3 +698,80 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) {
   pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) {
+  if (pLimitInfo->remainGroupOffset > 0) {
+    if (pLimitInfo->currentGroupId == 0) {  // it is the first group
+      pLimitInfo->currentGroupId = pBlock->info.groupId;
+      blockDataCleanup(pBlock);
+      return PROJECT_RETRIEVE_CONTINUE;
+    } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
+      // now it is the data from a new group
+      pLimitInfo->remainGroupOffset -= 1;
+
+      // ignore data block in current group
+      if (pLimitInfo->remainGroupOffset > 0) {
+        blockDataCleanup(pBlock);
+        return PROJECT_RETRIEVE_CONTINUE;
+      }
+    }
+
+    // set current group id of the project operator
+    pLimitInfo->currentGroupId = pBlock->info.groupId;
+  }
+
+  // here check for a new group data, we need to handle the data of the previous group.
+  if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
+    pLimitInfo->numOfOutputGroups += 1;
+    if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
+      pOperator->status = OP_EXEC_DONE;
+      blockDataCleanup(pBlock);
+
+      return PROJECT_RETRIEVE_DONE;
+    }
+
+    // reset the value for a new group data
+    pLimitInfo->numOfOutputRows = 0;
+    pLimitInfo->remainOffset = pLimitInfo->limit.offset;
+
+    // existing rows that belongs to previous group.
+    if (pBlock->info.rows > 0) {
+      return PROJECT_RETRIEVE_DONE;
+    }
+  }
+
+  // here we reach the start position, according to the limit/offset requirements.
+
+  // set current group id
+  pLimitInfo->currentGroupId = pBlock->info.groupId;
+
+  if (pLimitInfo->remainOffset >= pBlock->info.rows) {
+    pLimitInfo->remainOffset -= pBlock->info.rows;
+    blockDataCleanup(pBlock);
+    return PROJECT_RETRIEVE_CONTINUE;
+  } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) {
+    blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
+    pLimitInfo->remainOffset = 0;
+  }
+
+  // check for the limitation in each group
+  if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) {
+    int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
+    blockDataKeepFirstNRows(pBlock, keepRows);
+    if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) {
+      pOperator->status = OP_EXEC_DONE;
+    }
+
+    return PROJECT_RETRIEVE_DONE;
+  }
+
+  // todo optimize performance
+  // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the
+  // they may not belong to the same group the limit/offset value is not valid in this case.
+  if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 ||
+      pLimitInfo->slimit.limit != -1) {
+    return PROJECT_RETRIEVE_DONE;
+  } else {  // not full enough, continue to accumulate the output data in the buffer.
+    return PROJECT_RETRIEVE_CONTINUE;
+  }
+}

source/libs/executor/src/executor.c

@@ -480,51 +480,6 @@ _error:
   return code;
 }
 
-#ifdef TEST_IMPL
-// wait moment
-int waitMoment(SQInfo* pQInfo) {
-  if (pQInfo->sql) {
-    int   ms = 0;
-    char* pcnt = strstr(pQInfo->sql, " count(*)");
-    if (pcnt) return 0;
-
-    char* pos = strstr(pQInfo->sql, " t_");
-    if (pos) {
-      pos += 3;
-      ms = atoi(pos);
-      while (*pos >= '0' && *pos <= '9') {
-        pos++;
-      }
-      char unit_char = *pos;
-      if (unit_char == 'h') {
-        ms *= 3600 * 1000;
-      } else if (unit_char == 'm') {
-        ms *= 60 * 1000;
-      } else if (unit_char == 's') {
-        ms *= 1000;
-      }
-    }
-    if (ms == 0) return 0;
-    printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql);
-
-    if (ms < 1000) {
-      taosMsleep(ms);
-    } else {
-      int used_ms = 0;
-      while (used_ms < ms) {
-        taosMsleep(1000);
-        used_ms += 1000;
-        if (isTaskKilled(pQInfo)) {
-          printf("test check query is canceled, sleep break.%s\n", pQInfo->sql);
-          break;
-        }
-      }
-    }
-  }
-  return 1;
-}
-#endif
-
 static void freeBlock(void* param) {
   SSDataBlock* pBlock = *(SSDataBlock**)param;
   blockDataDestroy(pBlock);

source/libs/executor/src/executorimpl.c

@@ -24,7 +24,6 @@
 #include "tdatablock.h"
 #include "tglobal.h"
 #include "tmsg.h"
-#include "tsort.h"
 #include "ttime.h"
 
 #include "executorimpl.h"
@@ -297,8 +296,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
   colDataAppendInt64(pColData, 4, &pQueryWindow->ekey);
 }
 
-void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { colDataDestroy(pColData); }
-
 typedef struct {
   bool    hasAgg;
   int32_t numOfRows;
@@ -1347,42 +1344,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) {
   }
 }
 
-// static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) {
-//   STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
-//   STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
-//
-//   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
-//
-//   if (pQueryAttr->limit.offset == pBlockInfo->rows) {  // current block will ignore completed
-//     pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step :
-//     pBlockInfo->window.skey + step; pQueryAttr->limit.offset = 0; return;
-//   }
-//
-//   if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
-//     pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset;
-//   } else {
-//     pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1;
-//   }
-//
-//   assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1);
-//
-//   SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
-//   SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
-//
-//   // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value
-//   TSKEY *keys = (TSKEY *) pColInfoData->pData;
-//
-//   // update the offset value
-//   pTableQueryInfo->lastKey = keys[pQueryAttr->pos];
-//   pQueryAttr->limit.offset = 0;
-//
-//   int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
-//
-//   //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d,
-//   lastKey:%"PRId64, GET_TASKID(pRuntimeEnv),
-//          pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey);
-// }
-
 // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) {
 //   STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
 //
@@ -1723,159 +1684,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
   return (rows == 0) ? NULL : pInfo->pRes;
 }
 
-int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length) {
-  if (result == NULL || length == NULL) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
-  }
-  SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info);
-  SAggSupporter*  pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo));
-  int32_t         size = tSimpleHashGetSize(pSup->pResultRowHashTable);
-  size_t          keyLen = sizeof(uint64_t) * 2;  // estimate the key length
-  int32_t         totalSize =
-      sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize);
-
-  // no result
-  if (getTotalBufSize(pSup->pResultBuf) == 0) {
-    *result = NULL;
-    *length = 0;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  *result = (char*)taosMemoryCalloc(1, totalSize);
-  if (*result == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  int32_t offset = sizeof(int32_t);
-  *(int32_t*)(*result + offset) = size;
-  offset += sizeof(int32_t);
-
-  // prepare memory
-  SResultRowPosition* pos = &pInfo->resultRowInfo.cur;
-  void*               pPage = getBufPage(pSup->pResultBuf, pos->pageId);
-  SResultRow*         pRow = (SResultRow*)((char*)pPage + pos->offset);
-  setBufPageDirty(pPage, true);
-  releaseBufPage(pSup->pResultBuf, pPage);
-
-  int32_t iter = 0;
-  void*   pIter = NULL;
-  while ((pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter))) {
-    void*               key = tSimpleHashGetKey(pIter, &keyLen);
-    SResultRowPosition* p1 = (SResultRowPosition*)pIter;
-
-    pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId);
-    pRow = (SResultRow*)((char*)pPage + p1->offset);
-    setBufPageDirty(pPage, true);
-    releaseBufPage(pSup->pResultBuf, pPage);
-
-    // recalculate the result size
-    int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize;
-    if (realTotalSize > totalSize) {
-      char* tmp = (char*)taosMemoryRealloc(*result, realTotalSize);
-      if (tmp == NULL) {
-        taosMemoryFree(*result);
-        *result = NULL;
-        return TSDB_CODE_OUT_OF_MEMORY;
-      } else {
-        *result = tmp;
-      }
-    }
-    // save key
-    *(int32_t*)(*result + offset) = keyLen;
-    offset += sizeof(int32_t);
-    memcpy(*result + offset, key, keyLen);
-    offset += keyLen;
-
-    // save value
-    *(int32_t*)(*result + offset) = pSup->resultRowSize;
-    offset += sizeof(int32_t);
-    memcpy(*result + offset, pRow, pSup->resultRowSize);
-    offset += pSup->resultRowSize;
-  }
-
-  *(int32_t*)(*result) = offset;
-  *length = offset;
-
-  return TDB_CODE_SUCCESS;
-}
-
-int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) {
-  if (pLimitInfo->remainGroupOffset > 0) {
-    if (pLimitInfo->currentGroupId == 0) {  // it is the first group
-      pLimitInfo->currentGroupId = pBlock->info.groupId;
-      blockDataCleanup(pBlock);
-      return PROJECT_RETRIEVE_CONTINUE;
-    } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
-      // now it is the data from a new group
-      pLimitInfo->remainGroupOffset -= 1;
-
-      // ignore data block in current group
-      if (pLimitInfo->remainGroupOffset > 0) {
-        blockDataCleanup(pBlock);
-        return PROJECT_RETRIEVE_CONTINUE;
-      }
-    }
-
-    // set current group id of the project operator
-    pLimitInfo->currentGroupId = pBlock->info.groupId;
-  }
-
-  // here check for a new group data, we need to handle the data of the previous group.
-  if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
-    pLimitInfo->numOfOutputGroups += 1;
-    if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
-      pOperator->status = OP_EXEC_DONE;
-      blockDataCleanup(pBlock);
-
-      return PROJECT_RETRIEVE_DONE;
-    }
-
-    // reset the value for a new group data
-    pLimitInfo->numOfOutputRows = 0;
-    pLimitInfo->remainOffset = pLimitInfo->limit.offset;
-
-    // existing rows that belongs to previous group.
-    if (pBlock->info.rows > 0) {
-      return PROJECT_RETRIEVE_DONE;
-    }
-  }
-
-  // here we reach the start position, according to the limit/offset requirements.
-
-  // set current group id
-  pLimitInfo->currentGroupId = pBlock->info.groupId;
-
-  if (pLimitInfo->remainOffset >= pBlock->info.rows) {
-    pLimitInfo->remainOffset -= pBlock->info.rows;
-    blockDataCleanup(pBlock);
-    return PROJECT_RETRIEVE_CONTINUE;
-  } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) {
-    blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
-    pLimitInfo->remainOffset = 0;
-  }
-
-  // check for the limitation in each group
-  if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) {
-    int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
-    blockDataKeepFirstNRows(pBlock, keepRows);
-    if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) {
-      pOperator->status = OP_EXEC_DONE;
-    }
-
-    return PROJECT_RETRIEVE_DONE;
-  }
-
-  // todo optimize performance
-  // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the
-  // they may not belong to the same group the limit/offset value is not valid in this case.
-  if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 ||
-      pLimitInfo->slimit.limit != -1) {
-    return PROJECT_RETRIEVE_DONE;
-  } else {  // not full enough, continue to accumulate the output data in the buffer.
-    return PROJECT_RETRIEVE_CONTINUE;
-  }
-}
-
 static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
 static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
                                                SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
@@ -2056,6 +1864,8 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
     for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
       if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_COLUMN) {
         taosMemoryFreeClear(pExprInfo->base.pParam[j].pCol);
+      } else if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_VALUE) {
+        taosVariantDestroy(&pExprInfo->base.pParam[j].param);
       }
     }
 

source/libs/executor/src/groupoperator.c

@@ -27,6 +27,36 @@
 #include "thash.h"
 #include "ttypes.h"
 
+typedef struct SGroupbyOperatorInfo {
+  SOptrBasicInfo binfo;
+  SAggSupporter  aggSup;
+  SArray*        pGroupCols;     // group by columns, SArray<SColumn>
+  SArray*        pGroupColVals;  // current group column values, SArray<SGroupKeys>
+  bool           isInit;         // denote if current val is initialized or not
+  char*          keyBuf;         // group by keys for hash
+  int32_t        groupKeyLen;    // total group by column width
+  SGroupResInfo  groupResInfo;
+  SExprSupp      scalarSup;
+} SGroupbyOperatorInfo;
+
+// The sort in partition may be needed later.
+typedef struct SPartitionOperatorInfo {
+  SOptrBasicInfo binfo;
+  SArray*        pGroupCols;
+  SArray*        pGroupColVals;  // current group column values, SArray<SGroupKeys>
+  char*          keyBuf;         // group by keys for hash
+  int32_t        groupKeyLen;    // total group by column width
+  SHashObj*      pGroupSet;      // quick locate the window object for each result
+
+  SDiskbasedBuf* pBuf;              // query result buffer based on blocked-wised disk file
+  int32_t        rowCapacity;       // maximum number of rows for each buffer page
+  int32_t*       columnOffset;      // start position for each column data
+  SArray*        sortedGroupArray;  // SDataGroupInfo sorted by group id
+  int32_t        groupIndex;        // group index
+  int32_t        pageIndex;         // page index of current group
+  SExprSupp      scalarSup;
+} SPartitionOperatorInfo;
+
 static void*    getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len);
 static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity);
 static int32_t  setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData,
@@ -906,6 +936,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
       } else {
         pDest->info.parTbName[0] = 0;
       }
+      if (pParInfo->groupId && pDest->info.parTbName[0]) {
+        streamStatePutParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, pDest->info.parTbName);
+      }
       /*printf("\n\n set name %s\n\n", pDest->info.parTbName);*/
       blockDataDestroy(pTmpBlock);
       blockDataDestroy(pResBlock);

source/libs/executor/src/joinoperator.c

@@ -24,6 +24,21 @@
 #include "tmsg.h"
 #include "ttypes.h"
 
+typedef struct SJoinOperatorInfo {
+  SSDataBlock* pRes;
+  int32_t      joinType;
+  int32_t      inputOrder;
+
+  SSDataBlock* pLeft;
+  int32_t      leftPos;
+  SColumnInfo  leftCol;
+
+  SSDataBlock* pRight;
+  int32_t      rightPos;
+  SColumnInfo  rightCol;
+  SNode*       pCondAfterMerge;
+} SJoinOperatorInfo;
+
 static void         setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode);
 static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator);
 static void         destroyMergeJoinOperator(void* param);
@@ -39,22 +54,26 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown
     SColumnNode*   col2 = (SColumnNode*)pNode->pRight;
     SColumnNode*   leftTsCol = NULL;
     SColumnNode*   rightTsCol = NULL;
-    if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) {
+    if (col1->dataBlockId == col2->dataBlockId ) {
-      ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId);
       leftTsCol = col1;
       rightTsCol = col2;
     } else {
-      ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId);
+      if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) {
-      ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId);
+        ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId);
-      leftTsCol = col2;
+        leftTsCol = col1;
-      rightTsCol = col1;
+        rightTsCol = col2;
+      } else {
+        ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId);
+        ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId);
+        leftTsCol = col2;
+        rightTsCol = col1;
+      }
     }
     setJoinColumnInfo(&pInfo->leftCol, leftTsCol);
     setJoinColumnInfo(&pInfo->rightCol, rightTsCol);
   } else {
     ASSERT(false);
-  }
+  }}
-}
 
 SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
                                            SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {

source/libs/executor/src/projectoperator.c

@@ -17,6 +17,24 @@
 #include "executorimpl.h"
 #include "functionMgt.h"
 
+typedef struct SProjectOperatorInfo {
+  SOptrBasicInfo binfo;
+  SAggSupporter  aggSup;
+  SArray*        pPseudoColInfo;
+  SLimitInfo     limitInfo;
+  bool           mergeDataBlocks;
+  SSDataBlock*   pFinalRes;
+} SProjectOperatorInfo;
+
+typedef struct SIndefOperatorInfo {
+  SOptrBasicInfo binfo;
+  SAggSupporter  aggSup;
+  SArray*        pPseudoColInfo;
+  SExprSupp      scalarSup;
+  uint64_t       groupId;
+  SSDataBlock*   pNextGroupRes;
+} SIndefOperatorInfo;
+
 static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator);
 static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator);
 static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator);

source/libs/executor/src/sortoperator.c

@@ -17,6 +17,18 @@
 #include "executorimpl.h"
 #include "tdatablock.h"
 
+typedef struct SSortOperatorInfo {
+  SOptrBasicInfo binfo;
+  uint32_t       sortBufSize;  // max buffer size for in-memory sort
+  SArray*        pSortInfo;
+  SSortHandle*   pSortHandle;
+  SColMatchInfo  matchInfo;
+  int32_t        bufPageSize;
+  int64_t        startTs;      // sort start time
+  uint64_t       sortElapsed;  // sort elapsed time, time to flush to disk not included.
+  SLimitInfo     limitInfo;
+} SSortOperatorInfo;
+
 static SSDataBlock* doSort(SOperatorInfo* pOperator);
 static int32_t      doOpenSortOperator(SOperatorInfo* pOperator);
 static int32_t      getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len);
@@ -176,10 +188,10 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) {
 
   SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
   ps->param = pOperator->pDownstream[0];
+  ps->onlyRef = true;
   tsortAddSource(pInfo->pSortHandle, ps);
 
   int32_t code = tsortOpen(pInfo->pSortHandle);
-  taosMemoryFreeClear(ps);
 
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, terrno);
@@ -377,10 +389,10 @@ int32_t beginSortGroup(SOperatorInfo* pOperator) {
   param->childOpInfo = pOperator->pDownstream[0];
   param->grpSortOpInfo = pInfo;
   ps->param = param;
+  ps->onlyRef = false;
   tsortAddSource(pInfo->pCurrSortHandle, ps);
 
   int32_t code = tsortOpen(pInfo->pCurrSortHandle);
-  taosMemoryFreeClear(ps);
 
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, terrno);
@@ -471,6 +483,9 @@ void destroyGroupSortOperatorInfo(void* param) {
   taosArrayDestroy(pInfo->pSortInfo);
   taosArrayDestroy(pInfo->matchInfo.pList);
 
+  tsortDestroySortHandle(pInfo->pCurrSortHandle);
+  pInfo->pCurrSortHandle = NULL;
+
   taosMemoryFreeClear(param);
 }
 
@@ -563,6 +578,7 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
   for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
     ps->param = pOperator->pDownstream[i];
+    ps->onlyRef = true;
     tsortAddSource(pInfo->pSortHandle, ps);
   }
 

source/libs/executor/src/tfill.c

@@ -709,6 +709,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
   pFillSup->pResMap = NULL;
   releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal);
   pFillSup->cur.pRowVal = NULL;
+  cleanupExprSupp(&pFillSup->notFillExprSup);
 
   taosMemoryFree(pFillSup);
   return NULL;
@@ -1417,25 +1418,13 @@ static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock
   blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows);
   setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
   projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
+
+  pDstBlock->info.rows = 0;
+  pSup = &pInfo->pFillSup->notFillExprSup;
+  setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
+  projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
   pDstBlock->info.groupId = pSrcBlock->info.groupId;
 
-  SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, pInfo->primaryTsCol);
-  SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, pInfo->primarySrcSlotId);
-  colDataAssign(pDst, pSrc, pDstBlock->info.rows, &pDstBlock->info);
-
-  int32_t numOfNotFill = pInfo->pFillSup->numOfAllCols - pInfo->pFillSup->numOfFillCols;
-  for (int32_t i = 0; i < numOfNotFill; ++i) {
-    SFillColInfo* pCol = &pInfo->pFillSup->pAllColInfo[i + pInfo->pFillSup->numOfFillCols];
-    ASSERT(pCol->notFillCol);
-
-    SExprInfo* pExpr = pCol->pExpr;
-    int32_t    srcSlotId = pExpr->base.pParam[0].pCol->slotId;
-    int32_t    dstSlotId = pExpr->base.resSchema.slotId;
-
-    SColumnInfoData* pDst1 = taosArrayGet(pDstBlock->pDataBlock, dstSlotId);
-    SColumnInfoData* pSrc1 = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId);
-    colDataAssign(pDst1, pSrc1, pDstBlock->info.rows, &pDstBlock->info);
-  }
   blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol);
 }
 
@@ -1577,6 +1566,14 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
     destroyStreamFillSupporter(pFillSup);
     return NULL;
   }
+  
+  SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
+  code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols);
+  if (code != TSDB_CODE_SUCCESS) {
+    destroyStreamFillSupporter(pFillSup);
+    return NULL;
+  }
+
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pFillSup->pResMap = tSimpleHashInit(16, hashFn);
   pFillSup->hasDelete = false;

source/libs/executor/src/timewindowoperator.c

@@ -1618,6 +1618,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
   nodesDestroyNode((SNode*)pInfo->pPhyNode);
   colDataDestroy(&pInfo->twAggSup.timeWindowData);
   cleanupGroupResInfo(&pInfo->groupResInfo);
+  cleanupExprSupp(&pInfo->scalarSupp);
 
   taosMemoryFreeClear(param);
 }
@@ -3363,22 +3364,23 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
   }
 }
 
-void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t waterMark, uint16_t type,
+void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
-                    int32_t tsColIndex) {
+                    STimeWindowAggSupp* pTwSup) {
   if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
     SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
     pScanInfo->tsColIndex = tsColIndex;
   }
 
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    initDownStream(downstream->pDownstream[0], pAggSup, waterMark, type, tsColIndex);
+    initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup);
     return;
   }
   SStreamScanInfo* pScanInfo = downstream->info;
   pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
   if (!pScanInfo->pUpdateInfo) {
-    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
+    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
   }
+  pScanInfo->twAggSup = *pTwSup;
 }
 
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap,
@@ -4101,8 +4103,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
       createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL);
 
   if (downstream) {
-    initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType,
+    initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
-                   pInfo->primaryTsIndex);
     code = appendDownstream(pOperator, &downstream, 1);
   }
   return pOperator;
@@ -4605,8 +4606,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
                   pInfo, pTaskInfo);
   pOperator->fpSet =
       createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL);
-  initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType,
+  initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
-                 pInfo->primaryTsIndex);
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -5363,15 +5363,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
   initResultSizeInfo(&pOperator->resultInfo, 4096);
 
-  if (pIntervalPhyNode->window.pExprs != NULL) {
-    int32_t    numOfScalar = 0;
-    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
-    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
-    if (code != TSDB_CODE_SUCCESS) {
-      goto _error;
-    }
-  }
-
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
   code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
   if (code != TSDB_CODE_SUCCESS) {

source/libs/executor/src/tsort.c

@@ -110,6 +110,22 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
   return TSDB_CODE_SUCCESS;
 }
 
+void tsortClearOrderdSource(SArray *pOrderedSource) {
+  for (size_t i = 0; i < taosArrayGetSize(pOrderedSource); i++) {
+    SSortSource** pSource = taosArrayGet(pOrderedSource, i);
+    if (NULL == *pSource) {
+      continue;
+    }
+    
+    if ((*pSource)->param && !(*pSource)->onlyRef) {
+      taosMemoryFree((*pSource)->param);  
+    }
+    taosMemoryFreeClear(*pSource);
+  }
+
+  taosArrayClear(pOrderedSource);
+}
+
 void tsortDestroySortHandle(SSortHandle* pSortHandle) {
   if (pSortHandle == NULL) {
     return;
@@ -123,10 +139,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
   destroyDiskbasedBuf(pSortHandle->pBuf);
   taosMemoryFreeClear(pSortHandle->idStr);
   blockDataDestroy(pSortHandle->pDataBlock);
-  for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++) {
+
-    SSortSource** pSource = taosArrayGet(pSortHandle->pOrderedSource, i);
+  tsortClearOrderdSource(pSortHandle->pOrderedSource);
-    taosMemoryFreeClear(*pSource);
-  }
   taosArrayDestroy(pSortHandle->pOrderedSource);
   taosMemoryFreeClear(pSortHandle);
 }
@@ -561,7 +575,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
       }
     }
 
-    taosArrayClear(pHandle->pOrderedSource);
+    tsortClearOrderdSource(pHandle->pOrderedSource);
     taosArrayAddAll(pHandle->pOrderedSource, pResList);
     taosArrayDestroy(pResList);
 
@@ -598,8 +612,11 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
   size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize;
 
   if (pHandle->type == SORT_SINGLESOURCE_SORT) {
-    SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0);
+    SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
-    taosArrayClear(pHandle->pOrderedSource);
+    SSortSource* source = *pSource;
+    *pSource = NULL;
+    
+    tsortClearOrderdSource(pHandle->pOrderedSource);
 
     while (1) {
       SSDataBlock* pBlock = pHandle->fetchfp(source->param);
@@ -623,6 +640,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
 
       int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
       if (code != 0) {
+        if (source->param && !source->onlyRef) {
+          taosMemoryFree(source->param);
+        }
+        taosMemoryFree(source);
         return code;
       }
 
@@ -632,6 +653,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
         int64_t p = taosGetTimestampUs();
         code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
         if (code != 0) {
+          if (source->param && !source->onlyRef) {
+            taosMemoryFree(source->param);
+          }
+          taosMemoryFree(source);
           return code;
         }
 
@@ -642,6 +667,11 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
       }
     }
 
+    if (source->param && !source->onlyRef) {
+      taosMemoryFree(source->param);
+    }
+    taosMemoryFree(source);
+
     if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {
       size_t size = blockDataGetSize(pHandle->pDataBlock);
 

source/libs/function/src/builtinsimpl.c

@@ -5672,12 +5672,12 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) {
   int32_t numOfElems = 0;
   int32_t startOffset = pCtx->offset;
   for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
-    char* data = colDataGetData(pInputCol, i);
     if (colDataIsNull_s(pInputCol, i)) {
       continue;
     }
-
     numOfElems++;
+
+    char* data = colDataGetData(pInputCol, i);
     doModeAdd(pInfo, i, pCtx, data);
 
     if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) {
@@ -6568,7 +6568,9 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
   for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
     numOfElems++;
 
-    char* data = colDataGetData(pInputCol, i);
+    bool  isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL);
+    char* data = isNull ? NULL : colDataGetData(pInputCol, i);
+
     TSKEY cts = getRowPTs(pInput->pPTS, i);
     if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
       doSaveLastrow(pCtx, data, i, cts, pInfo);

source/libs/parser/src/parInsertStmt.c

@@ -47,6 +47,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
                            TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
   STableDataBlocks*   pDataBlock = (STableDataBlocks*)pBlock;
   SMsgBuf             pBuf = {.buf = msgBuf, .len = msgBufLen};
+  int32_t  code = TSDB_CODE_SUCCESS;
   SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
   if (NULL == tags) {
     return TSDB_CODE_QRY_APP_ERROR;
@@ -59,10 +60,10 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
 
   SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
   if (!tagName) {
-    return buildInvalidOperationMsg(&pBuf, "out of memory");
+    code = buildInvalidOperationMsg(&pBuf, "out of memory");
+    goto end;
   }
 
-  int32_t  code = TSDB_CODE_SUCCESS;
   SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
 
   bool  isJson = false;
@@ -77,6 +78,10 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
     int32_t  colLen = pTagSchema->bytes;
     if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
       colLen = bind[c].length[0];
+      if ((colLen + VARSTR_HEADER_SIZE) > pTagSchema->bytes) {
+        code = buildInvalidOperationMsg(&pBuf, "tag length is too big");
+        goto end;
+      }
     }
     taosArrayPush(tagName, pTagSchema->name);
     if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {

source/libs/parser/src/parInsertUtil.c

@@ -139,8 +139,8 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i
     if (i > 0) {
       pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes;
       pColList->cols[i].toffset = pColList->flen;
+      pColList->flen += TYPE_BYTES[type];
     }
-    pColList->flen += TYPE_BYTES[type];
     switch (type) {
       case TSDB_DATA_TYPE_BINARY:
         pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);

source/libs/qworker/src/qwMsg.c

@@ -499,27 +499,22 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
-  SResFetchReq *msg = pMsg->pCont;
+  SResFetchReq req = {0};
   SQWorker     *mgmt = (SQWorker *)qWorkerMgmt;
 
   qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
   QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1);
 
-  if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
+  if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
-    QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
+    QW_ELOG("tDeserializeSResFetchReq %d failed", pMsg->contLen);
     QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
-  msg->sId = be64toh(msg->sId);
+  uint64_t sId = req.sId;
-  msg->queryId = be64toh(msg->queryId);
+  uint64_t qId = req.queryId;
-  msg->taskId = be64toh(msg->taskId);
+  uint64_t tId = req.taskId;
-  msg->execId = ntohl(msg->execId);
-
-  uint64_t sId = msg->sId;
-  uint64_t qId = msg->queryId;
-  uint64_t tId = msg->taskId;
   int64_t  rId = 0;
-  int32_t  eId = msg->execId;
+  int32_t  eId = req.execId;
 
   SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connInfo = pMsg->info, .msgType = pMsg->msgType};
 

source/libs/qworker/src/qworker.c

@@ -743,7 +743,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
     }
 
     QW_LOCK(QW_WRITE, &ctx->lock);
-    if (queryStop || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) {
+    if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) {
       // Note: query is not running anymore
       QW_SET_PHASE(ctx, 0);
       QW_UNLOCK(QW_WRITE, &ctx->lock);

source/libs/scalar/src/sclfunc.c

@@ -1758,18 +1758,45 @@ int32_t sumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
       break;
     }
 
-    if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
-      int64_t *in = (int64_t *)pInputData->pData;
       int64_t *out = (int64_t *)pOutputData->pData;
-      *out += in[i];
+      if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
+        int8_t *in = (int8_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_SMALLINT) {
+        int16_t *in = (int16_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_INT) {
+        int32_t *in = (int32_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_BIGINT) {
+        int64_t *in = (int64_t *)pInputData->pData;
+        *out += in[i];
+      }
     } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
-      uint64_t *in = (uint64_t *)pInputData->pData;
       uint64_t *out = (uint64_t *)pOutputData->pData;
-      *out += in[i];
+      if (type == TSDB_DATA_TYPE_UTINYINT) {
+        uint8_t *in = (uint8_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_USMALLINT) {
+        uint16_t *in = (uint16_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_UINT) {
+        uint32_t *in = (uint32_t *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_UBIGINT) {
+        uint64_t *in = (uint64_t *)pInputData->pData;
+        *out += in[i];
+      }
     } else if (IS_FLOAT_TYPE(type)) {
-      double *in = (double *)pInputData->pData;
       double *out = (double *)pOutputData->pData;
-      *out += in[i];
+      if (type == TSDB_DATA_TYPE_FLOAT) {
+        float *in = (float *)pInputData->pData;
+        *out += in[i];
+      } else if (type == TSDB_DATA_TYPE_DOUBLE) {
+        double *in = (double *)pInputData->pData;
+        *out += in[i];
+      }
     }
   }
 

source/libs/scheduler/src/schRemote.c

@@ -1083,22 +1083,29 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
     }
     case TDMT_SCH_FETCH:
     case TDMT_SCH_MERGE_FETCH: {
-      msgSize = sizeof(SResFetchReq);
+      SResFetchReq req = {0};
+      req.header.vgId = addr->nodeId;
+      req.sId = schMgmt.sId;
+      req.queryId = pJob->queryId;
+      req.taskId = pTask->taskId;
+      req.execId = pTask->execId;
+
+      msgSize = tSerializeSResFetchReq(NULL, 0, &req);
+      if (msgSize < 0) {
+        SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize);
+        SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+      }
+      
       msg = taosMemoryCalloc(1, msgSize);
       if (NULL == msg) {
         SCH_TASK_ELOG("calloc %d failed", msgSize);
         SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
 
-      SResFetchReq *pMsg = msg;
+      if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) {
-
+        SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize);
-      pMsg->header.vgId = htonl(addr->nodeId);
+        SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
-
+      }
-      pMsg->sId = htobe64(schMgmt.sId);
-      pMsg->queryId = htobe64(pJob->queryId);
-      pMsg->taskId = htobe64(pTask->taskId);
-      pMsg->execId = htonl(pTask->execId);
-
       break;
     }
     case TDMT_SCH_DROP_TASK: {

source/libs/sync/inc/syncInt.h

@@ -232,6 +232,7 @@ int32_t   syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, S
 int32_t   syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
 SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode);
 int32_t   syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h);
+bool      syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode);
 
 // raft state change --------------
 void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term);

source/libs/sync/inc/syncMessage.h

@@ -124,6 +124,7 @@ typedef struct SyncHeartbeat {
   SyncIndex commitIndex;
   SyncTerm  privateTerm;
   SyncTerm  minMatchIndex;
+  int64_t   timeStamp;
 } SyncHeartbeat;
 
 typedef struct SyncHeartbeatReply {
@@ -137,6 +138,7 @@ typedef struct SyncHeartbeatReply {
   SyncTerm term;
   SyncTerm privateTerm;
   int64_t  startTime;
+  int64_t  timeStamp;
 } SyncHeartbeatReply;
 
 typedef struct SyncPreSnapshot {

source/libs/sync/src/syncCommit.c

@@ -171,7 +171,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
     pSyncNode->pLogStore->syncLogUpdateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex);
 
     // execute fsm
-    if (pSyncNode->pFsm != NULL) {
+    if (pSyncNode != NULL && pSyncNode->pFsm != NULL) {
       int32_t code = syncNodeDoCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
       if (code != 0) {
         sNError(pSyncNode, "advance commit index error, do commit begin:%" PRId64 ", end:%" PRId64, beginIndex,

source/libs/sync/src/syncIndexMgr.c

@@ -50,9 +50,10 @@ void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) {
   memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm));
 
   // int64_t timeNow = taosGetMonotonicMs();
+  int64_t timeNow = taosGetTimestampMs();
   for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
     pSyncIndexMgr->startTimeArr[i] = 0;
-    pSyncIndexMgr->recvTimeArr[i] = 0;
+    pSyncIndexMgr->recvTimeArr[i] = timeNow;
   }
 
   /*
@@ -147,7 +148,7 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRa
       return recvTime;
     }
   }
-  ASSERT(0);
+
   return -1;
 }
 

source/libs/sync/src/syncMain.c

@@ -639,6 +639,14 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
     return -1;
   }
 
+  // heartbeat timeout
+  if (syncNodeHeartbeatReplyTimeout(pSyncNode)) {
+    terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY;
+    sNError(pSyncNode, "failed to sync propose since hearbeat timeout, type:%s, last:%" PRId64 ", cmt:%" PRId64,
+            TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex);
+    return -1;
+  }
+
   // optimized one replica
   if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) {
     SyncIndex retIndex;
@@ -2031,6 +2039,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) {
       pSyncMsg->commitIndex = pSyncNode->commitIndex;
       pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode);
       pSyncMsg->privateTerm = 0;
+      pSyncMsg->timeStamp = taosGetTimestampMs();
 
       // send msg
       syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg);
@@ -2086,6 +2095,29 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand
   return code;
 }
 
+bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) {
+  if (pSyncNode->replicaNum == 1) {
+    return false;
+  }
+
+  int32_t toCount = 0;
+  int64_t tsNow = taosGetTimestampMs();
+  for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
+    int64_t recvTime = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i]));
+    if (recvTime == 0 || recvTime == -1) {
+      continue;
+    }
+
+    if (tsNow - recvTime > SYNC_HEART_TIMEOUT_MS) {
+      toCount++;
+    }
+  }
+
+  bool b = (toCount >= pSyncNode->quorum ? true : false);
+
+  return b;
+}
+
 static int32_t syncNodeAppendNoop(SSyncNode* ths) {
   int32_t ret = 0;
 
@@ -2117,7 +2149,11 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 
 int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   SyncHeartbeat* pMsg = pRpcMsg->pCont;
-  syncLogRecvHeartbeat(ths, pMsg, "");
+
+  int64_t tsMs = taosGetTimestampMs();
+  char    buf[128];
+  snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs);
+  syncLogRecvHeartbeat(ths, pMsg, buf);
 
   SRpcMsg rpcMsg = {0};
   (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId);
@@ -2127,8 +2163,11 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   pMsgReply->srcId = ths->myRaftId;
   pMsgReply->term = ths->pRaftStore->currentTerm;
   pMsgReply->privateTerm = 8864;  // magic number
+  pMsgReply->timeStamp = taosGetTimestampMs();
 
   if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) {
+    syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs);
+
     syncNodeResetElectTimer(ths);
     ths->minMatchIndex = pMsg->minMatchIndex;
 
@@ -2188,10 +2227,14 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
 
 int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   SyncHeartbeatReply* pMsg = pRpcMsg->pCont;
-  syncLogRecvHeartbeatReply(ths, pMsg, "");
+
+  int64_t tsMs = taosGetTimestampMs();
+  char    buf[128];
+  snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs);
+  syncLogRecvHeartbeatReply(ths, pMsg, buf);
 
   // update last reply time, make decision whether the other node is alive or not
-  syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->destId, pMsg->startTime);
+  syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs);
   return 0;
 }
 

source/libs/sync/src/syncRaftEntry.c

@@ -64,7 +64,7 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn
 }
 
 SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) {
-  SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen);
+  SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen));
   if (pEntry == NULL) return NULL;
 
   memcpy(pEntry, pMsg->data, pMsg->dataLen);
@@ -91,15 +91,14 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId)
 
 void syncEntryDestory(SSyncRaftEntry* pEntry) {
   if (pEntry != NULL) {
-    taosMemoryFree(pEntry);
-
     sTrace("free entry: %p", pEntry);
+    taosMemoryFree(pEntry);
   }
 }
 
 void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) {
   pRpcMsg->msgType = pEntry->originalRpcType;
-  pRpcMsg->contLen = pEntry->dataLen;
+  pRpcMsg->contLen = (int32_t)(pEntry->dataLen);
   pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
   memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen);
 }
@@ -339,7 +338,8 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index,
   SSyncRaftEntry* pEntry = NULL;
   int32_t         code = raftEntryCacheGetEntryP(pCache, index, &pEntry);
   if (code == 1) {
-    *ppEntry = taosMemoryMalloc((int64_t)(pEntry->bytes));
+    int32_t bytes = (int32_t)pEntry->bytes;
+    *ppEntry = taosMemoryMalloc((int64_t)bytes);
     memcpy(*ppEntry, pEntry, pEntry->bytes);
     (*ppEntry)->rid = -1;
   } else {

source/libs/sync/src/syncReplication.c

@@ -91,7 +91,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
   if (code == 0) {
     ASSERT(pEntry != NULL);
 
-    code = syncBuildAppendEntries(&rpcMsg, pEntry->bytes, pSyncNode->vgId);
+    code = syncBuildAppendEntries(&rpcMsg, (int32_t)(pEntry->bytes), pSyncNode->vgId);
     ASSERT(code == 0);
 
     pMsg = rpcMsg.pCont;
@@ -213,9 +213,11 @@ int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcM
 }
 
 int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) {
+  int64_t ts = taosGetTimestampMs();
   for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
     SRpcMsg rpcMsg = {0};
     if (syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId) != 0) {
+      sError("vgId:%d, build sync-heartbeat error", pSyncNode->vgId);
       continue;
     }
 
@@ -226,6 +228,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) {
     pSyncMsg->commitIndex = pSyncNode->commitIndex;
     pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode);
     pSyncMsg->privateTerm = 0;
+    pSyncMsg->timeStamp = ts;
 
     // send msg
     syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg);

source/libs/sync/src/syncUtil.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "syncUtil.h"
+#include "syncIndexMgr.h"
 #include "syncMessage.h"
 #include "syncRaftCfg.h"
 #include "syncRaftStore.h"
@@ -175,6 +176,36 @@ void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) {
   }
 }
 
+// for leader
+static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) {
+  int32_t len = 5;
+
+  for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) {
+    int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->replicasId[i]));
+
+    if (i < pSyncNode->replicaNum - 1) {
+      len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs);
+    } else {
+      len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs);
+    }
+  }
+}
+
+// for follower
+static void syncHearbeatTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) {
+  int32_t len = 4;
+
+  for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) {
+    int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->replicasId[i]));
+
+    if (i < pSyncNode->replicaNum - 1) {
+      len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs);
+    } else {
+      len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs);
+    }
+  }
+}
+
 static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) {
   int32_t len = 1;
 
@@ -221,6 +252,12 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
   char peerStr[1024] = "{";
   syncPeerState2Str(pNode, peerStr, sizeof(peerStr));
 
+  char hbrTimeStr[256] = "hbr:{";
+  syncHearbeatReplyTime2Str(pNode, hbrTimeStr, sizeof(hbrTimeStr));
+
+  char hbTimeStr[256] = "hb:{";
+  syncHearbeatTime2Str(pNode, hbTimeStr, sizeof(hbTimeStr));
+
   int32_t quorum = syncNodeDynamicQuorum(pNode);
 
   char    eventLog[512];  // {0};
@@ -229,7 +266,10 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
   int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer);
   va_end(argpointer);
 
-  int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm);
+  int32_t aqItems = 0;
+  if (pNode != NULL && pNode->pFsm != NULL && pNode->pFsm->FpApplyQueueItems != NULL) {
+    aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm);
+  }
 
   // restore error code
   terrno = errCode;
@@ -240,12 +280,13 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
                  "%s"
                  ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64
                  ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64
-                 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s",
+                 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s",
                  pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex,
                  logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm,
                  pNode->pRaftCfg->isStandBy, aqItems, pNode->snapshottingIndex, pNode->replicaNum,
                  pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum,
-                 pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr);
+                 pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr,
+                 hbrTimeStr);
   }
 }
 
@@ -392,9 +433,8 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const
   syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
 
   sNTrace(pSyncNode,
-          "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64
+          "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s",
-          "}, %s",
+          host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s);
-          host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s);
 }
 
 void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const char* s) {
@@ -403,9 +443,9 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const
   syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
 
   sNTrace(pSyncNode,
-          "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64
+          "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
           "}, %s",
-          host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s);
+          host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s);
 }
 
 void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) {
@@ -413,16 +453,16 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p
   uint16_t port;
   syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
 
-  sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port,
+  sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port,
-          pMsg->term, pMsg->privateTerm, s);
+          pMsg->term, pMsg->timeStamp, s);
 }
 
 void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) {
   char     host[64];
   uint16_t port;
   syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
-  sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port,
+  sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port,
-          pMsg->term, pMsg->privateTerm, s);
+          pMsg->term, pMsg->timeStamp, s);
 }
 
 void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) {

source/libs/transport/src/transCli.c

@@ -445,9 +445,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
 
     if (pCtx == NULL || pCtx->pSem == NULL) {
       if (transMsg.info.ahandle == NULL) {
-        if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
+        if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) {
-        once = true;
+          destroyCmsg(pMsg);
-        continue;
+          once = true;
+          continue;
+        }
       }
     }
 
@@ -791,6 +793,7 @@ void cliSend(SCliConn* pConn) {
 
   int            msgLen = transMsgLenFromCont(pMsg->contLen);
   STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
+
   pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0;
   pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0;
   pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0;
@@ -820,10 +823,15 @@ void cliSend(SCliConn* pConn) {
     uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0);
   }
 
-  if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) {
+  if (pHead->comp == 0) {
-    msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead);
+    if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) {
-    pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
+      msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead);
+      pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
+    }
+  } else {
+    msgLen = (int32_t)ntohl((uint32_t)(pHead->msgLen));
   }
+
   tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn,
           TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, msgLen);
 
@@ -1211,6 +1219,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
   if (pMsg == NULL) {
     return;
   }
+
   transDestroyConnCtx(pMsg->ctx);
   destroyUserdata(&pMsg->msg);
   taosMemoryFree(pMsg);

source/libs/transport/src/transSvr.c

@@ -398,7 +398,7 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
   pHead->magicNum = htonl(TRANS_MAGIC_NUM);
 
   // handle invalid drop_task resp, TD-20098
-  if (pMsg->msgType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
+  if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
     transQueuePop(&pConn->srvMsgs);
     destroySmsg(smsg);
     return -1;

source/libs/wal/src/walWrite.c

@@ -324,24 +324,36 @@ int32_t walEndSnapshot(SWal *pWal) {
   // find files safe to delete
   SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
   if (pInfo) {
+    SWalFileInfo *pLastFileInfo = taosArrayGetLast(pWal->fileInfoSet);
+    wDebug("vgId:%d, wal search found file info: first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer,
+           pInfo->lastVer);
     if (ver >= pInfo->lastVer) {
-      //pInfo--;
       pInfo++;
+      wDebug("vgId:%d, wal remove advance one file: first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer,
+             pInfo->lastVer);
     }
-    if (POINTER_DISTANCE(pInfo, pWal->fileInfoSet->pData) > 0) {
+    if (pInfo <= pLastFileInfo) {
-      wDebug("vgId:%d, wal end remove for %" PRId64, pWal->cfg.vgId, pInfo->firstVer);
+      wDebug("vgId:%d, wal end remove for first:%" PRId64 " last:%" PRId64, pWal->cfg.vgId, pInfo->firstVer,
+             pInfo->lastVer);
     } else {
       wDebug("vgId:%d, wal no remove", pWal->cfg.vgId);
     }
     // iterate files, until the searched result
     for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
-      if ((pWal->cfg.retentionSize != -1 && newTotSize > pWal->cfg.retentionSize) ||
+      wDebug("vgId:%d, wal check remove file %" PRId64 "(file size %" PRId64 " close ts %" PRId64
-          (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) {
+             "), new tot size %" PRId64,
+             pWal->cfg.vgId, iter->firstVer, iter->fileSize, iter->closeTs, newTotSize);
+      if (((pWal->cfg.retentionSize == 0) || (pWal->cfg.retentionSize != -1 && newTotSize > pWal->cfg.retentionSize)) ||
+          ((pWal->cfg.retentionPeriod == 0) ||
+           (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts))) {
         // delete according to file size or close time
+        wDebug("vgId:%d, check pass", pWal->cfg.vgId);
         deleteCnt++;
         newTotSize -= iter->fileSize;
       }
+      wDebug("vgId:%d, check not pass", pWal->cfg.vgId);
     }
+    wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt);
     int32_t actualDelete = 0;
     char    fnameStr[WAL_FILE_LEN];
     // remove file

source/os/src/osFile.c

@@ -739,7 +739,6 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
   va_start(ap, format);
   vfprintf(pFile->fp, format, ap);
   va_end(ap);
-  fflush(pFile->fp);
 }
 
 bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; }

source/util/src/tpagedbuf.c

@@ -107,7 +107,7 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) {
 
 static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; }
 
-static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + 2; }
+static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + sizeof(SFilePage); }
 
 /**
  *   +--------------------------+-------------------+--------------+

tests/parallel_test/cases.task

@@ -231,7 +231,7 @@
 ,,y,script,./test.sh -f tsim/stream/windowClose.sim
 ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim
 ,,y,script,./test.sh -f tsim/stream/sliding.sim
-,,n,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim
+,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim
 ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim
 ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim
 ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim
@@ -278,8 +278,8 @@
 ,,y,script,./test.sh -f tsim/stable/values.sim
 ,,y,script,./test.sh -f tsim/stable/vnode3.sim
 ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim
-,,n,script,./test.sh -f tsim/sma/drop_sma.sim
+,,,script,./test.sh -f tsim/sma/drop_sma.sim
-,,n,script,./test.sh -f tsim/sma/sma_leak.sim
+,,y,script,./test.sh -f tsim/sma/sma_leak.sim
 ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
@@ -401,22 +401,22 @@
 
 #system test
 
-,,,system-test,python3 ./test.py -f 0-others/taosShell.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py
-,,,system-test,python3 ./test.py -f 0-others/taosShellError.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py
-,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py
-,,,system-test,python3 ./test.py -f 0-others/telemetry.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py
-,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py
-,,,system-test,python3 ./test.py -f 0-others/udfTest.py
+,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
-,,,system-test,python3 ./test.py -f 0-others/udf_create.py
+,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
-,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py
+,,n,system-test,python3 ./test.py -f 0-others/udf_create.py
-,,,system-test,python3 ./test.py -f 0-others/cachemodel.py
+,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py
-,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py
+,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py
-,,,system-test,python3 ./test.py -f 0-others/udf_cfg2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py
-,,,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py
-,,,system-test,python3 ./test.py -f 0-others/sysinfo.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sysinfo.py
-,,,system-test,python3 ./test.py -f 0-others/user_control.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py
-,,,system-test,python3 ./test.py -f 0-others/fsync.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py
-,,,system-test,python3 ./test.py -f 0-others/compatibility.py
+,,n,system-test,python3 ./test.py -f 0-others/compatibility.py
 ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py
 ,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
@@ -424,20 +424,20 @@
 ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
 ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py
+#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py
-,,,system-test,python3 ./test.py -f 1-insert/boundary.py
+,,n,system-test,python3 ./test.py -f 1-insert/boundary.py
-,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
+,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
-,,,system-test,python3 ./test.py -f 1-insert/table_comment.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py
 ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py
 ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py
 ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py
-,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/mutil_stage.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py
 ,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py
-,,,system-test,python3 ./test.py -f 1-insert/InsertFuturets.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R
@@ -451,8 +451,8 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -R
-,,,system-test,python3 ./test.py -f 2-query/avg.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py
-,,,system-test,python3 ./test.py -f 2-query/avg.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py
@@ -465,12 +465,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R
-,,,system-test,python3 ./test.py -f 2-query/concat.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py
-,,,system-test,python3 ./test.py -f 2-query/concat.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R
-,,,system-test,python3 ./test.py -f 2-query/concat_ws.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py
-,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -R
-,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py
-,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py
@@ -515,14 +515,14 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -R
-,,,system-test,python3 ./test.py -f 2-query/interp.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py
-,,,system-test,python3 ./test.py -f 2-query/interp.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -R
-,,,system-test,python3 ./test.py -f 2-query/last_row.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py
-,,,system-test,python3 ./test.py -f 2-query/last_row.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py
@@ -531,10 +531,10 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R
-,,,system-test,python3 ./test.py -f 2-query/lower.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py
-,,,system-test,python3 ./test.py -f 2-query/lower.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -R
-,,,system-test,python3 ./test.py -f 2-query/ltrim.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py
-,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py
@@ -543,8 +543,8 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -R
-,,,system-test,python3 ./test.py -f 2-query/mode.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py
-,,,system-test,python3 ./test.py -f 2-query/mode.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py
@@ -555,16 +555,16 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -R
-,,,system-test,python3 ./test.py -f 2-query/rtrim.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py
-,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R
-,,,system-test,python3 ./test.py -f 2-query/smaTest.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py
-,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R
-,,,system-test,python3 ./test.py -f 2-query/sml.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py
-,,,system-test,python3 ./test.py -f 2-query/sml.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py
@@ -573,12 +573,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -R
-,,,system-test,python3 ./test.py -f 2-query/substr.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py
-,,,system-test,python3 ./test.py -f 2-query/substr.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -R
-,,,system-test,python3 ./test.py -f 2-query/tail.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py
-,,,system-test,python3 ./test.py -f 2-query/tail.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py
@@ -595,31 +595,33 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R
-,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py
-,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -R
-,,,system-test,python3 ./test.py -f 2-query/unique.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py
-,,,system-test,python3 ./test.py -f 2-query/unique.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R
-,,,system-test,python3 ./test.py -f 1-insert/update_data.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py
 ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py
-,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py
-,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py
-,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py
+,,,system-test,python3 ./test.py -f 1-insert/drop.py
+,,,system-test,python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py
-,,,system-test,python3 ./test.py -f 2-query/union1.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py
-,,,system-test,python3 ./test.py -f 2-query/concat2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py
 ,,,system-test,python3 ./test.py -f 2-query/json_tag.py
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py
@@ -704,6 +706,7 @@
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
+,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
@@ -724,11 +727,13 @@
 ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
 ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py
 ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
+,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 
+,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py  -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 2
@@ -736,13 +741,13 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 2
@@ -753,7 +758,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 2
@@ -781,249 +786,254 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/interp.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2
 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/avg.py   -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/elapsed.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/csum.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/sample.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/function_diff.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/unique.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/statecount.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/tail.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py  -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/substr.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/union.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/union1.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/concat.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/spread.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/mode.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q  3
+,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3
 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/avg.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/csum.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/sample.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/unique.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/tail.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q  3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/join2.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/substr.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/union.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/union1.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/concat.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/spread.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/mode.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4
 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/avg.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/csum.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/sample.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/unique.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/tail.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4
-,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q  4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4
 
 #develop test
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R
-,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
+,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
 
 #docs-examples test
-,,,docs-examples-test,bash python.sh
+,,n,docs-examples-test,bash python.sh
-,,,docs-examples-test,bash node.sh
+,,n,docs-examples-test,bash node.sh
-,,,docs-examples-test,bash csharp.sh
+,,n,docs-examples-test,bash csharp.sh
-,,,docs-examples-test,bash jdbc.sh
+,,n,docs-examples-test,bash jdbc.sh
-,,,docs-examples-test,bash go.sh
+,,n,docs-examples-test,bash go.sh

tests/pytest/auto_crash_gen.py

@@ -0,0 +1,365 @@
+import os
+import socket
+import requests
+
+# -*- coding: utf-8 -*-
+import os ,sys
+import random
+import argparse
+import subprocess
+import time
+import platform
+
+# valgrind mode ?
+valgrind_mode =  False
+
+msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" }
+
+# formal
+hostname = socket.gethostname()
+
+group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
+
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Crash_gen Monitor",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+
+# set path about run instance
+
+core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+core_path = "/".join(core_path.split("/")[:-1])
+print(" =======   core path is %s ======== " %core_path)
+if not os.path.exists(core_path):
+    os.mkdir(core_path)
+
+base_dir = os.path.dirname(os.path.realpath(__file__))
+if base_dir.find("community")>0:
+    repo = "community"
+elif base_dir.find("TDengine")>0:
+    repo = "TDengine"
+else:
+    repo ="TDengine"
+print("base_dir:",base_dir)
+home_dir = base_dir[:base_dir.find(repo)]
+print("home_dir:",home_dir)
+run_dir = os.path.join(home_dir,'run_dir')
+run_dir = os.path.abspath(run_dir)
+print("run dir is *** :",run_dir)
+if not os.path.exists(run_dir):
+    os.mkdir(run_dir)
+run_log_file = run_dir+'/crash_gen_run.log'
+crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh')
+exit_status_logs =  os.path.join(run_dir, 'crash_exit.log')
+
+def get_path():
+    buildPath=''
+    selfPath = os.path.dirname(os.path.realpath(__file__))
+    if ("community" in selfPath):
+        projPath = selfPath[:selfPath.find("community")]
+    else:
+        projPath = selfPath[:selfPath.find("tests")]
+
+    for root, dirs, files in os.walk(projPath):
+        if ("taosd" in files):
+            rootRealPath = os.path.dirname(os.path.realpath(root))
+            if ("packaging" not in rootRealPath):
+                buildPath = root[:len(root) - len("/build/bin")]
+                break
+    return buildPath
+
+# generate crash_gen start script randomly
+
+def random_args(args_list):
+    nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"
+    ]  # record bool type arguments
+    strs_args_list = ["--connector-type"]  # record str type arguments
+
+    args_list["--auto-start-service"]= False
+    args_list["--continue-on-exception"]=True
+    # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test"
+    connect_types=['native']
+    # args_list["--connector-type"]=connect_types[random.randint(0,2)]
+    args_list["--connector-type"]= connect_types[0]
+    args_list["--max-dbs"]= random.randint(1,10)
+    
+    # dnodes = [1,3] # set single dnodes;
+    
+    # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
+    # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
+    args_list["--debug"]=False
+    args_list["--per-thread-db-connection"]=True
+    args_list["--track-memory-leaks"]=False
+
+    args_list["--max-steps"]=random.randint(500,2000)
+
+    # args_list["--ignore-errors"]=[]   ## can add error codes for detail
+
+    
+    args_list["--run-tdengine"]= False
+    args_list["--use-shadow-db"]= False
+    args_list["--dynamic-db-table-names"]= True
+    args_list["--verify-data"]= False
+    args_list["--record-ops"] = False
+
+    for key in bools_args_list:
+        set_bool_value = [True,False]
+        if key == "--auto-start-service" :
+            continue
+        elif key =="--run-tdengine":
+            continue
+        elif key == "--ignore-errors":
+            continue
+        elif key == "--debug":
+            continue
+        elif key == "--per-thread-db-connection":
+            continue
+        elif key == "--continue-on-exception":
+            continue
+        elif key == "--use-shadow-db":
+            continue
+        elif key =="--track-memory-leaks":
+            continue
+        elif key == "--dynamic-db-table-names":
+            continue
+        elif key == "--verify-data":
+            continue
+        elif key == "--record-ops":
+            continue
+        else:
+            args_list[key]=set_bool_value[random.randint(0,1)]
+
+    if args_list["--larger-data"]:
+        threads = [16,32]
+    else:
+        threads = [32,64,128,256] 
+    args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
+
+    return args_list
+
+def limits(args_list):
+    if args_list["--use-shadow-db"]==True:
+        if args_list["--max-dbs"] > 1:
+            print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1")
+            args_list["--max-dbs"]=1
+        else:
+            pass
+
+    # env is start by test frame , not crash_gen instance
+    
+    # elif args_list["--num-replicas"]==0:
+    #     print(" make sure num-replicas is at least 1 ")
+    #     args_list["--num-replicas"]=1
+    # elif args_list["--num-replicas"]==1:
+    #     pass
+
+    # elif args_list["--num-replicas"]>1:
+    #     if not args_list["--auto-start-service"]:
+    #         print("it should be deployed by crash_gen auto-start-service for multi replicas")
+            
+    # else:
+    #     pass
+         
+    return  args_list
+
+def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"]
+    arguments = ""
+    for k ,v in args_list.items():
+        if k == "--ignore-errors":
+            if v:
+                arguments+=(k+"="+str(v)+" ")
+            else:
+                arguments+=""
+        elif  k in bools_args_list and v==True:
+            arguments+=(k+" ")
+        elif k in bools_args_list and v==False:
+            arguments+=""
+        else:
+            arguments+=(k+"="+str(v)+" ")
+    
+    if valgrind :
+         
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203 '%(crash_gen_path ,arguments)
+
+    else:
+
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203'%(crash_gen_path ,arguments)
+
+    return crash_gen_cmd
+
+def start_taosd():
+    build_path = get_path()
+    if repo == "community":
+        start_path = build_path[:-5]+"community/tests/system-test/"
+    elif repo == "TDengine":
+        start_path = build_path[:-5]+"/tests/system-test/"
+    else:
+        pass
+
+    start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path)
+    os.system(start_cmd)
+
+def get_cmds(args_list):
+    # build_path = get_path()
+    # if repo == "community":
+    #     crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    # elif repo == "TDengine":
+    #     crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    # else:
+    #     pass
+
+    # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550  '%(crash_gen_path)
+
+    crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
+    return crash_gen_cmd
+
+def run_crash_gen(crash_cmds):
+
+    # prepare env of taosd
+    start_taosd()
+
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    result_file = os.path.join(crash_gen_path, 'valgrind.out')
+
+
+    # run crash_gen and back logs
+    os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file))
+    os.system("%s >>%s "%(crash_cmds,result_file))
+
+
+def check_status():
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    result_file = os.path.join(crash_gen_path, 'valgrind.out')
+    run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs))
+
+    core_check = subprocess.Popen('ls -l  %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+
+    if int(core_check.strip().rstrip()) > 0:
+        # it means core files has occured
+        return 3
+    
+    if "Crash_Gen is now exiting with status code: 1" in run_code:
+        return 1
+    elif "Crash_Gen is now exiting with status code: 0" in run_code:
+        return 0
+    else:
+        return 2 
+
+
+def main():
+
+    args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
+    "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
+    "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , 
+    "--continue-on-exception":False }
+
+    args = random_args(args_list)
+    args = limits(args)
+
+
+    build_path = get_path()
+    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git")
+    if repo =="community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo =="TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    
+    if os.path.exists(crash_gen_path+"crash_gen.sh"):
+        print(" make sure crash_gen.sh is ready")
+    else:
+        print( " crash_gen.sh is not exists ")
+        sys.exit(1)
+    
+    git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16]
+    
+    # crash_cmds = get_cmds()
+    
+    crash_cmds = get_cmds(args)
+    # clean run_dir
+    os.system('rm -rf %s'%run_dir )
+    if not os.path.exists(run_dir):
+        os.mkdir(run_dir)
+    print(crash_cmds)
+    run_crash_gen(crash_cmds)
+    status = check_status()
+    
+    print("exit status : ", status)
+    
+    if status ==4:
+        print('======== crash_gen found memory bugs   ========')
+    if status ==5:
+        print('======== crash_gen found memory errors   ========')
+    if status >0:
+        print('======== crash_gen run failed and not exit as expected ========')
+    else:
+        print('======== crash_gen run sucess and exit as expected ========')
+
+
+    if status!=0 :
+
+        try:
+            text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n  " + f" and git commit :  {git_commit}"
+            send_msg(get_msg(text))  
+        except Exception as e:
+            print("exception:", e)
+        exit(status)
+        
+
+if __name__ == '__main__':
+    main()
+
+

tests/pytest/auto_crash_gen_valgrind.py

@@ -0,0 +1,399 @@
+#!/usr/bin/python3
+
+
+import os
+import socket
+import requests
+
+# -*- coding: utf-8 -*-
+import os ,sys
+import random
+import argparse
+import subprocess
+import time
+import platform
+
+# valgrind mode ?
+valgrind_mode =  True
+
+msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" }
+
+# formal
+hostname = socket.gethostname()
+
+group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
+
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Crash_gen Monitor",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+
+# set path about run instance
+
+core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+core_path = "/".join(core_path.split("/")[:-1])
+print(" =======   core path is %s ======== " %core_path)
+if not os.path.exists(core_path):
+    os.mkdir(core_path)
+
+base_dir = os.path.dirname(os.path.realpath(__file__))
+if base_dir.find("community")>0:
+    repo = "community"
+elif base_dir.find("TDengine")>0:
+    repo = "TDengine"
+else:
+    repo ="TDengine"
+print("base_dir:",base_dir)
+home_dir = base_dir[:base_dir.find(repo)]
+print("home_dir:",home_dir)
+run_dir = os.path.join(home_dir,'run_dir')
+run_dir = os.path.abspath(run_dir)
+print("run dir is *** :",run_dir)
+if not os.path.exists(run_dir):
+    os.mkdir(run_dir)
+run_log_file = run_dir+'/crash_gen_run.log'
+crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh')
+exit_status_logs =  os.path.join(run_dir, 'crash_exit.log')
+
+def get_path():
+    buildPath=''
+    selfPath = os.path.dirname(os.path.realpath(__file__))
+    if ("community" in selfPath):
+        projPath = selfPath[:selfPath.find("community")]
+    else:
+        projPath = selfPath[:selfPath.find("tests")]
+
+    for root, dirs, files in os.walk(projPath):
+        if ("taosd" in files):
+            rootRealPath = os.path.dirname(os.path.realpath(root))
+            if ("packaging" not in rootRealPath):
+                buildPath = root[:len(root) - len("/build/bin")]
+                break
+    return buildPath
+
+# generate crash_gen start script randomly
+
+def random_args(args_list):
+    nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"
+    ]  # record bool type arguments
+    strs_args_list = ["--connector-type"]  # record str type arguments
+
+    args_list["--auto-start-service"]= False
+    args_list["--continue-on-exception"]=True
+    # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test"
+    connect_types=['native']
+    # args_list["--connector-type"]=connect_types[random.randint(0,2)]
+    args_list["--connector-type"]= connect_types[0]
+    args_list["--max-dbs"]= random.randint(1,10)
+    
+    # dnodes = [1,3] # set single dnodes;
+    
+    # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
+    # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
+    args_list["--debug"]=False
+    args_list["--per-thread-db-connection"]=True
+    args_list["--track-memory-leaks"]=False
+
+    args_list["--max-steps"]=random.randint(200,500)
+    
+    threads = [16,32]
+
+    args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
+    # args_list["--ignore-errors"]=[]   ## can add error codes for detail
+
+    
+    args_list["--run-tdengine"]= False
+    args_list["--use-shadow-db"]= False
+    args_list["--dynamic-db-table-names"]= True
+    args_list["--verify-data"]= False
+    args_list["--record-ops"] = False
+
+    for key in bools_args_list:
+        set_bool_value = [True,False]
+        if key == "--auto-start-service" :
+            continue
+        elif key =="--run-tdengine":
+            continue
+        elif key == "--ignore-errors":
+            continue
+        elif key == "--debug":
+            continue
+        elif key == "--per-thread-db-connection":
+            continue
+        elif key == "--continue-on-exception":
+            continue
+        elif key == "--use-shadow-db":
+            continue
+        elif key =="--track-memory-leaks":
+            continue
+        elif key == "--dynamic-db-table-names":
+            continue
+        elif key == "--verify-data":
+            continue
+        elif key == "--record-ops":
+            continue
+        elif key == "--larger-data":
+            continue
+        else:
+            args_list[key]=set_bool_value[random.randint(0,1)]
+    return args_list
+
+def limits(args_list):
+    if args_list["--use-shadow-db"]==True:
+        if args_list["--max-dbs"] > 1:
+            print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1")
+            args_list["--max-dbs"]=1
+        else:
+            pass
+
+    # env is start by test frame , not crash_gen instance
+    
+    # elif args_list["--num-replicas"]==0:
+    #     print(" make sure num-replicas is at least 1 ")
+    #     args_list["--num-replicas"]=1
+    # elif args_list["--num-replicas"]==1:
+    #     pass
+
+    # elif args_list["--num-replicas"]>1:
+    #     if not args_list["--auto-start-service"]:
+    #         print("it should be deployed by crash_gen auto-start-service for multi replicas")
+            
+    # else:
+    #     pass
+         
+    return  args_list
+
+def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"]
+    arguments = ""
+    for k ,v in args_list.items():
+        if k == "--ignore-errors":
+            if v:
+                arguments+=(k+"="+str(v)+" ")
+            else:
+                arguments+=""
+        elif  k in bools_args_list and v==True:
+            arguments+=(k+" ")
+        elif k in bools_args_list and v==False:
+            arguments+=""
+        else:
+            arguments+=(k+"="+str(v)+" ")
+    
+    if valgrind :
+         
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203 '%(crash_gen_path ,arguments)
+
+    else:
+
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203'%(crash_gen_path ,arguments)
+
+    return crash_gen_cmd
+
+
+def start_taosd():
+    build_path = get_path()
+    if repo == "community":
+        start_path = build_path[:-5]+"community/tests/system-test/"
+    elif repo == "TDengine":
+        start_path = build_path[:-5]+"/tests/system-test/"
+    else:
+        pass
+
+    start_cmd = 'cd %s && python3 test.py  '%(start_path)
+    os.system(start_cmd +">>/dev/null")
+
+def get_cmds(args_list):
+    # build_path = get_path()
+    # if repo == "community":
+    #     crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    # elif repo == "TDengine":
+    #     crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    # else:
+    #     pass
+
+    # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550  '%(crash_gen_path)
+
+    crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
+    return crash_gen_cmd
+
+def run_crash_gen(crash_cmds):
+
+    # prepare env of taosd
+    start_taosd()
+    # run crash_gen and back logs
+    os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file))
+    # os.system("cp %s %s"%(crash_gen_cmds_file, core_path))
+    os.system("%s"%(crash_cmds))
+
+def check_status():
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    result_file = os.path.join(crash_gen_path, 'valgrind.out')
+    run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs))
+
+    core_check = subprocess.Popen('ls -l  %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+
+    if int(core_check.strip().rstrip()) > 0:
+        # it means core files has occured
+        return 3
+    
+    mem_status = check_memory()
+    if mem_status >0:
+        return mem_status
+    if "Crash_Gen is now exiting with status code: 1" in run_code:
+        return 1
+    elif "Crash_Gen is now exiting with status code: 0" in run_code:
+        return 0
+    else:
+        return 2 
+    
+
+def check_memory():
+
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    '''
+    invalid read, invalid write
+    '''
+    back_path = os.path.join(core_path,"valgrind_report")
+    if not os.path.exists(back_path):
+        os.mkdir(back_path)
+
+    stderr_file = os.path.join(crash_gen_path , "valgrind.err")
+    
+    status = 0
+
+    grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 4
+    
+    grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 4
+        
+    grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 5
+    
+    return status
+
+def main():
+
+    args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
+    "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
+    "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , 
+    "--continue-on-exception":False }
+
+    args = random_args(args_list)
+    args = limits(args)
+
+    build_path = get_path()
+    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null")
+    if repo =="community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo =="TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    
+    if os.path.exists(crash_gen_path+"crash_gen.sh"):
+        print(" make sure crash_gen.sh is ready")
+    else:
+        print( " crash_gen.sh is not exists ")
+        sys.exit(1)
+    
+    git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16]
+
+    # crash_cmds = get_cmds()
+    
+    crash_cmds = get_cmds(args)
+
+    # clean run_dir
+    os.system('rm -rf %s'%run_dir )
+    if not os.path.exists(run_dir):
+        os.mkdir(run_dir)
+    print(crash_cmds)
+    run_crash_gen(crash_cmds)
+    status = check_status()
+    # back_path = os.path.join(core_path,"valgrind_report")
+    
+    print("exit status : ", status)
+    
+    if status ==4:
+        print('======== crash_gen found memory bugs   ========')
+    if status ==5:
+        print('======== crash_gen found memory errors   ========')
+    if status >0:
+        print('======== crash_gen run failed and not exit as expected ========')
+    else:
+        print('======== crash_gen run sucess and exit as expected ========')
+
+    if status!=0 :
+
+        try:
+            text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n  " + f" and git commit : {git_commit}"
+            send_msg(get_msg(text))  
+        except Exception as e:
+            print("exception:", e)
+        exit(status)
+        
+
+if __name__ == '__main__':
+    main()
+
+

tests/pytest/auto_crash_gen_valgrind_cluster.py

@@ -0,0 +1,399 @@
+#!/usr/bin/python3
+
+
+import os
+import socket
+import requests
+
+# -*- coding: utf-8 -*-
+import os ,sys
+import random
+import argparse
+import subprocess
+import time
+import platform
+
+# valgrind mode ?
+valgrind_mode =  True
+
+msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" }
+
+# formal
+hostname = socket.gethostname()
+
+group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
+
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Crash_gen Monitor",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+
+# set path about run instance
+
+core_path = subprocess.Popen("cat /proc/sys/kernel/core_pattern", shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+core_path = "/".join(core_path.split("/")[:-1])
+print(" =======   core path is %s ======== " %core_path)
+if not os.path.exists(core_path):
+    os.mkdir(core_path)
+
+base_dir = os.path.dirname(os.path.realpath(__file__))
+if base_dir.find("community")>0:
+    repo = "community"
+elif base_dir.find("TDengine")>0:
+    repo = "TDengine"
+else:
+    repo ="TDengine"
+print("base_dir:",base_dir)
+home_dir = base_dir[:base_dir.find(repo)]
+print("home_dir:",home_dir)
+run_dir = os.path.join(home_dir,'run_dir')
+run_dir = os.path.abspath(run_dir)
+print("run dir is *** :",run_dir)
+if not os.path.exists(run_dir):
+    os.mkdir(run_dir)
+run_log_file = run_dir+'/crash_gen_run.log'
+crash_gen_cmds_file = os.path.join(run_dir, 'crash_gen_cmds.sh')
+exit_status_logs =  os.path.join(run_dir, 'crash_exit.log')
+
+def get_path():
+    buildPath=''
+    selfPath = os.path.dirname(os.path.realpath(__file__))
+    if ("community" in selfPath):
+        projPath = selfPath[:selfPath.find("community")]
+    else:
+        projPath = selfPath[:selfPath.find("tests")]
+
+    for root, dirs, files in os.walk(projPath):
+        if ("taosd" in files):
+            rootRealPath = os.path.dirname(os.path.realpath(root))
+            if ("packaging" not in rootRealPath):
+                buildPath = root[:len(root) - len("/build/bin")]
+                break
+    return buildPath
+
+# generate crash_gen start script randomly
+
+def random_args(args_list):
+    nums_args_list = ["--max-dbs","--num-replicas","--num-dnodes","--max-steps","--num-threads",] # record int type arguments
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"
+    ]  # record bool type arguments
+    strs_args_list = ["--connector-type"]  # record str type arguments
+
+    args_list["--auto-start-service"]= False
+    args_list["--continue-on-exception"]=True
+    # connect_types=['native','rest','mixed'] # restful interface has change ,we should trans dbnames to connection or change sql such as "db.test"
+    connect_types=['native']
+    # args_list["--connector-type"]=connect_types[random.randint(0,2)]
+    args_list["--connector-type"]= connect_types[0]
+    args_list["--max-dbs"]= random.randint(1,10)
+    
+    # dnodes = [1,3] # set single dnodes;
+    
+    # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
+    # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
+    args_list["--debug"]=False
+    args_list["--per-thread-db-connection"]=True
+    args_list["--track-memory-leaks"]=False
+
+    args_list["--max-steps"]=random.randint(200,500)
+    
+    threads = [16,32]
+
+    args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
+    # args_list["--ignore-errors"]=[]   ## can add error codes for detail
+
+    
+    args_list["--run-tdengine"]= False
+    args_list["--use-shadow-db"]= False
+    args_list["--dynamic-db-table-names"]= True
+    args_list["--verify-data"]= False
+    args_list["--record-ops"] = False
+
+    for key in bools_args_list:
+        set_bool_value = [True,False]
+        if key == "--auto-start-service" :
+            continue
+        elif key =="--run-tdengine":
+            continue
+        elif key == "--ignore-errors":
+            continue
+        elif key == "--debug":
+            continue
+        elif key == "--per-thread-db-connection":
+            continue
+        elif key == "--continue-on-exception":
+            continue
+        elif key == "--use-shadow-db":
+            continue
+        elif key =="--track-memory-leaks":
+            continue
+        elif key == "--dynamic-db-table-names":
+            continue
+        elif key == "--verify-data":
+            continue
+        elif key == "--record-ops":
+            continue
+        elif key == "--larger-data":
+            continue
+        else:
+            args_list[key]=set_bool_value[random.randint(0,1)]
+    return args_list
+
+def limits(args_list):
+    if args_list["--use-shadow-db"]==True:
+        if args_list["--max-dbs"] > 1:
+            print("Cannot combine use-shadow-db with max-dbs of more than 1 ,set max-dbs=1")
+            args_list["--max-dbs"]=1
+        else:
+            pass
+
+    # env is start by test frame , not crash_gen instance
+    
+    # elif args_list["--num-replicas"]==0:
+    #     print(" make sure num-replicas is at least 1 ")
+    #     args_list["--num-replicas"]=1
+    # elif args_list["--num-replicas"]==1:
+    #     pass
+
+    # elif args_list["--num-replicas"]>1:
+    #     if not args_list["--auto-start-service"]:
+    #         print("it should be deployed by crash_gen auto-start-service for multi replicas")
+            
+    # else:
+    #     pass
+         
+    return  args_list
+
+def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+
+    bools_args_list = ["--auto-start-service" , "--debug","--run-tdengine","--ignore-errors","--track-memory-leaks","--larger-data","--mix-oos-data","--dynamic-db-table-names",
+    "--per-thread-db-connection","--record-ops","--verify-data","--use-shadow-db","--continue-on-exception"]
+    arguments = ""
+    for k ,v in args_list.items():
+        if k == "--ignore-errors":
+            if v:
+                arguments+=(k+"="+str(v)+" ")
+            else:
+                arguments+=""
+        elif  k in bools_args_list and v==True:
+            arguments+=(k+" ")
+        elif k in bools_args_list and v==False:
+            arguments+=""
+        else:
+            arguments+=(k+"="+str(v)+" ")
+    
+    if valgrind :
+         
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  --valgrind   -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707,0x0203 '%(crash_gen_path ,arguments)
+
+    else:
+
+        crash_gen_cmd = 'cd %s && ./crash_gen.sh  -i 3 %s  -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707,0x0203'%(crash_gen_path ,arguments)
+
+    return crash_gen_cmd
+
+
+def start_taosd():
+    build_path = get_path()
+    if repo == "community":
+        start_path = build_path[:-5]+"community/tests/system-test/"
+    elif repo == "TDengine":
+        start_path = build_path[:-5]+"/tests/system-test/"
+    else:
+        pass
+
+    start_cmd = 'cd %s && python3 test.py -N 4 -M 1 '%(start_path)
+    os.system(start_cmd +">>/dev/null")
+
+def get_cmds(args_list):
+    # build_path = get_path()
+    # if repo == "community":
+    #     crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    # elif repo == "TDengine":
+    #     crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    # else:
+    #     pass
+
+    # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550  '%(crash_gen_path)
+
+    crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
+    return crash_gen_cmd
+
+def run_crash_gen(crash_cmds):
+
+    # prepare env of taosd
+    start_taosd()
+    # run crash_gen and back logs
+    os.system('echo "%s">>%s'%(crash_cmds,crash_gen_cmds_file))
+    # os.system("cp %s %s"%(crash_gen_cmds_file, core_path))
+    os.system("%s"%(crash_cmds))
+
+def check_status():
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    result_file = os.path.join(crash_gen_path, 'valgrind.out')
+    run_code = subprocess.Popen("tail -n 50 %s"%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs))
+
+    core_check = subprocess.Popen('ls -l  %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+
+    if int(core_check.strip().rstrip()) > 0:
+        # it means core files has occured
+        return 3
+    
+    mem_status = check_memory()
+    if mem_status >0:
+        return mem_status
+    if "Crash_Gen is now exiting with status code: 1" in run_code:
+        return 1
+    elif "Crash_Gen is now exiting with status code: 0" in run_code:
+        return 0
+    else:
+        return 2 
+    
+
+def check_memory():
+
+    build_path = get_path()
+    if repo == "community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo == "TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    '''
+    invalid read, invalid write
+    '''
+    back_path = os.path.join(core_path,"valgrind_report")
+    if not os.path.exists(back_path):
+        os.mkdir(back_path)
+
+    stderr_file = os.path.join(crash_gen_path , "valgrind.err")
+    
+    status = 0
+
+    grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 4
+    
+    grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 4
+        
+    grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+    
+    if grep_res:
+        # os.system("cp %s %s"%(stderr_file , back_path))
+        status = 5
+    
+    return status
+
+def main():
+
+    args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
+    "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
+    "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , 
+    "--continue-on-exception":False }
+
+    args = random_args(args_list)
+    args = limits(args)
+
+    build_path = get_path()
+    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null")
+    if repo =="community":
+        crash_gen_path = build_path[:-5]+"community/tests/pytest/"
+    elif repo =="TDengine":
+        crash_gen_path = build_path[:-5]+"/tests/pytest/"
+    else:
+        pass
+    
+    if os.path.exists(crash_gen_path+"crash_gen.sh"):
+        print(" make sure crash_gen.sh is ready")
+    else:
+        print( " crash_gen.sh is not exists ")
+        sys.exit(1)
+    
+    git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16]
+
+    # crash_cmds = get_cmds()
+    
+    crash_cmds = get_cmds(args)
+
+    # clean run_dir
+    os.system('rm -rf %s'%run_dir )
+    if not os.path.exists(run_dir):
+        os.mkdir(run_dir)
+    print(crash_cmds)
+    run_crash_gen(crash_cmds)
+    status = check_status()
+    # back_path = os.path.join(core_path,"valgrind_report")
+    
+    print("exit status : ", status)
+    
+    if status ==4:
+        print('======== crash_gen found memory bugs   ========')
+    if status ==5:
+        print('======== crash_gen found memory errors   ========')
+    if status >0:
+        print('======== crash_gen run failed and not exit as expected ========')
+    else:
+        print('======== crash_gen run sucess and exit as expected ========')
+
+    if status!=0 :
+
+        try:
+            text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n  " + f" and git commit : {git_commit}"
+            send_msg(get_msg(text))  
+        except Exception as e:
+            print("exception:", e)
+        exit(status)
+        
+
+if __name__ == '__main__':
+    main()
+
+

tests/pytest/auto_run_regular.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# set LD_LIBRARY_PATH
+export PATH=$PATH:/home/TDengine/debug/build/bin
+export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
+ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null
+
+# run crash_gen auto script
+python3 /home/TDengine/tests/pytest/auto_crash_gen.py

tests/pytest/auto_run_valgrind.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# set LD_LIBRARY_PATH
+export PATH=$PATH:/home/TDengine/debug/build/bin
+export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
+ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null
+
+# run crash_gen auto script
+python3 /home/TDengine/tests/pytest/auto_crash_gen_valgrind.py

tests/pytest/auto_run_valgrind_cluster.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# set LD_LIBRARY_PATH
+export PATH=$PATH:/home/TDengine/debug/build/bin
+export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
+ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
+ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null
+
+# run crash_gen auto script
+python3 /home/TDengine/tests/pytest/auto_crash_gen_valgrind_cluster.py

tests/pytest/docker_exec_service.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+mkdir -p /data/wz/crash_gen_logs/
+logdir='/data/wz/crash_gen_logs/'
+date_tag=`date +%Y%m%d-%H%M%S`
+hostname='vm_valgrind_'
+
+for i in {1..50}
+do 
+        echo $i
+        # create docker and start crash_gen
+        log_dir=${logdir}${hostname}${date_tag}_${i}
+        docker run -d  --hostname=${hostname}${date_tag}_${i} --name ${hostname}${date_tag}_${i} --privileged  -v ${log_dir}:/corefile/ -- crash_gen:v1.0 sleep 99999999999999 
+                echo create docker ${hostname}${date_tag}_${i}
+                docker exec -d ${hostname}${date_tag}_${i} sh -c 'rm -rf /home/taos-connector-python'
+                docker cp /data/wz/TDengine ${hostname}${date_tag}_${i}:/home/TDengine
+                docker cp /data/wz/taos-connector-python ${hostname}${date_tag}_${i}:/home/taos-connector-python
+                echo copy TDengine in container done!
+        docker exec  ${hostname}${date_tag}_${i}  sh -c 'sh /home/TDengine/tests/pytest/auto_run_valgrind.sh '
+        if [ $? -eq 0 ] 
+        then
+                echo crash_gen exit as expect , run success
+
+                # # clear docker which success
+                docker stop ${hostname}${date_tag}_${i}
+                docker rm -f ${hostname}${date_tag}_${i}
+        else
+                docker stop ${hostname}${date_tag}_${i}
+                echo crash_gen exit error , run failed
+        fi
+done

tests/script/runAllSimCases.sh

@@ -11,6 +11,9 @@ set -e
 VALGRIND=0
 LOG_BK_DIR=/data/valgrind_log_backup     # 192.168.0.203
 SIM_FILES=./jenkins/basic.txt
+cases_task_file=../parallel_test/cases.task
+
+cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > $SIM_FILES
 
 while getopts "v:r:f:" arg
 do
@@ -21,9 +24,9 @@ do
     r)
       LOG_BK_DIR=$(echo $OPTARG)
       ;;
-    f)
+    #f)
-      SIM_FILES=$(echo $OPTARG)
+    #  SIM_FILES=$(echo $OPTARG)
-      ;;
+    #  ;;
     ?) #unknow option
       echo "unkonw argument"
       exit 1

tests/script/sh/checkAsan.sh

@@ -21,15 +21,24 @@ LOG_DIR=$TAOS_DIR/sim/asan
 error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l`
 memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l`
 indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l`
+python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l`
 
 # ignore
+
+# TD-20368
+# /root/TDengine/contrib/zlib/trees.c:873:5: runtime error: null pointer passed as argument 2, which is declared to never be null
+
+# TD-20494 TD-20452
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int'
 # /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int'
-runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l`
 
-python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l`
+# TD-20569  
+# /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int'
+# /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int'
+
+runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l`
 
 echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m"
 echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m"

tests/script/sh/stop_dnodes.sh

@@ -26,3 +26,17 @@ while [ -n "$PID" ]; do
   fi
   PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
 done
+
+PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'`
+while [ -n "$PID" ]; do
+  echo kill -9 $PID
+  #pkill -9 taosd
+  kill -9 $PID
+  echo "Killing taosd processes"
+  if [ "$OS_TYPE" != "Darwin" ]; then
+    fuser -k -n tcp 6030
+  else
+    lsof -nti:6030 | xargs kill -9
+  fi
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+done

tests/script/tsim/stream/state0.sim

@@ -731,5 +731,65 @@ if $data32 != 1 then
   goto loop9
 endi
 
+sql drop stream if exists streams5;
+sql drop database if exists test5;
+sql create database test5;
+sql use test5;
+sql create table tb (ts timestamp, a int);
+sql insert into tb values (now + 1m , 1 );
+sql create table b (c timestamp, d int, e int , f int, g double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a);
+sql insert into b values(1648791213000,NULL,NULL,NULL,NULL);
+sql select * from streamt order by c1, c2, c3;
+
+print data00:$data00
+print data01:$data01
+
+sql insert into b values(1648791213000,NULL,NULL,NULL,NULL);
+sql select * from streamt order by c1, c2, c3;
+
+print data00:$data00
+print data01:$data01
+
+sql insert into b values(1648791213001,1,2,2,2.0);
+sql insert into b values(1648791213002,1,3,3,3.0);
+sql insert into tb values(1648791213003,1);
+
+sql select * from streamt;
+print data00:$data00
+print data01:$data01
+
+sql delete from b where c >= 1648791213001 and c <= 1648791213002;
+sql insert into b values(1648791223003,2,2,3,1.0); insert into b values(1648791223002,2,3,3,3.0);
+sql insert into tb values (now + 1m , 1 );
+
+sql select * from streamt;
+print data00:$data00
+print data01:$data01
+
+sql insert into b(c,d) values (now + 6m , 6 );
+sql delete from b where c >= 1648791213001 and c <= 1648791233005;;
+
+$loop_count = 0
+loop10:
+
+sleep 200
+
+sql select c2 from streamt;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $rows != 1 then
+  print =====rows=$rows
+  goto loop10
+endi
+
+if $data00 != 2 then
+  print =====data00=$data00
+  goto loop10
+endi
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/system-test/0-others/taosShellNetChk.py

@@ -230,9 +230,11 @@ class TDTestCase:
                 tdLog.exit('taos -n client fail!')
         finally:
             if platform.system().lower() == 'windows':
+                tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9")
                 os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
             else:
-                os.system('pkill -9 taos')
+                tdLog.info("pkill -9 taos")
+                # os.system('pkill -9 taos')
 
     def stop(self):
         tdSql.close()

tests/system-test/1-insert/alter_database.py

@@ -18,7 +18,7 @@ class TDTestCase:
         tdSql.init(conn.cursor(), logSql)
         self.buffer_boundary = [3, 4097, 8193, 12289, 16384]
         self.buffer_error = [self.buffer_boundary[0] -
-                             1, self.buffer_boundary[-1]+1, 12289, 256]
+                             1, self.buffer_boundary[-1]+1, 256]
         # pages_boundary >= 64
         self.pages_boundary = [64, 128, 512]
         self.pages_error = [self.pages_boundary[0]-1]
@@ -53,7 +53,7 @@ class TDTestCase:
         tdSql.execute('drop database db')
 
     def run(self):
-        tdSql.error('create database db1 vgroups 10 buffer 12289')
+        
         self.alter_buffer()
         self.alter_pages()
 

tests/system-test/1-insert/alter_table.py

@@ -87,7 +87,6 @@ class TDTestCase:
 
         }
     def alter_check_ntb(self):      
-        
         tdSql.prepare()
         tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
         for i in self.values_list:
@@ -107,6 +106,7 @@ class TDTestCase:
                 v = f'binary({self.binary_length+1})'
                 v_error = f'binary({self.binary_length-1})'
                 tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}')
+                tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"')
                 tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}')
                 tdSql.query(f'describe {self.ntbname}')
                 result = tdCom.getOneRow(1,'VARCHAR')
@@ -115,6 +115,7 @@ class TDTestCase:
                 v = f'nchar({self.binary_length+1})'
                 v_error = f'nchar({self.binary_length-1})'
                 tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}')
+                tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"')
                 tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}')
                 tdSql.query(f'describe {self.ntbname}')
                 result = tdCom.getOneRow(1,'NCHAR')
@@ -122,6 +123,7 @@ class TDTestCase:
             else:
                 for v in self.column_dict.values():
                     tdSql.error(f'alter table {self.ntbname} modify column {key} {v}')
+                    tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"')
         for key,values in self.column_dict.items():
             rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}'
             tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}')
@@ -285,6 +287,7 @@ class TDTestCase:
                 v = f'binary({self.binary_length+1})'
                 v_error = f'binary({self.binary_length-1})'
                 tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}')
+                tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"')
                 tdSql.execute(f'alter table {self.stbname} modify column {key} {v}')
                 tdSql.query(f'describe {self.stbname}')
                 result = tdCom.getOneRow(1,'VARCHAR')
@@ -297,6 +300,7 @@ class TDTestCase:
                 v = f'nchar({self.binary_length+1})'
                 v_error = f'nchar({self.binary_length-1})'
                 tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}')
+                tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"')
                 tdSql.execute(f'alter table {self.stbname} modify column {key} {v}')
                 tdSql.query(f'describe {self.stbname}')
                 result = tdCom.getOneRow(1,'NCHAR')
@@ -308,6 +312,7 @@ class TDTestCase:
             else:
                 for v in self.column_dict.values():
                     tdSql.error(f'alter table {self.stbname} modify column {key} {v}')
+                    tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"')
         for key,values in self.column_dict.items():
             rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}'
             tdSql.error(f'alter table {self.stbname} rename column {key} {rename_str}')

tests/system-test/1-insert/drop.py

@@ -0,0 +1,161 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+import math
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+        self.dbname = 'db'
+        self.ntbname = f"{self.dbname}.ntb"
+        self.rowNum = 10
+        self.tbnum = 20
+        self.ts = 1537146000000
+        self.binary_str = 'taosdata'
+        self.nchar_str = '涛思数据'
+        self.column_dict = {
+            'ts'  : 'timestamp',
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            'col11': 'bool',
+            'col12': 'binary(20)',
+            'col13': 'nchar(20)'
+        }
+    def insert_data(self,column_dict,tbname,row_num):
+        insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str)
+        for i in range(row_num):
+            insert_list = []
+            self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) 
+    def drop_ntb_check(self):
+        tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar}')
+        tdSql.execute(f'use {self.dbname}')
+        tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+        self.insert_data(self.column_dict,self.ntbname,self.rowNum)
+        for k,v in self.column_dict.items():
+            if v.lower() == "timestamp":
+                tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}')
+                tdSql.checkRows(1)
+        tdSql.execute(f'drop table {self.ntbname}')
+        tdSql.execute(f'flush database {self.dbname}')
+        tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
+        self.insert_data(self.column_dict,self.ntbname,self.rowNum)
+        for k,v in self.column_dict.items():
+            if v.lower() == "timestamp":
+                tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}')
+                tdSql.checkRows(1)
+        tdSql.execute(f'drop database {self.dbname}')
+    
+    def drop_stb_ctb_check(self):
+        stbname = f'{self.dbname}.{tdCom.getLongName(5,"letters")}'
+        tag_dict = {
+            't0':'int'
+        }
+        tag_values = [
+            f'1'
+            ]
+        tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar}")
+        tdSql.execute(f'use {self.dbname}')
+        tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+        for i in range(self.tbnum):
+            tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})")
+            self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)
+        for k,v in self.column_dict.items():
+            for i in range(self.tbnum):
+                if v.lower() == "timestamp":
+                    tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}')
+                    tdSql.checkRows(1)
+                    tdSql.execute(f'drop table {stbname}_{i}')
+        tdSql.execute(f'flush database {self.dbname}')
+        for i in range(self.tbnum):
+            tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})")
+            self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)
+        for k,v in self.column_dict.items():
+            for i in range(self.tbnum):
+                if v.lower() == "timestamp":
+                    tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}')
+                    tdSql.checkRows(1)
+            if v.lower() == "timestamp":
+                tdSql.query(f'select * from {stbname} where {k} = {self.ts}')
+                tdSql.checkRows(self.tbnum) 
+        tdSql.execute(f'drop table {stbname}')
+        tdSql.execute(f'flush database {self.dbname}')
+        tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+        for i in range(self.tbnum):
+            tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})")
+            self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)
+        for k,v in self.column_dict.items():
+            if v.lower() == "timestamp":
+                tdSql.query(f'select * from {stbname} where {k} = {self.ts}')
+                tdSql.checkRows(self.tbnum) 
+        tdSql.execute(f'drop database {self.dbname}')
+    def drop_topic_check(self):
+        tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}')
+        tdSql.execute(f'use {self.dbname}')
+        stbname = tdCom.getLongName(5,"letters")
+        topic_name = tdCom.getLongName(5,"letters")
+        tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)')
+        tdSql.execute(f'create topic {topic_name} as select * from {self.dbname}.{stbname}')
+        tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"')
+        tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select * from {self.dbname}.{stbname}')
+        tdSql.execute(f'drop topic {topic_name}')
+        tdSql.execute(f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}')
+        tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"')
+        tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}')
+        tdSql.execute(f'drop topic {topic_name}')
+        tdSql.execute(f'drop database {self.dbname}')
+
+    def drop_stream_check(self):
+        tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}')
+        tdSql.execute(f'use {self.dbname}')
+        stbname = tdCom.getLongName(5,"letters")
+        stream_name = tdCom.getLongName(5,"letters")
+        tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)')
+        tdSql.execute(f'create table tb using {stbname} tags(1)')
+        tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname')
+        tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"')
+        print(tdSql.queryResult)
+        tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname')
+        tdSql.execute(f'drop stream {stream_name}')
+        tdSql.execute(f'create stream {stream_name} into stb1 as select * from tb')
+        tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"')
+        tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb1 as select * from tb')
+        tdSql.execute(f'drop database {self.dbname}')
+    def run(self):
+        self.drop_ntb_check()
+        self.drop_stb_ctb_check()
+        self.drop_topic_check()
+        self.drop_stream_check()
+        pass
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/test_stmt_muti_insert_query.py

@@ -165,6 +165,7 @@ class TDTestCase:
 
             # conn.execute("drop database if exists %s" % dbname)
             conn.close()
+            tdLog.success("%s successfully executed" % __file__)
 
         except Exception as err:
             # conn.execute("drop database if exists %s" % dbname)

tests/system-test/1-insert/test_stmt_set_tbname_tag.py

@@ -239,6 +239,7 @@ class TDTestCase:
 
             # conn.execute("drop database if exists %s" % dbname)
             conn.close()
+            tdLog.success("%s successfully executed" % __file__)
 
         except Exception as err:
             # conn.execute("drop database if exists %s" % dbname)

tests/system-test/2-query/insert_select.py

@@ -0,0 +1,108 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import os
+import time
+import subprocess
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+class TDTestCase:
+    updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
+    
+    def init(self, conn, logSql, replicaVar):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.testcasePath = os.path.split(__file__)[0]
+        self.testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
+        
+        self.db = "insert_select"
+
+    def dropandcreateDB_random(self,database,n):
+        ts = 1604298064000
+
+        tdSql.execute('''drop database if exists %s ;''' %database)
+        tdSql.execute('''create database %s keep 36500 ;'''%(database))
+        tdSql.execute('''use %s;'''%database)
+
+        tdSql.execute('''create table %s.tb (ts timestamp , i tinyint );'''%database)
+        tdSql.execute('''create table %s.tb1 (speed timestamp , c1 int , c2 int , c3 int );'''%database)
+
+        sql_before = "insert into %s.tb1 values" %database
+        sql_value = ''
+        for i in range(n):         
+            sql_value = sql_value +"(%d, %d, %d, %d)"  % (ts + i, i, i, i)
+
+        sql = sql_before + sql_value
+        tdSql.execute(sql)
+
+        tdSql.query("select count(*) from %s.tb1;"%database)    
+        sql_result = tdSql.getData(0,0)
+        tdLog.info("result: %s" %(sql_result))
+        tdSql.query("reset query cache;")
+        tdSql.query("insert into %s.tb1 select * from %s.tb1;"%(database,database))        
+        tdSql.query("select count(*) from %s.tb1;"%database)
+        sql_result = tdSql.getData(0,0)
+        tdLog.info("result: %s" %(sql_result))
+        
+        
+    def users_bug_TD_20592(self,database):    
+        tdSql.execute('''drop database if exists %s ;''' %database)
+        tdSql.execute('''create database %s keep 36500 ;'''%(database))
+        tdSql.execute('''use %s;'''%database)
+
+        tdSql.execute('''create table %s.sav_stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 int);'''%database)
+        tdSql.execute('''create table %s.tb1 using %s.sav_stb tags( 9 , 0);'''%(database,database))
+        
+        tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database))
+
+    
+                            
+    def run(self):      
+        
+        startTime = time.time()  
+                  
+        os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) 
+        
+        self.dropandcreateDB_random("%s" %self.db, random.randint(10000,30000))
+            
+        #taos -f sql 
+        print("taos -f sql start!")
+        taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename)
+        _ = subprocess.check_output(taos_cmd1, shell=True)
+        print("taos -f sql over!")   
+        
+        self.users_bug_TD_20592("%s" %self.db) 
+        
+        #taos -f sql 
+        print("taos -f sql start!")
+        taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename)
+        _ = subprocess.check_output(taos_cmd1, shell=True)
+        print("taos -f sql over!")   
+
+        endTime = time.time()
+        print("total time %ds" % (endTime - startTime))
+    
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/2-query/select_index.py

@@ -26,7 +26,7 @@ from util.dnodes import *
 class TDTestCase:
     updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
     
-    def init(self, conn, logSql):
+    def init(self, conn, logSql, replicaVar):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
@@ -109,7 +109,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), 
+                        % (database,ts + i*1000+1, fake.random_int(min=-2147483647, max=2147483647, step=1), 
                         fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), 
                         fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -118,7 +118,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1) , 
+                        % (database,ts + i*1000+1, fake.random_int(min=-2147483647, max=2147483647, step=1) , 
                         fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) , 
                         fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -128,7 +128,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8)\
                         values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), 
+                        % (database,ts + i*1000+2, fake.random_int(min=0, max=2147483647, step=1), 
                         fake.random_int(min=0, max=9223372036854775807, step=1), 
                         fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -137,7 +137,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), 
+                        % (database,ts + i*1000+2, fake.random_int(min=0, max=2147483647, step=1), 
                         fake.random_int(min=0, max=9223372036854775807, step=1), 
                         fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -147,7 +147,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000, fake.random_int(min=-0, max=2147483647, step=1), 
+                        % (database,ts + i*1000+3, fake.random_int(min=-0, max=2147483647, step=1), 
                         fake.random_int(min=-0, max=9223372036854775807, step=1), 
                         fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -157,7 +157,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000 +1, fake.random_int(min=-0, max=2147483647, step=1), 
+                        % (database,ts + i*1000 +4, fake.random_int(min=-0, max=2147483647, step=1), 
                         fake.random_int(min=-0, max=9223372036854775807, step=1), 
                         fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -167,7 +167,7 @@ class TDTestCase:
                         q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \
                         values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \
                         'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' 
-                        % (database,ts + i*1000 +10, fake.random_int(min=-0, max=2147483647, step=1), 
+                        % (database,ts + i*1000 +5, fake.random_int(min=-0, max=2147483647, step=1), 
                         fake.random_int(min=-0, max=9223372036854775807, step=1), 
                         fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , 
                         fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , 
@@ -182,7 +182,7 @@ class TDTestCase:
         fake = Faker('zh_CN')
         fake_data =  fake.random_int(min=1, max=20, step=1) 
         tdLog.info("\n=============constant(%s)_check ====================\n" %func)  
-        tdSql.execute(" create sma index sma_index_name1 on stable_1 function(%s(%s)) interval(%dm); " %(func,column,fake_data))  
+        tdSql.execute(" create sma index %s.sma_index_name1 on %s.stable_1 function(%s(%s)) interval(%dm); " %(database,database,func,column,fake_data))  
         sql = " select %s(%s) from %s.stable_1 interval(1m) "%(func,column,database)
         tdLog.info(sql)
         tdSql.query(sql) 
@@ -205,7 +205,7 @@ class TDTestCase:
         
         tdLog.info("\n=============drop index ====================\n")
         
-        tdSql.execute(" drop index sma_index_name1;")
+        tdSql.execute(" drop index %s.sma_index_name1;" %database)
         
         tdSql.query(sql)  
         queryRows = len(tdSql.queryResult)    
@@ -215,6 +215,29 @@ class TDTestCase:
         
         self.value_check(flush_before_value,drop_index_value)
     
+    def constant_speical_check(self,database,func,column):    
+        tdLog.info("\n=============constant(%s)_check ====================\n" %column) 
+        sql_no_from = " select %s(%s) ; "%(func,column)
+
+        tdLog.info(sql_no_from)
+        tdSql.query(sql_no_from) 
+        queryRows = len(tdSql.queryResult)    
+        for i in range(queryRows):
+            print("row=%d,  result=%s " %(i,tdSql.queryResult[i][0]))
+            flush_before_value_no_from = tdSql.queryResult[i][0]
+        
+        tdLog.info("\n=============flush database ====================\n")
+        
+        tdSql.execute(" flush database %s;" %database)
+        
+        tdSql.query(sql_no_from)  
+        queryRows = len(tdSql.queryResult)    
+        for i in range(queryRows):
+            print("row=%d,  result=%s " %(i,tdSql.queryResult[i][0]))
+            flush_after_value_no_from = tdSql.queryResult[i][0]
+        
+        self.value_check(flush_before_value_no_from,flush_after_value_no_from)
+               
         
     def constant_check(self,database,func,column):    
         tdLog.info("\n=============constant(%s)_check ====================\n" %column) 
@@ -253,7 +276,7 @@ class TDTestCase:
             print("row=%d,  result=%s " %(i,tdSql.queryResult[i][0]))
             flush_after_value_no_from = tdSql.queryResult[i][0]
         
-        self.value_check(flush_before_value_no_from,flush_after_value_no_from)
+        #self.value_check(flush_before_value_no_from,flush_after_value_no_from)#越界后值不唯一
         
     def constant_table_check(self,database,func,column):    
         tdLog.info("\n=============constant(%s)_check ====================\n" %column) 
@@ -292,7 +315,7 @@ class TDTestCase:
             print("row=%d,  result=%s " %(i,tdSql.queryResult[i][0]))
             flush_after_value_no_from = tdSql.queryResult[i][0]
         
-        self.value_check(flush_before_value_no_from,flush_after_value_no_from)
+        #self.value_check(flush_before_value_no_from,flush_after_value_no_from)#越界后值不唯一
         
     def constant_str_check(self,database,func,column):    
         tdLog.info("\n=============constant(%s)_check ====================\n" %column) 
@@ -338,7 +361,7 @@ class TDTestCase:
         
     def derivative_sql(self,database):    
         fake = Faker('zh_CN')
-        fake_data =  fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1)
+        fake_data =  fake.random_int(min=1, max=10000000000000, step=1)
         tdLog.info("\n=============derivative sql ====================\n" )  
         sql = " select derivative(%s,%ds,0) from %s.stable_1 "%('q_smallint',fake_data,database) 
         self.derivative_data_check("%s" %self.db,"%s" %sql)
@@ -493,11 +516,11 @@ class TDTestCase:
         
                       
     def value_check(self,flush_before_value,flush_after_value):
-        # if flush_before_value==flush_after_value:
+        if flush_before_value==flush_after_value:
-        #     tdLog.info(f"checkEqual success, flush_before_value={flush_before_value},flush_after_value={flush_after_value}") 
+            tdLog.info(f"checkEqual success, flush_before_value={flush_before_value},flush_after_value={flush_after_value}") 
-        # else :
+        else :
-        #     tdLog.exit(f"checkEqual error, flush_before_value=={flush_before_value},flush_after_value={flush_after_value}") 
+            tdLog.exit(f"checkEqual error, flush_before_value=={flush_before_value},flush_after_value={flush_after_value}") 
-        pass
+        #pass
                             
     def run(self):      
         fake = Faker('zh_CN')
@@ -511,6 +534,11 @@ class TDTestCase:
         
         self.dropandcreateDB_random("%s" %self.db, 1,2)
         
+        self.constant_speical_check("%s" %self.db,'','%d' %fake_data)
+        self.constant_speical_check("%s" %self.db,'','%f' %fake_float)
+        self.constant_speical_check("%s" %self.db,'','\'%s\'' %fake_str)
+        self.constant_speical_check("%s" %self.db,'','NULL')
+        
         #TD-19818
         self.func_index_check("%s" %self.db,'max','q_int')
         self.func_index_check("%s" %self.db,'max','q_bigint')

tests/system-test/2-query/smaTest.py

@@ -44,8 +44,8 @@ class TDTestCase:
     def run(self):
         # insert data
         dbname = "db"
-        self.insert_data1(f"{dbname}.t1", self.ts, 1000*10000)
+        self.insert_data1(f"{dbname}.t1", self.ts, 10*10000)
-        self.insert_data1(f"{dbname}.t4", self.ts, 1000*10000)
+        self.insert_data1(f"{dbname}.t4", self.ts, 10*10000)
         # test base case
         # self.test_case1()
         tdLog.debug(" LIMIT test_case1 ............ [OK]")
@@ -53,7 +53,6 @@ class TDTestCase:
         # self.test_case2()
         tdLog.debug(" LIMIT test_case2 ............ [OK]")
 
-
     # stop
     def stop(self):
         tdSql.close()
@@ -77,15 +76,17 @@ class TDTestCase:
 
     # insert data1
     def insert_data(self, tbname, ts_start, count):
-        pre_insert = "insert into %s values"%tbname
+        pre_insert = "insert into %s values" % tbname
         sql = pre_insert
-        tdLog.debug("insert table %s rows=%d ..."%(tbname, count))
+        tdLog.debug("insert table %s rows=%d ..." % (tbname, count))
         for i in range(count):
-            sql += " (%d,%d)"%(ts_start + i*1000, i )
+            sql += " (%d,%d)" % (ts_start + i*1000, i)
-            if i >0 and i%30000 == 0:
+            if i > 0 and i % 20000 == 0:
+                tdLog.info("%d rows inserted" % i)
                 tdSql.execute(sql)
                 sql = pre_insert
         # end sql
+        tdLog.info("insert_data end")
         if sql != pre_insert:
             tdSql.execute(sql)
 
@@ -93,15 +94,17 @@ class TDTestCase:
         return
 
     def insert_data1(self, tbname, ts_start, count):
-        pre_insert = "insert into %s values"%tbname
+        pre_insert = "insert into %s values" % tbname
         sql = pre_insert
-        tdLog.debug("insert table %s rows=%d ..."%(tbname, count))
+        tdLog.debug("insert table %s rows=%d ..." % (tbname, count))
         for i in range(count):
-            sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1)
+            sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1)
-            if i >0 and i%30000 == 0:
+            if i > 0 and i % 20000 == 0:
+                tdLog.info("%d rows inserted" % i)
                 tdSql.execute(sql)
                 sql = pre_insert
         # end sql
+        tdLog.info("insert_data1 end")
         if sql != pre_insert:
             tdSql.execute(sql)
 

tests/system-test/2-query/sml.py

@@ -27,7 +27,7 @@ class TDTestCase:
         tdLog.info(cmdStr)
         ret = os.system(cmdStr)
         if ret != 0:
-            tdLog.exit("sml_test failed")
+            tdLog.info("sml_test ret != 0")
 
         # tdSql.execute('use sml_db')
         tdSql.query(f"select * from {dbname}.t_b7d815c9222ca64cdf2614c61de8f211")

tests/system-test/7-tmq/tmq3mnodeSwitch.py

@@ -54,9 +54,9 @@ class TDTestCase:
             time.sleep(1)
             tdLog.debug("............... waiting for all dnodes ready!")
 
-        tdLog.info("==============create two new mnodes ========")
+        # tdLog.info("==============create two new mnodes ========")
-        tdSql.execute("create mnode on dnode 2")
+        # tdSql.execute("create mnode on dnode 2")
-        tdSql.execute("create mnode on dnode 3")
+        # tdSql.execute("create mnode on dnode 3")
         self.check3mnode()
         return
 
@@ -179,17 +179,20 @@ class TDTestCase:
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
                     'ctbNum':     1,
-                    'rowsPerTbl': 100000,
+                    'rowsPerTbl': 40000,
                     'batchNum':   10,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  10,
+                    'pollDelay':  30,
                     'showMsg':    1,
                     'showRow':    1}
         
+        if self.replicaVar == 3:
+            paraDict["rowsPerTbl"] = 20000
+
         topicNameList = ['topic1']
         expectRowsList = []
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=self.replicaVar)
         tdLog.info("create stb")
         tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
         tdLog.info("create ctb")
@@ -198,7 +201,9 @@ class TDTestCase:
         pThread = tmqCom.asyncInsertData(paraDict)
 
         tdLog.info("create topics from stb with filter")
-        queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+        # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+        
+        queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
         sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
         tdLog.info("create topic sql: %s"%sqlString)
         tdSql.execute(sqlString)