From 83c1db07a36d03cef8a53481d4c528cbd82fc6c7 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Tue, 21 Nov 2023 13:53:10 +0800
Subject: [PATCH 01/43] enhance: tbname col to func

---
 source/libs/parser/inc/parTranslater.h |  2 ++
 source/libs/parser/src/parTranslater.c | 25 ++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h
index e06626fb00..bec4e07e1c 100644
--- a/source/libs/parser/inc/parTranslater.h
+++ b/source/libs/parser/inc/parTranslater.h
@@ -46,6 +46,8 @@ typedef struct STranslateContext {
   SNode*           pPostRoot;
 } STranslateContext;
 
+EDealRes biRewriteToTbnameFuncAndTranslate(STranslateContext* pCxt, SColumnNode** ppCol);
+EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc);
 int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
 int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
 int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1a65a29259..61ac39ffd4 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1091,6 +1091,22 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
   return DEAL_RES_CONTINUE;
 }
 
+#ifndef TD_ENTERPRISE
+EDealRes biRewriteToTbnameFuncAndTranslate(STranslateContext* pCxt, SColumnNode** ppCol) {
+  SFunctionNode* tbnameFuncNode = NULL;
+  // tbnameFuncNode = biMakeTbnameProjectAstNode(NULL, ((*ppCol)->tableAlias[0]!='\0') ? (*ppCol)->tableAlias : NULL);
+  tbnameFuncNode->node.resType = (*ppCol)->node.resType;
+  strcpy(tbnameFuncNode->node.aliasName, (*ppCol)->node.aliasName);
+  strcpy(tbnameFuncNode->node.userAlias, (*ppCol)->node.userAlias);
+
+  nodesDestroyNode(*(SNode**)ppCol);
+  *(SNode**)ppCol = (SNode*)tbnameFuncNode;
+
+  EDealRes res = translateFunction(pCxt, &tbnameFuncNode);
+  return res;
+}
+#endif
+
 static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
   if (NULL == pCxt->pCurrStmt ||
       (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
@@ -1102,6 +1118,13 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
     return DEAL_RES_CONTINUE;
   }
 
+  if (pCxt->pParseCxt->biMode && (strcasecmp((*pCol)->colName, "tbname") == 0) &&
+      ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
+      QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
+    EDealRes res = biRewriteToTbnameFuncAndTranslate(pCxt, pCol);
+    return res;
+  }
+
   EDealRes res = DEAL_RES_CONTINUE;
   if ('\0' != (*pCol)->tableAlias[0]) {
     res = translateColumnWithPrefix(pCxt, pCol);
@@ -2204,7 +2227,7 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF
   return translateNormalFunction(pCxt, (SNode**)pFunc);
 }
 
-static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
+EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
   SNode* pParam = NULL;
   FOREACH(pParam, (*pFunc)->pParameterList) {
     if (isMultiResFunc(pParam)) {

From ebbbb3825d3004d81398cb43ab518123eaec4e0f Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 21 Nov 2023 14:33:00 +0800
Subject: [PATCH 02/43] enhance: bi mode tbname col to tbname func

---
 source/libs/parser/src/parTranslater.c | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 61ac39ffd4..b868d39cfd 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1093,17 +1093,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
 
 #ifndef TD_ENTERPRISE
 EDealRes biRewriteToTbnameFuncAndTranslate(STranslateContext* pCxt, SColumnNode** ppCol) {
-  SFunctionNode* tbnameFuncNode = NULL;
-  // tbnameFuncNode = biMakeTbnameProjectAstNode(NULL, ((*ppCol)->tableAlias[0]!='\0') ? (*ppCol)->tableAlias : NULL);
-  tbnameFuncNode->node.resType = (*ppCol)->node.resType;
-  strcpy(tbnameFuncNode->node.aliasName, (*ppCol)->node.aliasName);
-  strcpy(tbnameFuncNode->node.userAlias, (*ppCol)->node.userAlias);
-
-  nodesDestroyNode(*(SNode**)ppCol);
-  *(SNode**)ppCol = (SNode*)tbnameFuncNode;
-
-  EDealRes res = translateFunction(pCxt, &tbnameFuncNode);
-  return res;
+  return DEAL_RES_CONTINUE;
 }
 #endif
 
@@ -1118,11 +1108,13 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
     return DEAL_RES_CONTINUE;
   }
 
-  if (pCxt->pParseCxt->biMode && (strcasecmp((*pCol)->colName, "tbname") == 0) &&
+  if (pCxt->pParseCxt->biMode) {
+    if ((strcasecmp((*pCol)->colName, "tbname") == 0) &&
       ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
       QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
-    EDealRes res = biRewriteToTbnameFuncAndTranslate(pCxt, pCol);
-    return res;
+      EDealRes res = biRewriteToTbnameFuncAndTranslate(pCxt, pCol);
+      return res;
+    }
   }
 
   EDealRes res = DEAL_RES_CONTINUE;

From 1fc16355def4c0557addd32176d581c13817c38b Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 21 Nov 2023 14:34:09 +0800
Subject: [PATCH 03/43] enhance: bi mode tbname col to tbname func

---
 tests/script/tsim/query/bi_tbname_col.sim | 34 +++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 tests/script/tsim/query/bi_tbname_col.sim

diff --git a/tests/script/tsim/query/bi_tbname_col.sim b/tests/script/tsim/query/bi_tbname_col.sim
new file mode 100644
index 0000000000..7cf897280b
--- /dev/null
+++ b/tests/script/tsim/query/bi_tbname_col.sim
@@ -0,0 +1,34 @@
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql drop database if exists db1;
+sql create database db1 vgroups 3;
+sql create database db1;
+sql use db1;
+sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create table tba1 using sta tags(1, 1, 1);
+sql create table tba2 using sta tags(2, 2, 2);
+sql insert into tba1 values(now, 1, "1")(now+3s, 3, "3")(now+5s, 5, "5");
+sql insert into tba2 values(now + 1s, 2, "2")(now+2s, 2, "2")(now+4s, 4, "4");
+sql create table tbn1 (ts timestamp, f1 int);
+
+set_bi_mode 1
+
+sql select `tbname`, f1, f2 from sta order by ts
+print $rows
+print $data00 $data01 $data02 $data10 $data11 $data12
+if $rows != 6 then
+  return -1
+endi
+if $data00 != @tba1@ then
+  return -1
+endi
+if $data10 != @tba2@ then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 40a28a6e4bb57bdebe3a6b8228fbdbf944d2a668 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 22 Nov 2023 08:47:35 +0800
Subject: [PATCH 04/43] fix: refactor dependency between community and
 enterprise

---
 source/libs/parser/inc/parTranslater.h |  3 +--
 source/libs/parser/src/parTranslater.c | 15 +++++++--------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h
index bec4e07e1c..0fe99e04ab 100644
--- a/source/libs/parser/inc/parTranslater.h
+++ b/source/libs/parser/inc/parTranslater.h
@@ -46,8 +46,7 @@ typedef struct STranslateContext {
   SNode*           pPostRoot;
 } STranslateContext;
 
-EDealRes biRewriteToTbnameFuncAndTranslate(STranslateContext* pCxt, SColumnNode** ppCol);
-EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc);
+bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode);
 int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
 int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
 int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index b868d39cfd..729a0d8855 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -263,6 +263,7 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
 static int32_t  translateSubquery(STranslateContext* pCxt, SNode* pNode);
 static int32_t  translateQuery(STranslateContext* pCxt, SNode* pNode);
 static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal);
+static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc);
 static int32_t  createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
                                                    SSelectStmt** pStmt);
 static int32_t  createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
@@ -1092,8 +1093,8 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
 }
 
 #ifndef TD_ENTERPRISE
-EDealRes biRewriteToTbnameFuncAndTranslate(STranslateContext* pCxt, SColumnNode** ppCol) {
-  return DEAL_RES_CONTINUE;
+bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
+  return false;
 }
 #endif
 
@@ -1109,11 +1110,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
   }
 
   if (pCxt->pParseCxt->biMode) {
-    if ((strcasecmp((*pCol)->colName, "tbname") == 0) &&
-      ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
-      QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
-      EDealRes res = biRewriteToTbnameFuncAndTranslate(pCxt, pCol);
-      return res;
+    SNode** ppNode = (SNode**)pCol;
+    if (biRewriteToTbnameFunc(pCxt, ppNode)) {
+      translateFunction(pCxt, (SFunctionNode**)ppNode);
     }
   }
 
@@ -2219,7 +2218,7 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF
   return translateNormalFunction(pCxt, (SNode**)pFunc);
 }
 
-EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
+static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) {
   SNode* pParam = NULL;
   FOREACH(pParam, (*pFunc)->pParameterList) {
     if (isMultiResFunc(pParam)) {

From 668e9e0a8b3ede75604d978a3f2ba1839621c712 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 22 Nov 2023 08:59:29 +0800
Subject: [PATCH 05/43] fix: add test cases

---
 source/libs/parser/src/parTranslater.c | 2 +-
 tests/parallel_test/cases.task         | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 729a0d8855..4996063578 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1112,7 +1112,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
   if (pCxt->pParseCxt->biMode) {
     SNode** ppNode = (SNode**)pCol;
     if (biRewriteToTbnameFunc(pCxt, ppNode)) {
-      translateFunction(pCxt, (SFunctionNode**)ppNode);
+      return translateFunction(pCxt, (SFunctionNode**)ppNode);
     }
   }
 
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 92eaec52b5..fc7281d6e8 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -1065,6 +1065,7 @@ e
 ,,y,script,./test.sh -f tsim/query/show_db_table_kind.sim
 ,,y,script,./test.sh -f tsim/query/bi_star_table.sim
 ,,y,script,./test.sh -f tsim/query/bi_tag_scan.sim
+,,y,script,./test.sh -f tsim/query/bi_tbname_col.sim
 ,,y,script,./test.sh -f tsim/query/tag_scan.sim
 ,,y,script,./test.sh -f tsim/query/nullColSma.sim
 ,,y,script,./test.sh -f tsim/query/bug3398.sim

From 3527860c75b03b10b78b8843889b357621a4e5fe Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 10:00:34 +0800
Subject: [PATCH 06/43] refactor: do some internal refactor.

---
 contrib/CMakeLists.txt | 86 ------------------------------------------
 1 file changed, 86 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index c5715bd53f..b786f2df25 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -109,11 +109,6 @@ cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 # cJson
 cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
-# leveldb
-if(${BUILD_WITH_LEVELDB})
-    cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-endif(${BUILD_WITH_LEVELDB})
-
 if (${BUILD_CONTRIB})
     if(${BUILD_WITH_ROCKSDB})
         cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -132,28 +127,11 @@ else()
     endif()
 endif()
 
-# canonical-raft
-if(${BUILD_WITH_CRAFT})
-    cat("${TD_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE)
-endif(${BUILD_WITH_CRAFT})
-
-# traft
-if(${BUILD_WITH_TRAFT})
-    cat("${TD_SUPPORT_DIR}/traft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    SET(BUILD_WITH_UV ON CACHE BOOL "traft need libuv" FORCE)
-endif(${BUILD_WITH_TRAFT})
-
 #libuv
 if(${BUILD_WITH_UV})
     cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_UV})
 
-# bdb
-if(${BUILD_WITH_BDB})
-    cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-endif(${BUILD_WITH_BDB})
-
 # sqlite
 if(${BUILD_WITH_SQLITE})
     cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -178,17 +156,6 @@ elseif(${BUILD_WITH_COS})
 
 endif()
 
-# lucene
-if(${BUILD_WITH_LUCENE})
-    cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    add_definitions(-DUSE_LUCENE)
-endif(${BUILD_WITH_LUCENE})
-
-# NuRaft
-if(${BUILD_WITH_NURAFT})
-    cat("${TD_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-endif(${BUILD_WITH_NURAFT})
-
 # crashdump
 if(${BUILD_CRASHDUMP})
     cat("${TD_SUPPORT_DIR}/crashdump_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -437,23 +404,6 @@ elseif(${BUILD_WITH_COS})
 
 endif()
 
-# lucene
-# To support build on ubuntu: sudo apt-get install libboost-all-dev
-if(${BUILD_WITH_LUCENE})
-    option(ENABLE_TEST "Enable the tests" OFF)
-    add_subdirectory(lucene EXCLUDE_FROM_ALL)
-    target_include_directories(
-      lucene++
-      PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
-    )
-
-endif(${BUILD_WITH_LUCENE})
-
-# NuRaft
-if(${BUILD_WITH_NURAFT})
-    add_subdirectory(nuraft EXCLUDE_FROM_ALL)
-endif(${BUILD_WITH_NURAFT})
-
 # pthread
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE debug)
@@ -524,30 +474,6 @@ if(${BUILD_WCWIDTH})
     SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
 endif(${BUILD_WCWIDTH})
 
-# CRAFT
-if(${BUILD_WITH_CRAFT})
-    add_library(craft STATIC IMPORTED GLOBAL)
-    set_target_properties(craft PROPERTIES
-        IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/craft/.libs/libraft.a"
-        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft/include"
-    )
-    # target_link_libraries(craft
-    #     INTERFACE pthread
-    # )
-endif(${BUILD_WITH_CRAFT})
-
-# TRAFT
-if(${BUILD_WITH_TRAFT})
-    add_library(traft STATIC IMPORTED GLOBAL)
-    set_target_properties(traft PROPERTIES
-        IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/traft/.libs/libraft.a"
-        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/traft/include"
-    )
-    # target_link_libraries(craft
-    #     INTERFACE pthread
-    # )
-endif(${BUILD_WITH_TRAFT})
-
 # LIBUV
 if(${BUILD_WITH_UV})
     if (TD_WINDOWS)
@@ -559,18 +485,6 @@ if(${BUILD_WITH_UV})
     add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})
 
-# BDB
-if(${BUILD_WITH_BDB})
-    add_library(bdb STATIC IMPORTED GLOBAL)
-    set_target_properties(bdb PROPERTIES
-        IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a"
-        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb"
-    )
-    target_link_libraries(bdb
-        INTERFACE pthread
-    )
-endif(${BUILD_WITH_BDB})
-
 # SQLite
 # see https://stackoverflow.com/questions/8774593/cmake-link-to-external-library#comment58570736_10550334
 if(${BUILD_WITH_SQLITE})

From 9814109175400b8812d72517d5e734f986bf9dfc Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 10:00:50 +0800
Subject: [PATCH 07/43] refactor: do some internal refactor.

---
 cmake/bdb_CMakeLists.txt.in     | 13 -------------
 cmake/leveldb_CMakeLists.txt.in | 13 -------------
 cmake/lucene_CMakeLists.txt.in  | 12 ------------
 3 files changed, 38 deletions(-)
 delete mode 100644 cmake/bdb_CMakeLists.txt.in
 delete mode 100644 cmake/leveldb_CMakeLists.txt.in
 delete mode 100644 cmake/lucene_CMakeLists.txt.in

diff --git a/cmake/bdb_CMakeLists.txt.in b/cmake/bdb_CMakeLists.txt.in
deleted file mode 100644
index dd21020105..0000000000
--- a/cmake/bdb_CMakeLists.txt.in
+++ /dev/null
@@ -1,13 +0,0 @@
-
-# bdb
-ExternalProject_Add(bdb
-        GIT_REPOSITORY https://github.com/berkeleydb/libdb.git
-        GIT_TAG v5.3.28
-        SOURCE_DIR "${TD_CONTRIB_DIR}/bdb"
-        BINARY_DIR "${TD_CONTRIB_DIR}/bdb"
-        #BUILD_IN_SOURCE TRUE
-        CONFIGURE_COMMAND COMMAND ./dist/configure --enable-debug
-        BUILD_COMMAND "$(MAKE)"
-        INSTALL_COMMAND ""
-        TEST_COMMAND ""
-)
diff --git a/cmake/leveldb_CMakeLists.txt.in b/cmake/leveldb_CMakeLists.txt.in
deleted file mode 100644
index 6878711bc1..0000000000
--- a/cmake/leveldb_CMakeLists.txt.in
+++ /dev/null
@@ -1,13 +0,0 @@
-
-# leveldb
-ExternalProject_Add(leveldb
-        GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
-        GIT_TAG master
-        SOURCE_DIR "${TD_CONTRIB_DIR}/leveldb"
-        BINARY_DIR ""
-        #BUILD_IN_SOURCE TRUE
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ""
-        INSTALL_COMMAND ""
-        TEST_COMMAND ""
-)
\ No newline at end of file
diff --git a/cmake/lucene_CMakeLists.txt.in b/cmake/lucene_CMakeLists.txt.in
deleted file mode 100644
index 9fd7471705..0000000000
--- a/cmake/lucene_CMakeLists.txt.in
+++ /dev/null
@@ -1,12 +0,0 @@
-
-# lucene
-ExternalProject_Add(lucene
-        GIT_REPOSITORY https://github.com/yihaoDeng/LucenePlusPlus.git
-        SOURCE_DIR "${TD_CONTRIB_DIR}/lucene"
-        BINARY_DIR ""
-        #BUILD_IN_SOURCE TRUE
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ""
-        INSTALL_COMMAND ""
-        TEST_COMMAND ""
-)

From 281f25fdcc71ad5fd8705890b9e96163b6569e68 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 10:01:42 +0800
Subject: [PATCH 08/43] refactor: do some internal refactor.

---
 cmake/nuraft_CMakeLists.txt.in | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 cmake/nuraft_CMakeLists.txt.in

diff --git a/cmake/nuraft_CMakeLists.txt.in b/cmake/nuraft_CMakeLists.txt.in
deleted file mode 100644
index 593c6fed26..0000000000
--- a/cmake/nuraft_CMakeLists.txt.in
+++ /dev/null
@@ -1,12 +0,0 @@
-
-# NuRaft
-ExternalProject_Add(NuRaft
-        GIT_REPOSITORY https://github.com/eBay/NuRaft.git
-        GIT_TAG v1.3.0
-        SOURCE_DIR "${TD_CONTRIB_DIR}/nuraft"
-        BINARY_DIR "${TD_CONTRIB_DIR}/nuraft"
-        CONFIGURE_COMMAND "./prepare.sh"
-        BUILD_COMMAND ""
-        INSTALL_COMMAND ""
-        TEST_COMMAND ""
-        )
\ No newline at end of file

From 3de91de7f958751cd0bb7ce54d5974d510669cec Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 23 Nov 2023 11:18:05 +0800
Subject: [PATCH 09/43] fix: bi mode create tbname col error

---
 source/libs/parser/inc/parTranslater.h    |  2 ++
 source/libs/parser/src/parTranslater.c    | 10 +++++++++-
 tests/script/tsim/query/bi_tbname_col.sim |  2 ++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h
index 0fe99e04ab..55ea71a368 100644
--- a/source/libs/parser/inc/parTranslater.h
+++ b/source/libs/parser/inc/parTranslater.h
@@ -23,6 +23,7 @@ extern "C" {
 #include "parToken.h"
 #include "parUtil.h"
 #include "parser.h"
+#include "cmdnodes.h"
 
 typedef struct STranslateContext {
   SParseContext*   pParseCxt;
@@ -48,6 +49,7 @@ typedef struct STranslateContext {
 
 bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode);
 int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
+int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt);
 int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
 int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView);
 
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 496e3423e6..74a462dfda 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -5726,6 +5726,12 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions
   return code;
 }
 
+#ifndef TD_ENTERPRISE
+int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+  return TSDB_CODE_SUCCESS;
+}
+#endif
+
 static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) {
   if (NULL != strchr(pStmt->tableName, '.')) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME,
@@ -5764,7 +5770,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
                                      "configured with the 'TTL' option");
     }
   }
-
+  if (pCxt->pParseCxt->biMode != 0 && TSDB_CODE_SUCCESS == code) {
+    code = biCheckCreateTableTbnameCol(pCxt, pStmt);
+  }
   return code;
 }
 
diff --git a/tests/script/tsim/query/bi_tbname_col.sim b/tests/script/tsim/query/bi_tbname_col.sim
index 7cf897280b..d351a92539 100644
--- a/tests/script/tsim/query/bi_tbname_col.sim
+++ b/tests/script/tsim/query/bi_tbname_col.sim
@@ -31,4 +31,6 @@ if $data10 != @tba2@ then
   return -1
 endi
 
+sql_error create table stc(ts timestamp, `tbname` binary(200));
+sql_error create table std(ts timestamp, f1 int) tags(`tbname` binary(200));
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 6e3a62803c22ea7d71be75e8a8b2d9d6c08f449a Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 23 Nov 2023 14:08:31 +0800
Subject: [PATCH 10/43] fix: add check in mnd when alter table col/tag

---
 source/dnode/mnode/impl/src/mndStb.c |  34 ++++++
 tests/script/api/alterTableTest.c    | 175 +++++++++++++++++++++++++++
 2 files changed, 209 insertions(+)
 create mode 100644 tests/script/api/alterTableTest.c

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index eb2f9dcbe3..1ddd2f34e6 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -1014,6 +1014,20 @@ static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *col
   return -1;
 }
 
+static bool mndValidateSchema(SSchema *pSchemas, int32_t nSchema, SArray *pFields, int32_t maxLen) {
+  int32_t rowLen = 0;
+  for (int32_t i = 0; i < nSchema; ++i) {
+    rowLen += (pSchemas + i)->bytes;
+  }
+
+  int32_t nField = taosArrayGetSize(pFields);
+  for (int32_t i = 0; i < nField; ++i) {
+    rowLen += ((SField *)TARRAY_GET_ELEM(pFields, i))->bytes;
+  }
+
+  return rowLen <= maxLen;
+}
+
 static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq *createReq) {
   taosRLockLatch(&pStb->lock);
   memcpy(pDst, pStb, sizeof(SStbObj));
@@ -1269,6 +1283,11 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
     return -1;
   }
 
+  if (!mndValidateSchema(pOld->pTags, pOld->numOfTags, pFields, TSDB_MAX_TAGS_LEN)) {
+    terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH;
+    return -1;
+  }
+
   pNew->numOfTags = pNew->numOfTags + ntags;
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
@@ -1558,6 +1577,16 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
     return -1;
   }
 
+  uint32_t nLen = 0;
+  for (int32_t i = 0; i < pOld->numOfTags; ++i) {
+    nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes;
+  }
+  
+  if (nLen > TSDB_MAX_TAGS_LEN) {
+    terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH;
+    return -1;
+  }
+
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
@@ -1592,6 +1621,11 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
     return -1;
   }
 
+  if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pFields, TSDB_MAX_BYTES_PER_ROW)) {
+    terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH;
+    return -1;
+  }
+
   pNew->numOfColumns = pNew->numOfColumns + ncols;
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c
new file mode 100644
index 0000000000..14605f4b8b
--- /dev/null
+++ b/tests/script/api/alterTableTest.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// TAOS standard API example. The same syntax as MySQL, but only a subset
+// to compile: gcc -o demo demo.c -ltaos
+
+/**
+ *  passwdTest.c
+ *   - Run the test case in clear TDengine environment with default root passwd 'taosdata'
+ */
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "taos.h"  // TAOS header file
+
+#define nDup      3
+#define USER_LEN  24
+#define BUF_LEN   1024
+#define DB        "d0"
+#define DB_BUFFER 32
+#define STB       "stb"
+#define CTB       "ctb"
+#define COL       "c"
+
+#if 0
+#define STB_NUM 125
+#define CTB_NUM 960
+#define COL_NUM 52
+#define COL_NCHAR_LEN 320
+#else
+#define STB_NUM       5
+#define CTB_NUM       1
+#define COL_NUM       505
+#define COL_NCHAR_LEN 32
+#endif
+typedef uint16_t VarDataLenT;
+
+int32_t isDropDb = 0;
+
+#define TSDB_NCHAR_SIZE    sizeof(int32_t)
+#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
+
+#define GET_FLOAT_VAL(x)  (*(float *)(x))
+#define GET_DOUBLE_VAL(x) (*(double *)(x))
+
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+
+static int32_t queryDB(TAOS *taos, char *command, bool skipError) {
+  int       i;
+  TAOS_RES *pSql = NULL;
+  int32_t   code = -1;
+
+  for (i = 0; i < nDup; ++i) {
+    if (NULL != pSql) {
+      taos_free_result(pSql);
+      pSql = NULL;
+    }
+
+    pSql = taos_query(taos, command);
+    code = taos_errno(pSql);
+    if (0 == code) {
+      break;
+    }
+  }
+
+  if (code != 0) {
+    fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql));
+    if (!skipError) {
+      taos_free_result(pSql);
+      taos_close(taos);
+      exit(EXIT_FAILURE);
+    }
+  } else {
+    fprintf(stderr, "success to run: %s\n", command);
+  }
+
+  taos_free_result(pSql);
+}
+
+static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qstr, int32_t addColMode) {
+  if (isDropDb) {
+    sprintf(qstr, "drop database if exists %s", DB);
+    queryDB(taos, qstr, false);
+    sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER);
+    queryDB(taos, qstr, false);
+  }
+  sprintf(qstr, "use %s", DB);
+  queryDB(taos, qstr, false);
+
+  // create stb
+  if (isDropDb) {
+    for (int i = 0; i < STB_NUM; ++i) {
+      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(16));", STB, i, COL,
+              0);
+      queryDB(taos, qstr, false);
+      // create ctb
+      for (int j = 0; j < CTB_NUM; ++j) {
+        sprintf(qstr, "CREATE table %s_%d_%s_%d using %s_%d tags('%d_%d');", STB, i, CTB, j, STB, i, i, j);
+        queryDB(taos, qstr, false);
+      }
+    }
+  }
+
+  if (isDropDb) {
+    printf("sleep 86400s to wait other terminal\n");
+    sleep(86400);
+  }
+
+  // alter stb cols
+  if (addColMode == 0) {
+    for (int i = 0; i < STB_NUM; ++i) {
+      for (int c = 1; c < COL_NUM; ++c) {
+        sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN);
+        queryDB(taos, qstr, true);
+      }
+      sprintf(qstr, "desc %s_%d;", STB, i);
+      queryDB(taos, qstr, false);
+    }
+  } else if (addColMode == 1) {
+    for (int c = 1; c < COL_NUM; ++c) {
+      for (int i = 0; i < STB_NUM; ++i) {
+        sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN);
+        queryDB(taos, qstr, true);
+      }
+    }
+  }
+
+  // check
+  for (int i = 0; i < STB_NUM; ++i) {
+    sprintf(qstr, "desc %s_%d;", STB, i);
+    queryDB(taos, qstr, false);
+  }
+}
+
+int main(int argc, char *argv[]) {
+  char qstr[1024];
+
+  // connect to server
+  if (argc < 2) {
+    printf("please input server-ip \n");
+    return 0;
+  }
+
+  if (argc < 3) {
+    printf("please specify if drop DB\n");
+    return 0;
+  }
+
+  isDropDb = atoi(argv[2]);
+
+  TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
+  if (taos == NULL) {
+    printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/);
+    exit(1);
+  }
+  createDatabaseAlterStbColumns(taos, argv[1], qstr, 0);
+
+  taos_close(taos);
+  taos_cleanup();
+}
\ No newline at end of file

From 484c045c05ea71580154d21c9ccd799cc922ab3f Mon Sep 17 00:00:00 2001
From: xsren <285808407@qq.com>
Date: Thu, 23 Nov 2023 14:24:58 +0800
Subject: [PATCH 11/43] param num error

---
 source/libs/parser/src/parTranslater.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1f8af22241..ca0f290c88 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2212,7 +2212,8 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc
   SNode* pParam = NULL;
   FOREACH(pParam, (*pFunc)->pParameterList) {
     if (isMultiResFunc(pParam)) {
-      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pParam)->aliasName);
+      pCxt->errCode = TSDB_CODE_FUNC_FUNTION_PARA_NUM;
+      return DEAL_RES_ERROR;
     }
   }
 

From 1a1319908d82286ec1c65d2bdc122329ec0ca750 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 14:25:52 +0800
Subject: [PATCH 12/43] refactor: do some internal refactor.

---
 source/libs/stream/inc/streamInt.h    | 20 +++++++++-----------
 source/libs/stream/src/streamData.c   |  2 +-
 source/libs/stream/src/streamQueue.c  |  2 +-
 source/libs/stream/src/streamStart.c  |  4 ++--
 source/libs/stream/src/streamTaskSm.c |  6 +++---
 5 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 6dd1e5c1c3..b54f19e4ce 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -113,25 +113,23 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32
 int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask);
 int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask);
 int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask);
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*);
+STaskId streamTaskExtractKey(const SStreamTask* pTask);
+void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
+void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);
+int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen);
+int32_t streamTaskFillHistoryFinished(SStreamTask* pTask);
 
 int32_t     streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, int32_t* blockSize);
 int32_t     streamQueueItemGetSize(const SStreamQueueItem* pItem);
 void        streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size);
 const char* streamQueueItemGetTypeStr(int32_t type);
+SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem);
 
-SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem);
-
-int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen);
 int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq);
 int32_t streamNotifyUpstreamContinue(SStreamTask* pTask);
-int32_t streamTaskFillHistoryFinished(SStreamTask* pTask);
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask);
 
-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*);
-STaskId streamTaskExtractKey(const SStreamTask* pTask);
-void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
-void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);
-
 SStreamQueue* streamQueueOpen(int64_t cap);
 void          streamQueueClose(SStreamQueue* pQueue, int32_t taskId);
 void          streamQueueProcessSuccess(SStreamQueue* queue);
@@ -152,8 +150,8 @@ int downloadCheckpoint(char* id, char* path);
 int deleteCheckpoint(char* id);
 int deleteCheckpointFile(char* id, char* name);
 
-int32_t onNormalTaskReady(SStreamTask* pTask);
-int32_t onScanhistoryTaskReady(SStreamTask* pTask);
+int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
+int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c
index f6ec6e9fdb..bcda85e7a7 100644
--- a/source/libs/stream/src/streamData.c
+++ b/source/libs/stream/src/streamData.c
@@ -158,7 +158,7 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm
 }
 
 // todo handle memory error
-SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) {
+SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) {
   terrno = 0;
   
   if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) {
diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c
index d19dfc13bf..d1610362f9 100644
--- a/source/libs/stream/src/streamQueue.c
+++ b/source/libs/stream/src/streamQueue.c
@@ -221,7 +221,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
         *pInput = qItem;
       } else {
         // merge current block failed, let's handle the already merged blocks.
-        void* newRet = streamMergeQueueItem(*pInput, qItem);
+        void* newRet = streamQueueMergeQueueItem(*pInput, qItem);
         if (newRet == NULL) {
           if (terrno != 0) {
             stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks,
diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c
index 9b27281915..745d030e15 100644
--- a/source/libs/stream/src/streamStart.c
+++ b/source/libs/stream/src/streamStart.c
@@ -322,7 +322,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
   }
 }
 
-int32_t onNormalTaskReady(SStreamTask* pTask) {
+int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
   const char* id = pTask->id.idStr;
 
   streamTaskSetReady(pTask);
@@ -347,7 +347,7 @@ int32_t onNormalTaskReady(SStreamTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t onScanhistoryTaskReady(SStreamTask* pTask) {
+int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) {
   const char* id = pTask->id.idStr;
 
   // set the state to be ready
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 1c951e1452..cac3766893 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -457,11 +457,11 @@ void doInitStateTransferTable(void) {
   streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans));
 
   // initialization event handle
-  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, onNormalTaskReady, false, false);
+  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, false, false);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // scan-history related event

From 023821a80b9fcf23a5cae2a205c8a9227c7b391e Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 15:16:36 +0800
Subject: [PATCH 13/43] enh(tsdb/read-by-column): calc hint size for preloading

---
 source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 36 ++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
index df6b85a889..222b04ac7c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
@@ -300,6 +300,42 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
+    int64_t szHint = 0;
+    if (bData->nColData > 3) {
+      int64_t    offset = 0;
+      SBlockCol  bc = {.cid = 0};
+      SBlockCol *blockCol = &bc;
+
+      size = 0;
+      SColData *colData = tBlockDataGetColDataByIdx(bData, 0);
+      while (blockCol && blockCol->cid < colData->cid) {
+        if (size < hdr->szBlkCol) {
+          size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol);
+        } else {
+          ASSERT(size == hdr->szBlkCol);
+          blockCol = NULL;
+        }
+      }
+
+      if (blockCol && blockCol->flag == HAS_VALUE) {
+        offset = blockCol->offset;
+
+        SColData *colDataEnd = tBlockDataGetColDataByIdx(bData, bData->nColData);
+        while (blockCol && blockCol->cid < colDataEnd->cid) {
+          if (size < hdr->szBlkCol) {
+            size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol);
+          } else {
+            ASSERT(size == hdr->szBlkCol);
+            blockCol = NULL;
+          }
+        }
+
+        if (blockCol && blockCol->flag == HAS_VALUE) {
+          szHint = blockCol->offset + blockCol->szBitmap + blockCol->szOffset + blockCol->szValue - offset;
+        }
+      }
+    }
+
     SBlockCol  bc[1] = {{.cid = 0}};
     SBlockCol *blockCol = bc;
 

From 08e55250127f3a9d4fd68ba7193a24d57a258551 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 15:37:26 +0800
Subject: [PATCH 14/43] tsdb/read-file-page: new arg for hint size

---
 source/dnode/vnode/src/tsdb/tsdbDataFileRW.c  | 31 ++++++++-------
 source/dnode/vnode/src/tsdb/tsdbDef.h         |  2 +-
 .../dnode/vnode/src/tsdb/tsdbReaderWriter.c   | 38 +++++++++++--------
 3 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
index 222b04ac7c..68661a01cc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
@@ -45,7 +45,7 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) {
   int32_t ftype = TSDB_FTYPE_HEAD;
   if (reader->fd[ftype]) {
     code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter),
-                        (uint8_t *)reader->headFooter, sizeof(SHeadFooter));
+                        (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -67,7 +67,7 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) {
   int32_t ftype = TSDB_FTYPE_TOMB;
   if (reader->fd[ftype]) {
     code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter),
-                        (uint8_t *)reader->tombFooter, sizeof(STombFooter));
+                        (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
   reader->ctx->tombFooterLoaded = true;
@@ -161,7 +161,7 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b
       }
 
       code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data,
-                          reader->headFooter->brinBlkPtr->size);
+                          reader->headFooter->brinBlkPtr->size, 0);
       if (code) {
         taosMemoryFree(data);
         TSDB_CHECK_CODE(code, lino, _exit);
@@ -191,7 +191,8 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
   code = tRealloc(&reader->config->bufArr[0], brinBlk->dp->size);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size);
+  code =
+      tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   int32_t size = 0;
@@ -232,7 +233,8 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re
   code = tRealloc(&reader->config->bufArr[0], record->blockSize);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize);
+  code =
+      tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   code = tDecmprBlockData(reader->config->bufArr[0], record->blockSize, bData, &reader->config->bufArr[1]);
@@ -257,8 +259,8 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
   code = tRealloc(&reader->config->bufArr[0], record->blockKeySize);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code =
-      tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize);
+  code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize,
+                      0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // hdr
@@ -296,7 +298,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
       TSDB_CHECK_CODE(code, lino, _exit);
 
       code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize,
-                          reader->config->bufArr[0], hdr->szBlkCol);
+                          reader->config->bufArr[0], hdr->szBlkCol, 0);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
@@ -374,7 +376,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
 
           code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA],
                               record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset,
-                              reader->config->bufArr[1], size1);
+                              reader->config->bufArr[1], size1, szHint);
           TSDB_CHECK_CODE(code, lino, _exit);
 
           code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData,
@@ -402,7 +404,7 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec
     code = tRealloc(&reader->config->bufArr[0], record->smaSize);
     TSDB_CHECK_CODE(code, lino, _exit);
 
-    code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize);
+    code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize, 0);
     TSDB_CHECK_CODE(code, lino, _exit);
 
     // decode sma data
@@ -441,7 +443,7 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t
       }
 
       code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data,
-                          reader->tombFooter->tombBlkPtr->size);
+                          reader->tombFooter->tombBlkPtr->size, 0);
       if (code) {
         taosMemoryFree(data);
         TSDB_CHECK_CODE(code, lino, _exit);
@@ -471,7 +473,8 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB
   code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size);
+  code =
+      tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   int32_t size = 0;
@@ -524,8 +527,8 @@ struct SDataFileWriter {
     STombBlock           tombBlock[1];
     int32_t              tombBlockIdx;
     // range
-    SVersionRange        range;
-    SVersionRange        tombRange;
+    SVersionRange range;
+    SVersionRange tombRange;
   } ctx[1];
 
   STFile   files[TSDB_FTYPE_MAX];
diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h
index da2445dee5..0f512e1306 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDef.h
+++ b/source/dnode/vnode/src/tsdb/tsdbDef.h
@@ -34,7 +34,7 @@ typedef struct SFDataPtr {
 extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD);
 extern void    tsdbCloseFile(STsdbFD **ppFD);
 extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
-extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size);
+extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint);
 extern int32_t tsdbFsyncFile(STsdbFD *pFD);
 
 #ifdef __cplusplus
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 8b9cae42fc..217df923ef 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -283,7 +283,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
+static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
   int32_t code = 0;
   int64_t n = 0;
   int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
@@ -339,7 +339,12 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
     uint8_t *pBlock = NULL;
     int64_t  retrieve_offset = PAGE_OFFSET(pgno, pFD->szPage);
     int64_t  pgnoEnd = pgno - 1 + (bOffset + size - n + szPgCont - 1) / szPgCont;
-    int64_t  retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
+
+    if (szHint > 0) {
+      pgnoEnd = pgno - 1 + (bOffset + szHint - n + szPgCont - 1) / szPgCont;
+    }
+
+    int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
     code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
@@ -350,6 +355,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
     for (int i = 0; i < nPage; ++i) {
       tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
 
+      if (szHint > 0 && n >= size) {
+        continue;
+      }
       memcpy(pFD->pBuf, pBlock + i * pFD->szPage, pFD->szPage);
 
       // check
@@ -375,7 +383,7 @@ _exit:
   return code;
 }
 
-int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
+int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
   int32_t code = 0;
   if (!pFD->pFD) {
     code = tsdbOpenFileImpl(pFD);
@@ -385,7 +393,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size)
   }
 
   if (pFD->s3File && tsS3BlockSize < 0) {
-    return tsdbReadFileS3(pFD, offset, pBuf, size);
+    return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
   } else {
     return tsdbReadFileImp(pFD, offset, pBuf, size);
   }
@@ -1141,7 +1149,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
   if (code) goto _err;
 
   // decode
@@ -1178,7 +1186,7 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size);
+  code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0);
   if (code) goto _err;
 
   // decode
@@ -1211,7 +1219,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
   if (code) goto _err;
 
   // decode
@@ -1242,7 +1250,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size);
+  code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size, 0);
   if (code) goto _err;
 
   // decode
@@ -1276,7 +1284,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
   code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey);
   if (code) goto _err;
 
-  code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey);
+  code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey, 0);
   if (code) goto _err;
 
   SDiskDataHdr hdr;
@@ -1322,7 +1330,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
     code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol);
     if (code) goto _err;
 
-    code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol);
+    code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol, 0);
     if (code) goto _err;
   }
 
@@ -1366,7 +1374,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
         code = tRealloc(&pReader->aBuf[1], size);
         if (code) goto _err;
 
-        code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size);
+        code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size, 0);
         if (code) goto _err;
 
         code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]);
@@ -1392,7 +1400,7 @@ int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockDat
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock);
+  code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock, 0);
   if (code) goto _err;
 
   // decmpr
@@ -1444,7 +1452,7 @@ int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // read
-  code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock);
+  code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // decmpr
@@ -1700,7 +1708,7 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
+  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
   if (code) goto _err;
 
   // // decode
@@ -1740,7 +1748,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
   if (code) goto _err;
 
   // read
-  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
+  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
   if (code) goto _err;
 
   // decode

From 0a90c33b4d1e30607852b2ee6d3b8a20d739939f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 21 Nov 2023 18:22:38 +0800
Subject: [PATCH 15/43] fix(stream): drop the related fill-history task & set
 the task to be ready, if the drop fill-history task msg is missing due to the
 vnode restarting.

---
 include/common/tmsg.h                       |   2 +-
 include/common/tmsgdef.h                    |   1 +
 include/libs/stream/tstream.h               |  20 +-
 source/dnode/mgmt/mgmt_snode/src/smHandle.c |   1 +
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |   1 +
 source/dnode/mnode/impl/src/mndStream.c     | 361 +++++++++++++-------
 source/dnode/vnode/src/inc/vnodeInt.h       |   1 +
 source/dnode/vnode/src/tq/tq.c              |  31 ++
 source/dnode/vnode/src/vnd/vnodeSvr.c       |   5 +
 source/libs/stream/src/streamExec.c         |   2 +-
 10 files changed, 295 insertions(+), 130 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index e17a72992c..73d1ab2473 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3271,7 +3271,7 @@ typedef struct {
   SMsgHead head;
   int64_t  streamId;
   int32_t  taskId;
-} SVPauseStreamTaskReq, SVResetStreamTaskReq;
+} SVPauseStreamTaskReq, SVResetStreamTaskReq, SVDropHTaskReq;
 
 typedef struct {
   int8_t reserved;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 4eb8328caa..2d1148a209 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -271,6 +271,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_PAUSE, "stream-task-pause", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index ad2e9514d1..58439d142a 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -655,17 +655,19 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea
 typedef struct STaskStatusEntry {
   STaskId id;
   int32_t status;
-  int64_t stage;
+  int32_t statusLastDuration; // to record the last duration of current status
+  int32_t stage;
   int32_t nodeId;
-  int64_t verStart;            // start version in WAL, only valid for source task
-  int64_t verEnd;              // end version in WAL, only valid for source task
-  int64_t processedVer;        // only valid for source task
-  int64_t activeCheckpointId;  // current active checkpoint id
-  bool    checkpointFailed;    // denote if the checkpoint is failed or not
-  double  inputQUsed;          // in MiB
+  int64_t verStart;         // start version in WAL, only valid for source task
+  int64_t verEnd;           // end version in WAL, only valid for source task
+  int64_t processedVer;     // only valid for source task
+  int32_t relatedHTask;     // has related fill-history task
+  int64_t activeCheckpointId;     // current active checkpoint id
+  bool    checkpointFailed; // denote if the checkpoint is failed or not
+  double  inputQUsed;       // in MiB
   double  inputRate;
-  double  sinkQuota;     // existed quota size for sink task
-  double  sinkDataSize;  // sink to dest data size
+  double  sinkQuota;        // existed quota size for sink task
+  double  sinkDataSize;     // sink to dst data size
 } STaskStatusEntry;
 
 typedef struct SStreamHbMsg {
diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
index cd81b9873f..6de29f8513 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
@@ -84,6 +84,7 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_HTASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 670eb48bd2..770a0ac6f7 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -833,6 +833,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_HTASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index d2f0a13038..924e6475f1 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -78,6 +78,8 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
 static SArray *extractNodeListFromStream(SMnode *pMnode);
 static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
 
+static SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
+
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
 
 static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char* pMsg);
@@ -1570,6 +1572,123 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {
   sdbCancelFetch(pSdb, pIter);
 }
 
+static void setTaskAttrInResBlock(SStreamObj* pStream, SStreamTask* pTask, SSDataBlock* pBlock, int32_t numOfRows) {
+  SColumnInfoData *pColInfo;
+  int32_t          cols = 0;
+
+  // stream name
+  char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+  STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
+
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false);
+
+  // task id
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+
+  char    idstr[128] = {0};
+  int32_t len = tintToHex(pTask->id.taskId, &idstr[4]);
+  idstr[2] = '0';
+  idstr[3] = 'x';
+  varDataSetLen(idstr, len + 2);
+  colDataSetVal(pColInfo, numOfRows, idstr, false);
+
+  // node type
+  char nodeType[20 + VARSTR_HEADER_SIZE] = {0};
+  varDataSetLen(nodeType, 5);
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  if (pTask->info.nodeId > 0) {
+    memcpy(varDataVal(nodeType), "vnode", 5);
+  } else {
+    memcpy(varDataVal(nodeType), "snode", 5);
+  }
+  colDataSetVal(pColInfo, numOfRows, nodeType, false);
+
+  // node id
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  int64_t nodeId = TMAX(pTask->info.nodeId, 0);
+  colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false);
+
+  // level
+  char level[20 + VARSTR_HEADER_SIZE] = {0};
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    memcpy(varDataVal(level), "source", 6);
+    varDataSetLen(level, 6);
+  } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
+    memcpy(varDataVal(level), "agg", 3);
+    varDataSetLen(level, 3);
+  } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+    memcpy(varDataVal(level), "sink", 4);
+    varDataSetLen(level, 4);
+  }
+
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)level, false);
+
+  // status
+  char status[20 + VARSTR_HEADER_SIZE] = {0};
+  STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
+
+  STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
+  if (pe == NULL) {
+    return;
+  }
+
+  const char *pStatus = streamTaskGetStatusStr(pe->status);
+  STR_TO_VARSTR(status, pStatus);
+
+  // status
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)status, false);
+
+  // stage
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
+
+  // input queue
+  char        vbuf[30] = {0};
+  char        buf[25] = {0};
+  const char *queueInfoStr = "%4.2fMiB (%5.2f%)";
+  sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate);
+  STR_TO_VARSTR(vbuf, buf);
+
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+
+  // output queue
+  //        sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
+  //        STR_TO_VARSTR(vbuf, buf);
+
+  //        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  //        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+    const char *sinkStr = "%.2fMiB";
+    sprintf(buf, sinkStr, pe->sinkDataSize);
+  } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    // offset info
+    const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
+    sprintf(buf, offsetStr, pe->processedVer, pe->verStart, pe->verEnd);
+  }
+
+  STR_TO_VARSTR(vbuf, buf);
+
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+}
+
+static int32_t getNumOfTasks(SArray* pTaskList) {
+  int32_t numOfLevels = taosArrayGetSize(pTaskList);
+
+  int32_t count = 0;
+  for (int32_t i = 0; i < numOfLevels; i++) {
+    SArray *pLevel = taosArrayGetP(pTaskList, i);
+    count += taosArrayGetSize(pLevel);
+  }
+
+  return count;
+}
+
 static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
   SMnode *    pMnode = pReq->info.node;
   SSdb *      pSdb = pMnode->pSdb;
@@ -1585,137 +1704,25 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
     // lock
     taosRLockLatch(&pStream->lock);
 
-    // count task num
-    int32_t sz = taosArrayGetSize(pStream->tasks);
-
-    int32_t count = 0;
-    for (int32_t i = 0; i < sz; i++) {
-      SArray *pLevel = taosArrayGetP(pStream->tasks, i);
-      count += taosArrayGetSize(pLevel);
-    }
-
+    int32_t count = getNumOfTasks(pStream->tasks);
     if (numOfRows + count > rowsCapacity) {
       blockDataEnsureCapacity(pBlock, numOfRows + count);
     }
 
     // add row for each task
-    for (int32_t i = 0; i < sz; i++) {
+    for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); i++) {
       SArray *pLevel = taosArrayGetP(pStream->tasks, i);
-      int32_t levelCnt = taosArrayGetSize(pLevel);
 
-      for (int32_t j = 0; j < levelCnt; j++) {
+      int32_t numOfLevels = taosArrayGetSize(pLevel);
+      for (int32_t j = 0; j < numOfLevels; j++) {
         SStreamTask *pTask = taosArrayGetP(pLevel, j);
-
-        SColumnInfoData *pColInfo;
-        int32_t          cols = 0;
-
-        // stream name
-        char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
-        STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
-
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false);
-
-        // task id
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-
-        char    idstr[128] = {0};
-        int32_t len = tintToHex(pTask->id.taskId, &idstr[4]);
-        idstr[2] = '0';
-        idstr[3] = 'x';
-        varDataSetLen(idstr, len + 2);
-        colDataSetVal(pColInfo, numOfRows, idstr, false);
-
-        // node type
-        char nodeType[20 + VARSTR_HEADER_SIZE] = {0};
-        varDataSetLen(nodeType, 5);
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        if (pTask->info.nodeId > 0) {
-          memcpy(varDataVal(nodeType), "vnode", 5);
-        } else {
-          memcpy(varDataVal(nodeType), "snode", 5);
-        }
-        colDataSetVal(pColInfo, numOfRows, nodeType, false);
-
-        // node id
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        int64_t nodeId = TMAX(pTask->info.nodeId, 0);
-        colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false);
-
-        // level
-        char level[20 + VARSTR_HEADER_SIZE] = {0};
-        if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-          memcpy(varDataVal(level), "source", 6);
-          varDataSetLen(level, 6);
-        } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
-          memcpy(varDataVal(level), "agg", 3);
-          varDataSetLen(level, 3);
-        } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-          memcpy(varDataVal(level), "sink", 4);
-          varDataSetLen(level, 4);
-        }
-
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)level, false);
-
-        // status
-        char status[20 + VARSTR_HEADER_SIZE] = {0};
-
-        STaskId           id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-        STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
-        if (pe == NULL) {
-          continue;
-        }
-
-        const char *pStatus = streamTaskGetStatusStr(pe->status);
-        STR_TO_VARSTR(status, pStatus);
-
-        // status
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)status, false);
-
-        // stage
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
-
-        // input queue
-        char        vbuf[30] = {0};
-        char        buf[25] = {0};
-        const char *queueInfoStr = "%4.2fMiB (%5.2f%)";
-        sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate);
-        STR_TO_VARSTR(vbuf, buf);
-
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
-
-        // output queue
-        //        sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
-        //        STR_TO_VARSTR(vbuf, buf);
-
-        //        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        //        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
-
-        if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-          const char *sinkStr = "%.2fMiB";
-          sprintf(buf, sinkStr, pe->sinkDataSize);
-        } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-          // offset info
-          const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
-          sprintf(buf, offsetStr, pe->processedVer, pe->verStart, pe->verEnd);
-        }
-
-        STR_TO_VARSTR(vbuf, buf);
-
-        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
-
+        setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
         numOfRows++;
       }
     }
 
     // unlock
     taosRUnLockLatch(&pStream->lock);
-
     sdbRelease(pSdb, pStream);
   }
 
@@ -2729,7 +2736,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) {
+static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode) {
   STrans* pTrans = mndAcquireTrans(pMnode, transId);
   if (pTrans != NULL) {
     mInfo("kill checkpoint transId:%d to reset task status", transId);
@@ -2765,6 +2772,88 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) {
   return 0;
 }
 
+static SStreamTask* mndGetStreamTask(STaskId* pId, SStreamObj* pStream) {
+  for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); i++) {
+    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+
+    int32_t numOfLevels = taosArrayGetSize(pLevel);
+    for (int32_t j = 0; j < numOfLevels; j++) {
+      SStreamTask *pTask = taosArrayGetP(pLevel, j);
+      if (pTask->id.taskId == pId->taskId) {
+        return pTask;
+      }
+    }
+  }
+
+  return NULL;
+}
+
+static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) {
+  if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) {
+    if (fabs(pTaskEntry->inputQUsed) <= DBL_EPSILON) {
+      int32_t numOfReady = 0;
+      int32_t numOfTotal = 0;
+      for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+        STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
+        if (pTaskEntry->id.streamId == pId->streamId) {
+          numOfTotal++;
+
+          if (pTaskEntry->id.taskId != pId->taskId && pTaskEntry->status == TASK_STATUS__READY) {
+            numOfReady++;
+          }
+        }
+      }
+
+      if (numOfReady > 0) {
+        mDebug("stream:0x%" PRIx64
+               " %d tasks are ready, %d tasks in stream-scan-history for more than 50s, drop related fill-history task",
+               pTaskEntry->id.streamId, numOfReady, numOfTotal - numOfReady);
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  return false;
+}
+
+// currently only handle the sink task
+// 1. sink task, drop related fill-history task msg is missing
+// 2. other tasks are in ready state for at least 3 * hb_interval
+static int32_t mndDropRelatedFillhistoryTask(SMnode *pMnode, STaskStatusEntry *pTaskEntry, SStreamObj *pStream) {
+  SStreamTask *pTask = mndGetStreamTask(&pTaskEntry->id, pStream);
+  if (pTask == NULL) {
+    mError("failed to get the stream task:0x%x, may have been dropped", (int32_t) pTaskEntry->id.taskId);
+    return -1;
+  }
+
+  SVDropHTaskReq *pReq = rpcMallocCont(sizeof(SVDropHTaskReq));
+  if (pReq == NULL) {
+    mError("failed to malloc in drop related fill-history task, size:%" PRIzu ", code:%s", sizeof(SVDropHTaskReq),
+           tstrerror(TSDB_CODE_OUT_OF_MEMORY));
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReq->head.vgId = htonl(pTask->info.nodeId);
+  pReq->taskId = pTask->id.taskId;
+  pReq->streamId = pTask->id.streamId;
+
+  SRpcMsg msg = {.info.noResp = 1};
+
+  initRpcMsg(&msg, TDMT_STREAM_HTASK_DROP, pReq, sizeof(SVDropHTaskReq));
+
+  mDebug("build and send drop related fill-history task for task:0x%x", pTask->id.taskId);
+
+  SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
+  SEpSet  epset = mndGetVgroupEpset(pMnode, pVgObj);
+  mndReleaseVgroup(pMnode, pVgObj);
+
+  tmsgSendReq(&epset, &msg);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
   int32_t num = taosArrayGetSize(pNodeList);
   mInfo("set node expired for %d nodes", num);
@@ -2865,9 +2954,28 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
       }
     }
 
-    pTaskEntry->status = p->status;
+    if (p->status == pTaskEntry->status) {
+      pTaskEntry->statusLastDuration++;
+    } else {
+      pTaskEntry->status = p->status;
+      pTaskEntry->statusLastDuration = 0;
+    }
+
     if (p->status != TASK_STATUS__READY) {
       mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
+
+      if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) {
+        bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo);
+        if(drop) {
+          SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId);
+          if (pStreamObj == NULL) {
+            mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid);
+          } else {
+            mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj);
+            mndReleaseStream(pMnode, pStreamObj);
+          }
+        }
+      }
     }
   }
 
@@ -2894,3 +3002,18 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   taosArrayDestroy(req.pUpdateNodes);
   return TSDB_CODE_SUCCESS;
 }
+
+SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) {
+  void       *pIter = NULL;
+  SSdb       *pSdb = pMnode->pSdb;
+  SStreamObj *pStream = NULL;
+
+  while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) {
+    if (pStream->uid == streamId) {
+      sdbCancelFetch(pSdb, pIter);
+      return pStream;
+    }
+  }
+
+  return NULL;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index df1720d4a7..16379db053 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -232,6 +232,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg);
 
 int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart);
 int32_t tqRestartStreamTasks(STQ* pTq);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 7be2a357d3..9d16402ee6 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -2014,3 +2014,34 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
   streamMetaReleaseTask(pMeta, pTask);
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) {
+  SVDropHTaskReq* pReq = (SVDropHTaskReq*) pMsg->pCont;
+
+  SStreamMeta* pMeta = pTq->pStreamMeta;
+  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
+  if (pTask == NULL) {
+    tqError("vgId:%d process drop fill-history task req, failed to acquire task:0x%x, it may have been dropped already",
+            pMeta->vgId, pReq->taskId);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  tqDebug("s-task:%s receive drop fill-history msg from mnode", pTask->id.idStr);
+  if (pTask->hTaskInfo.id.taskId == 0) {
+    tqError("vgId:%d s-task:%s not have related fill-history task", pMeta->vgId, pTask->id.idStr);
+    streamMetaReleaseTask(pMeta, pTask);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+  ASSERT(status == TASK_STATUS__STREAM_SCAN_HISTORY);
+
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
+
+  SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId};
+  streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id);
+
+  streamMetaReleaseTask(pMeta, pTask);
+  return TSDB_CODE_SUCCESS;
+}
+
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 33b4114009..8cbca403e3 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -595,6 +595,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
         tqProcessTaskResetReq(pVnode->pTq, pMsg);
       }
     } break;
+    case TDMT_STREAM_HTASK_DROP: {
+      if (pVnode->restored && vnodeIsLeader(pVnode)) {
+        tqProcessTaskDropHTask(pVnode->pTq, pMsg);
+      }
+    } break;
     case TDMT_VND_ALTER_CONFIRM:
       needCommit = pVnode->config.hashChange;
       if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) {
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 43875319b7..8b14846414 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -401,7 +401,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   // 5. save to disk
   pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask, NULL);
 
-  // 6. pause allowed.
+  // 6. add empty delete block
   if ((pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) && taosQueueEmpty(pStreamTask->inputq.queue->pQueue)) {
     SStreamRefDataBlock* pItem = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
 

From c6721be1126b47062ff5722146acf1a4732d1bab Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 15:49:42 +0800
Subject: [PATCH 16/43] fix stt read compilation

---
 source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 23 +++++++++++----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
index 7c3b185e20..f26c6540df 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
@@ -60,7 +60,7 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
   int64_t offset = config->file->size - sizeof(SSttFooter);
   ASSERT(offset >= TSDB_FHDR_SIZE);
 
-  code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter));
+  code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -97,7 +97,7 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
       if (!data) return TSDB_CODE_OUT_OF_MEMORY;
 
       int32_t code =
-          tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size);
+          tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0);
       if (code) {
         taosMemoryFree(data);
         return code;
@@ -125,7 +125,7 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
       if (!data) return TSDB_CODE_OUT_OF_MEMORY;
 
       int32_t code =
-          tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size);
+          tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0);
       if (code) {
         taosMemoryFree(data);
         return code;
@@ -152,7 +152,8 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
       void   *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
       if (!data) return TSDB_CODE_OUT_OF_MEMORY;
 
-      int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size);
+      int32_t code =
+          tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0);
       if (code) {
         taosMemoryFree(data);
         return code;
@@ -177,7 +178,7 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
   code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szBlock);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock);
+  code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   code = tDecmprBlockData(reader->config->bufArr[0], sttBlk->bInfo.szBlock, bData, &reader->config->bufArr[1]);
@@ -209,7 +210,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
   code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey);
+  code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // hdr
@@ -255,7 +256,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
       TSDB_CHECK_CODE(code, lino, _exit);
 
       code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0],
-                          hdr->szBlkCol);
+                          hdr->szBlkCol, 0);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
@@ -296,7 +297,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
           TSDB_CHECK_CODE(code, lino, _exit);
 
           code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr->szBlkCol + blockCol->offset,
-                              reader->config->bufArr[1], size1);
+                              reader->config->bufArr[1], size1, 0);
           TSDB_CHECK_CODE(code, lino, _exit);
 
           code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData,
@@ -321,7 +322,7 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
   code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size);
+  code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0);
   if (code) TSDB_CHECK_CODE(code, lino, _exit);
 
   int64_t size = 0;
@@ -352,7 +353,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
   code = tRealloc(&reader->config->bufArr[0], statisBlk->dp->size);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size);
+  code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size, 0);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   int64_t size = 0;
@@ -405,7 +406,7 @@ struct SSttFileWriter {
 };
 
 static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize,
-                                 TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) {
+                                           TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) {
   if (blockData->nRow == 0) return 0;
 
   int32_t code = 0;

From de4cc2009dc142dce6048272190c7a0aad2d3062 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 16:04:06 +0800
Subject: [PATCH 17/43] fix(stream): fix error.

---
 include/libs/stream/tstream.h           | 2 +-
 source/dnode/mnode/impl/src/mndStream.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 58439d142a..cf9fc1d826 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -656,7 +656,7 @@ typedef struct STaskStatusEntry {
   STaskId id;
   int32_t status;
   int32_t statusLastDuration; // to record the last duration of current status
-  int32_t stage;
+  int64_t stage;
   int32_t nodeId;
   int64_t verStart;         // start version in WAL, only valid for source task
   int64_t verEnd;           // end version in WAL, only valid for source task
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 924e6475f1..aa07172d2a 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -2736,7 +2736,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode) {
+static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t transId) {
   STrans* pTrans = mndAcquireTrans(pMnode, transId);
   if (pTrans != NULL) {
     mInfo("kill checkpoint transId:%d to reset task status", transId);

From 460638d29465940b497a813cdd78e8b9d638d529 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 16:09:02 +0800
Subject: [PATCH 18/43] enh(tsdb/open-file): new s3 failed to connect error
 code

---
 include/util/taoserror.h                       |  1 +
 source/common/src/cos.c                        |  2 ++
 source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 +++++++++++-
 source/util/src/terror.c                       |  1 +
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index ce8db162b6..0256a496df 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -124,6 +124,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_CFG_VALUE             TAOS_DEF_ERROR_CODE(0, 0x0133)
 
 #define TSDB_CODE_IP_NOT_IN_WHITE_LIST          TAOS_DEF_ERROR_CODE(0, 0x0134)
+#define TSDB_CODE_FAILED_TO_CONNECT_S3          TAOS_DEF_ERROR_CODE(0, 0x0135)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)
diff --git a/source/common/src/cos.c b/source/common/src/cos.c
index 1dca5eda9b..2a334a34b8 100644
--- a/source/common/src/cos.c
+++ b/source/common/src/cos.c
@@ -891,6 +891,8 @@ long s3Size(const char *object_name) {
 
   if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
     s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
+
+    return -1;
   }
 
   size = cbd.content_length;
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 217df923ef..d36d15f9b4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -26,7 +26,17 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
   if (pFD->pFD == NULL) {
     int         errsv = errno;
     const char *object_name = taosDirEntryBaseName((char *)path);
-    long        s3_size = tsS3Enabled ? s3Size(object_name) : 0;
+    long        s3_size = 0;
+    if (tsS3Enabled) {
+      long size = s3Size(object_name);
+      if (size < 0) {
+        code = terrno = TSDB_CODE_FAILED_TO_CONNECT_S3;
+        goto _exit;
+      }
+
+      s3_size = size;
+    }
+
     if (tsS3Enabled && !strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
 #ifndef S3_BLOCK_CACHE
       s3EvictCache(path, s3_size);
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index f310db53ef..8847b7d894 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -101,6 +101,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING,              "Database is closing d
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DATA_FMT,             "Invalid data format")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE,            "Invalid configuration value")
 TAOS_DEFINE_ERROR(TSDB_CODE_IP_NOT_IN_WHITE_LIST,          "Not allowed to connect")
+TAOS_DEFINE_ERROR(TSDB_CODE_FAILED_TO_CONNECT_S3,          "Failed to connect to s3 server")
 
 //client
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION,        "Invalid operation")

From f076bfdfcaaa2b0c352f8a05a86dbed6ad4d15cb Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 16:19:26 +0800
Subject: [PATCH 19/43] fix(tsdb/skip-row): remove s3 size condition

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 +-
 source/dnode/vnode/src/tsdb/tsdbFS2.c     | 2 +-
 source/dnode/vnode/src/tsdb/tsdbMerge.c   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 48b622e324..a974eb27bf 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -421,7 +421,7 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
         if (mtime < committer->ctx->now - tsS3UploadDelaySec) {
           committer->ctx->skipTsRow = true;
         }
-      } else if (s3Size(object_name) > 0) {
+      } else /*if (s3Size(object_name) > 0) */ {
         committer->ctx->skipTsRow = true;
       }
     }
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c
index 70a83ebdbb..2feef20649 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@@ -901,7 +901,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
                 if (mtime < now - tsS3UploadDelaySec) {
                   skipMerge = true;
                 }
-              } else if (s3Size(object_name) > 0) {
+              } else /* if (s3Size(object_name) > 0) */ {
                 skipMerge = true;
               }
             }
diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c
index 7babaa6e28..6d968d0828 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMerge.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c
@@ -568,7 +568,7 @@ static int32_t tsdbMerge(void *arg) {
           if (mtime < now - tsS3UploadDelaySec) {
             skipMerge = true;
           }
-        } else if (s3Size(object_name) > 0) {
+        } else /* if (s3Size(object_name) > 0) */ {
           skipMerge = true;
         }
       }

From 810eb8e0e388b12db67b02cef175d6fdd35add6b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 16:25:12 +0800
Subject: [PATCH 20/43] fix(stream): fix syntax error.

---
 source/dnode/mnode/impl/src/mndStream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index aa07172d2a..230e54f5ce 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -2736,7 +2736,7 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char* pDBName, size_t le
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t transId) {
+static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) {
   STrans* pTrans = mndAcquireTrans(pMnode, transId);
   if (pTrans != NULL) {
     mInfo("kill checkpoint transId:%d to reset task status", transId);

From 3034ca2875b15e95b5916abcccf75550c1f87447 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 16:25:39 +0800
Subject: [PATCH 21/43] tsdb/open-file: log not exist file path

---
 source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index d36d15f9b4..abcba75b53 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -58,6 +58,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
       // pFD->szFile = s3_size;
 #endif
     } else {
+      tsdbInfo("no file: %s", path);
       code = TAOS_SYSTEM_ERROR(errsv);
       // taosMemoryFree(pFD);
       goto _exit;

From f7f9546d1ec4a0742192846a23f2473438816b73 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 22 Nov 2023 15:23:00 +0800
Subject: [PATCH 22/43] enh: trigger vnode commit at exit if meta changed, for
 dbs of single replica

---
 source/dnode/vnode/src/inc/meta.h        |  1 +
 source/dnode/vnode/src/meta/metaCommit.c |  2 +-
 source/dnode/vnode/src/meta/metaTable.c  | 10 ++++++++++
 source/dnode/vnode/src/vnd/vnodeCommit.c |  6 ++++--
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index c74ccf6c11..7dbaa66d44 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -79,6 +79,7 @@ struct SMeta {
 
   char*   path;
   SVnode* pVnode;
+  bool    changed;
   TDB*    pEnv;
   TXN*    txn;
   TTB*    pTbDb;
diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c
index f5572e68dd..f8b41e413b 100644
--- a/source/dnode/vnode/src/meta/metaCommit.c
+++ b/source/dnode/vnode/src/meta/metaCommit.c
@@ -56,7 +56,7 @@ int  metaPrepareAsyncCommit(SMeta *pMeta) {
   code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
   metaULock(pMeta);
   code = tdbCommit(pMeta->pEnv, pMeta->txn);
-
+  pMeta->changed = false;
   return code;
 }
 
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index f600925a0b..fe058018d7 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -251,6 +251,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
 
   ++pMeta->pVnode->config.vndStats.numOfSTables;
 
+  pMeta->changed = true;
   metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
 
   return 0;
@@ -325,6 +326,8 @@ _drop_super_table:
 
   metaUpdTimeSeriesNum(pMeta);
 
+  pMeta->changed = true;
+
 _exit:
   tdbFree(pKey);
   tdbFree(pData);
@@ -424,6 +427,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
     metaTimeSeriesNotifyCheck(pMeta);
   }
 
+  pMeta->changed = true;
+
 _exit:
   if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
   tDecoderClear(&dc);
@@ -847,6 +852,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
     }
   }
 
+  pMeta->changed = true;
   metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
             pReq->type);
   return 0;
@@ -895,6 +901,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
     *tbUid = uid;
   }
 
+  pMeta->changed = true;
 _exit:
   tdbFree(pData);
   return rc;
@@ -938,6 +945,8 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
     }
   }
   tSimpleHashCleanup(suidHash);
+
+  pMeta->changed = true;
 }
 
 static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) {
@@ -1970,6 +1979,7 @@ _err:
 }
 
 int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) {
+  pMeta->changed = true;
   switch (pReq->action) {
     case TSDB_ALTER_TABLE_ADD_COLUMN:
     case TSDB_ALTER_TABLE_DROP_COLUMN:
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index f9b5e9168c..a178e1f772 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -13,9 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "meta.h"
+#include "sync.h"
 #include "vnd.h"
 #include "vnodeInt.h"
-#include "sync.h"
 
 extern int32_t tsdbPreCommit(STsdb *pTsdb);
 extern int32_t tsdbCommitBegin(STsdb *pTsdb, SCommitInfo *pInfo);
@@ -155,7 +156,8 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
 
   taosThreadMutexLock(&pVnode->mutex);
   if (pVnode->inUse && diskAvail) {
-    needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) || (pVnode->inUse->size > 0 && atExit);
+    needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) ||
+                 (atExit && (pVnode->inUse->size > 0 || pVnode->pMeta->changed));
   }
   taosThreadMutexUnlock(&pVnode->mutex);
   return needCommit;

From 699c14e78104345dcc9740fdbce8febfcf0893ad Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Thu, 23 Nov 2023 16:51:07 +0800
Subject: [PATCH 23/43] fix: slow down report when error occurs

---
 source/client/src/clientEnv.c                   | 3 +++
 source/dnode/mgmt/mgmt_dnode/src/dmWorker.c     | 3 +++
 source/libs/executor/src/dynqueryctrloperator.c | 6 +++---
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index da24bc0a3b..212c42125e 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -555,6 +555,9 @@ static void *tscCrashReportThreadFp(void *param) {
         if (pFile) {
           taosReleaseCrashLogFile(pFile, false);
           pFile = NULL;
+
+          taosMsleep(sleepTime);
+          loopTimes = 0;
           continue;
         }
       } else {
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
index 18da1d638c..d6bdaf51bc 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
@@ -127,6 +127,9 @@ static void *dmCrashReportThreadFp(void *param) {
         if (pFile) {
           taosReleaseCrashLogFile(pFile, false);
           pFile = NULL;
+
+          taosMsleep(sleepTime);
+          loopTimes = 0;
           continue;
         }
       } else {
diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c
index 8fc46e0239..9e7f1144f8 100755
--- a/source/libs/executor/src/dynqueryctrloperator.c
+++ b/source/libs/executor/src/dynqueryctrloperator.c
@@ -503,15 +503,15 @@ static void seqJoinLaunchNewRetrieveImpl(SOperatorInfo* pOperator, SSDataBlock**
     T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
   }
 
-  qError("dynamic post task begin");
+  qDebug("%s dynamic post task begin", GET_TASKID(pOperator->pTaskInfo));
   *ppRes = pOperator->pDownstream[1]->fpSet.getNextExtFn(pOperator->pDownstream[1], pParam);
   if (*ppRes) {
     pPost->isStarted = true;
     pStbJoin->execInfo.postBlkNum++;
     pStbJoin->execInfo.postBlkRows += (*ppRes)->info.rows;
-    qError("join res block retrieved");
+    qDebug("%s join res block retrieved", GET_TASKID(pOperator->pTaskInfo));
   } else {
-    qError("Empty join res block retrieved");
+    qDebug("%s Empty join res block retrieved", GET_TASKID(pOperator->pTaskInfo));
   }
 }
 

From 205077d23037e048a12e0693cee64d38d39d3f8c Mon Sep 17 00:00:00 2001
From: charles <flyingangel_2008@163.com>
Date: Thu, 23 Nov 2023 17:09:33 +0800
Subject: [PATCH 24/43] add test case for ts-4295

---
 tests/parallel_test/cases.task            |  1 +
 tests/system-test/1-insert/test_ts4295.py | 49 +++++++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 tests/system-test/1-insert/test_ts4295.py

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 92eaec52b5..262a5d573c 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -296,6 +296,7 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
diff --git a/tests/system-test/1-insert/test_ts4295.py b/tests/system-test/1-insert/test_ts4295.py
new file mode 100644
index 0000000000..89e445f3c1
--- /dev/null
+++ b/tests/system-test/1-insert/test_ts4295.py
@@ -0,0 +1,49 @@
+import os
+import sys 
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from math import inf
+import taos
+
+class TDTestCase:
+    """Verify inserting varbinary type data of ts-4295
+    """
+    def init(self, conn, logSql, replicaVer=1):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), True)
+        self.conn = conn
+        self.db_name = "db"
+        self.stable_name = "st"
+
+    def run(self):
+        tdSql.execute("create database if not exists %s" % self.db_name)
+        tdSql.execute("use %s" % self.db_name)
+        # create super table
+        tdSql.execute("create table %s (ts timestamp, c1 varbinary(32)) tags (t1 int)" % self.stable_name)
+        # create child table
+        child_table_list = []
+        for i in range(10):
+            child_table_name = "ct_" + str(i+1)
+            child_table_list.append(child_table_name)
+            tdSql.execute("create table %s using st tags(%s);" % (child_table_name, str(i+1)))
+            tdLog.info("create table %s successfully" % child_table_name)
+        # insert data
+        for i in range(100):
+            sql = "insert into table_name values"
+            for j in range(10000):
+                sql += "(now+%ss, '0x7661726331')," % str(j+1)
+            for child_table in child_table_list:
+                tdSql.execute(sql.replace("table_name", child_table))
+                tdLog.info("Insert data into %s successfully" % child_table)
+            tdLog.info("Insert data round %s successfully" % str(i+1))
+        tdSql.execute("flush database %s" % self.db_name)
+
+    def stop(self):
+        tdSql.execute("drop database if exists %s" % self.db_name)
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From 9334fbcb06d9859e1073dee77c2951b813ea5962 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 17:14:26 +0800
Subject: [PATCH 25/43] tsdb/read-by-column: zero hint size when i's nonzero

---
 source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
index 68661a01cc..4847ab0e33 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
@@ -376,7 +376,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
 
           code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA],
                               record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset,
-                              reader->config->bufArr[1], size1, szHint);
+                              reader->config->bufArr[1], size1, i > 0 ? 0 : szHint);
           TSDB_CHECK_CODE(code, lino, _exit);
 
           code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData,

From 3c3763a7742208544c62392d2259965502676635 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 23 Nov 2023 17:30:49 +0800
Subject: [PATCH 26/43] tsdb/read-by-column: fix last column index

---
 source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
index 4847ab0e33..e1625c9ddb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
@@ -322,7 +322,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
       if (blockCol && blockCol->flag == HAS_VALUE) {
         offset = blockCol->offset;
 
-        SColData *colDataEnd = tBlockDataGetColDataByIdx(bData, bData->nColData);
+        SColData *colDataEnd = tBlockDataGetColDataByIdx(bData, bData->nColData - 1);
         while (blockCol && blockCol->cid < colDataEnd->cid) {
           if (size < hdr->szBlkCol) {
             size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol);

From 2e922fefc6ef5f5b3287bf7441861546f4cb2c22 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Nov 2023 18:04:09 +0800
Subject: [PATCH 27/43] fix(stream): reduce the sink throughput. and fix the
 bug in status check.

---
 source/dnode/mnode/impl/src/mndStream.c | 7 +++++--
 source/libs/stream/src/streamTask.c     | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 230e54f5ce..db013017e3 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -2798,8 +2798,11 @@ static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStream
         if (pTaskEntry->id.streamId == pId->streamId) {
           numOfTotal++;
 
-          if (pTaskEntry->id.taskId != pId->taskId && pTaskEntry->status == TASK_STATUS__READY) {
-            numOfReady++;
+          if (pTaskEntry->id.taskId != pId->taskId) {
+            STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
+            if (pEntry->status == TASK_STATUS__READY) {
+              numOfReady++;
+            }
           }
         }
       }
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index c93d658adb..beaab0a415 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -451,7 +451,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
 
   // 2MiB per second for sink task
   // 50 times sink operator per second
-  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate, pTask->id.idStr);
+  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr);
 
   TdThreadMutexAttr attr = {0};
   int               code = taosThreadMutexAttrInit(&attr);

From a3737fe54dd6c8fac4c772743ff51ec69eb357db Mon Sep 17 00:00:00 2001
From: root <root@DESKTOP-8H5AKPB>
Date: Thu, 23 Nov 2023 18:08:08 +0800
Subject: [PATCH 28/43] Fix bug [4258]: add process ignore/no timezone for
 timetruncate unit is 1w

---
 source/libs/scalar/src/sclfunc.c          | 24 +++++++++++++++++++----
 tests/system-test/2-query/timetruncate.py | 18 ++++++++++++++---
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 6144ebd340..734d1e7d17 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1457,13 +1457,29 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
       }
       case 604800000: { /* 1w */
         if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-          timeVal = timeVal / 1000 / 604800 * 604800 * 1000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)604800) * 1000);
+          } else {
+            timeVal = timeVal / 1000 / 604800 * 604800 * 1000;
+          }
         } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
-          timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)604800) * 1000000);
+          } else {
+            timeVal = timeVal / 1000000 / 604800 * 604800 * 1000000;
+          }
         } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
-          timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)604800) * 1000000000);
+          } else {
+            timeVal = timeVal / 1000000000 / 604800 * 604800 * 1000000000;
+          }
         } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
-          timeVal = timeVal * factor / factor / 604800 * 604800 * factor;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1)) % (((int64_t)604800L) * factor);
+          } else {
+            timeVal = timeVal * factor / factor / 604800 * 604800 * factor;
+          }
         } else {
           colDataSetNULL(pOutput->columnData, i);
           continue;
diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py
index a59180c2b1..09bdfcef63 100644
--- a/tests/system-test/2-query/timetruncate.py
+++ b/tests/system-test/2-query/timetruncate.py
@@ -58,7 +58,11 @@ class TDTestCase:
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0]))
-                tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000)
+                if ignore_tz == 0:
+                    tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000)
+                else:
+                    # assuming the client timezone is UTC+0800
+                    tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000) % (86400 * 7 * 1000)))
 
     def check_us_timestamp(self,unit,date_time, ignore_tz):
         if unit.lower() == '1u':
@@ -92,7 +96,11 @@ class TDTestCase:
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0]))
-                tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000)
+                if ignore_tz == 0:
+                    tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000)
+                else:
+                    # assuming the client timezone is UTC+0800
+                    tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 7 * 1000000)))
 
     def check_ns_timestamp(self,unit,date_time, ignore_tz):
         if unit.lower() == '1b':
@@ -130,7 +138,11 @@ class TDTestCase:
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 if self.rest_tag != 'rest':
-                    tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000)
+                    if ignore_tz == 0:
+                        tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000)
+                    else:
+                        # assuming the client timezone is UTC+0800
+                        tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600  * 1000000000) % (86400 * 7 * 1000000000)))
 
     def check_tb_type(self,unit,tb_type,ignore_tz):
         if tb_type.lower() == 'ntb':

From e2c5902d3f2552c6504c9cc6d815b531c8529d36 Mon Sep 17 00:00:00 2001
From: charles <flyingangel_2008@163.com>
Date: Thu, 23 Nov 2023 19:23:02 +0800
Subject: [PATCH 29/43] add test case for task td-27388

---
 tests/parallel_test/cases.task             |  1 +
 tests/system-test/1-insert/test_td27388.py | 97 ++++++++++++++++++++++
 2 files changed, 98 insertions(+)
 create mode 100644 tests/system-test/1-insert/test_td27388.py

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 262a5d573c..9e669c9212 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -297,6 +297,7 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
diff --git a/tests/system-test/1-insert/test_td27388.py b/tests/system-test/1-insert/test_td27388.py
new file mode 100644
index 0000000000..7b49a63dbb
--- /dev/null
+++ b/tests/system-test/1-insert/test_td27388.py
@@ -0,0 +1,97 @@
+import random
+import string
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.sqlset import *
+from util import constant
+from util.common import *
+
+
+class TDTestCase:
+    """Verify the insert with format exception for task TD-27388
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.dbname = 'db'
+        self.stbname = 'st'
+        tdSql.execute("create database {};".format(self.dbname))
+        tdSql.execute("use {};".format(self.dbname))
+        tdSql.execute("create table st (ts timestamp, col1 int, col2 varchar(64)) tags (t1 int, t2 varchar(32));")
+
+    def test_half_quotes(self):
+        sql_list = [
+            "insert into t1 using st tags(1, 'tag1) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, tag1') values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, test msg');",
+            "insert into t1 using st tags(1, 'tag1' values(now, 1, test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg)';",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2) values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, tag2') values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1) values(now, 1, 'test msg') t2 using st tags(2, 'tag2) values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg) t2 using st tags(2, 'tag2') values(now, 2, test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2') values(now, 2, 'test msg);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2') values(now, 2, 'test msg);"
+        ]
+        for sql in sql_list:
+            tdLog.debug("execute harlf quotes sql: %s" % sql)
+            tdSql.error(sql)
+
+    def test_esc(self):
+        sql_list = [
+            "insert into t1 using st tags(1, 'tag1\\') values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, \\'tag1') values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg\\');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, \\'test msg');",
+            "insert into t1 using st tags(1, \\'tag1\\') values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, \\'test msg\\');",
+            "insert into t1 using st tags(1, \\'tag1\\') values(now, 1, \\'test msg\\');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2\\') values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, \\'tag2') values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2') values(now, 2, \\'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2') values(now, 2, 'test msg\\');",
+            "insert into t1 using st tags(1, \\'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2\\') values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2') values(now, 2, \\'test msg\\');"
+        ]
+        for sql in sql_list:
+            tdLog.debug("execute escape character sql: %s" % sql)
+            tdSql.error(sql)
+
+    def test_specific_character(self):
+        sql_list = [
+            "insert into t1 using st tags(1, 'tag1$) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1,) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1'') values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1() values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1*) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1+) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1,) values(now, 1, 'test msg');",
+            "isnert into t1 using st tags(1, 'tag1-) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1.) values(now, 1, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg$);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg,);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg+%+-.);",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2$) values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2,) values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2'') values(now, 2, 'test msg');",
+            "insert into t1 using st tags(1, 'tag1') values(now, 1, 'test msg') t2 using st tags(2, 'tag2() values(now, 2, 'test msg');"
+        ]
+        for sql in sql_list:
+            tdLog.debug("execute specific character sql: %s" % sql)
+            tdSql.error(sql)
+
+    def run(self):
+        self.test_half_quotes()
+        self.test_esc()
+        self.test_specific_character()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From f7ac2d9d3922489c762d6d568d7b441d7ebf4b6b Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 23 Nov 2023 19:40:02 +0800
Subject: [PATCH 30/43] test: add alter table test

---
 tests/script/api/alterTableTest.c | 264 +++++++++++++++++++++++-------
 tests/script/api/makefile         |   2 +
 2 files changed, 209 insertions(+), 57 deletions(-)

diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c
index 14605f4b8b..402cdb347d 100644
--- a/tests/script/api/alterTableTest.c
+++ b/tests/script/api/alterTableTest.c
@@ -17,8 +17,15 @@
 // to compile: gcc -o demo demo.c -ltaos
 
 /**
- *  passwdTest.c
+ *  alterTableTest.c
+ *   - for JIRA: PI-23
  *   - Run the test case in clear TDengine environment with default root passwd 'taosdata'
+ *
+ * Usage Example: check add column for stable
+ *  step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables.
+ *  step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously.
+ * 
+ *  Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..."
  */
 
 #include <inttypes.h>
@@ -28,37 +35,35 @@
 #include <unistd.h>
 #include "taos.h"  // TAOS header file
 
-#define nDup      3
-#define USER_LEN  24
-#define BUF_LEN   1024
-#define DB        "d0"
-#define DB_BUFFER 32
-#define STB       "stb"
-#define CTB       "ctb"
-#define COL       "c"
+typedef enum {
+  CHECK_ALTER_STABLE_ADD_COL = 0,
+  CHECK_ALTER_STABLE_ADD_TAG = 1,
+  CHECK_ALTER_STABLE_MODIFY_COL = 2,
+  CHECK_ALTER_STABLE_MODIFY_TAG = 3,
+  CHECK_ALTER_NTABLE_ADD_COL = 4,
+  CHECK_ALTER_NTABLE_MODIFY_COL = 5,
+} ENUM_CHECK_ALTER_TYPE;
 
-#if 0
-#define STB_NUM 125
-#define CTB_NUM 960
-#define COL_NUM 52
-#define COL_NCHAR_LEN 320
-#else
-#define STB_NUM       5
-#define CTB_NUM       1
-#define COL_NUM       505
-#define COL_NCHAR_LEN 32
-#endif
-typedef uint16_t VarDataLenT;
+#define nDup           3
+#define USER_LEN       24
+#define BUF_LEN        1024
+#define DB             "d0"
+#define DB_BUFFER      32
+#define STB            "stb"
+#define NTB            "ntb"
+#define CTB            "ctb"
+#define COL            "c"
+#define STB_NUM        10
+#define NTB_NUM        20
+#define CTB_NUM        1
+#define COL_NUM        505
+#define TAG_NUM        127
+#define STB_NUM_MODIFY 100  // for modify columns/tags(increase the number if not easy to reproduced)
+#define NTB_NUM_MODIFY 500
+#define COL_NCHAR_LEN  32
 
 int32_t isDropDb = 0;
-
-#define TSDB_NCHAR_SIZE    sizeof(int32_t)
-#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
-
-#define GET_FLOAT_VAL(x)  (*(float *)(x))
-#define GET_DOUBLE_VAL(x) (*(double *)(x))
-
-#define varDataLen(v) ((VarDataLenT *)(v))[0]
+int32_t checkType = 0;
 
 static int32_t queryDB(TAOS *taos, char *command, bool skipError) {
   int       i;
@@ -92,16 +97,7 @@ static int32_t queryDB(TAOS *taos, char *command, bool skipError) {
   taos_free_result(pSql);
 }
 
-static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qstr, int32_t addColMode) {
-  if (isDropDb) {
-    sprintf(qstr, "drop database if exists %s", DB);
-    queryDB(taos, qstr, false);
-    sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER);
-    queryDB(taos, qstr, false);
-  }
-  sprintf(qstr, "use %s", DB);
-  queryDB(taos, qstr, false);
-
+static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
   // create stb
   if (isDropDb) {
     for (int i = 0; i < STB_NUM; ++i) {
@@ -117,27 +113,21 @@ static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qs
   }
 
   if (isDropDb) {
-    printf("sleep 86400s to wait other terminal\n");
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
     sleep(86400);
   }
 
+  int32_t     colNum = type == CHECK_ALTER_STABLE_ADD_COL ? COL_NUM : TAG_NUM;
+  const char *colName = type == CHECK_ALTER_STABLE_ADD_COL ? "column" : "tag";
+
   // alter stb cols
-  if (addColMode == 0) {
-    for (int i = 0; i < STB_NUM; ++i) {
-      for (int c = 1; c < COL_NUM; ++c) {
-        sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN);
-        queryDB(taos, qstr, true);
-      }
-      sprintf(qstr, "desc %s_%d;", STB, i);
-      queryDB(taos, qstr, false);
-    }
-  } else if (addColMode == 1) {
+  for (int i = 0; i < STB_NUM; ++i) {
     for (int c = 1; c < COL_NUM; ++c) {
-      for (int i = 0; i < STB_NUM; ++i) {
-        sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN);
-        queryDB(taos, qstr, true);
-      }
+      sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN);
+      queryDB(taos, qstr, true);
     }
+    sprintf(qstr, "desc %s_%d;", STB, i);
+    queryDB(taos, qstr, false);
   }
 
   // check
@@ -147,28 +137,188 @@ static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qs
   }
 }
 
+static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
+  // create stb
+  if (isDropDb) {
+    for (int i = 0; i < STB_NUM_MODIFY; ++i) {
+      sprintf(
+          qstr,
+          "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 "
+          "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 "
+          "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 "
+          "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 "
+          "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 "
+          "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 "
+          "NCHAR(160),c_35 NCHAR(160)) tags(t_0 NCHAR(80), t_1 NCHAR(80), t_2 NCHAR(80), t_3 NCHAR(80),t_4 "
+          "NCHAR(80),t_5 NCHAR(80),t_6 NCHAR(80),t_7 NCHAR(80),t_8 NCHAR(80),t_9 NCHAR(80),t_10 NCHAR(80),t_11 "
+          "NCHAR(80),t_12 NCHAR(80),t_13 NCHAR(80),t_14 NCHAR(80),t_15 NCHAR(80),t_16 NCHAR(80),t_17 NCHAR(80),t_18 "
+          "NCHAR(80),t_19 NCHAR(80),t_20 NCHAR(80),t_21 NCHAR(80),t_22 NCHAR(80),t_23 NCHAR(80),t_24 NCHAR(80),t_25 "
+          "NCHAR(80),t_26 NCHAR(80),t_27 NCHAR(80),t_28 NCHAR(80),t_29 NCHAR(80),t_30 NCHAR(80),t_31 NCHAR(80),t_32 "
+          "NCHAR(80),t_33 NCHAR(80),t_34 NCHAR(80),t_35 NCHAR(80));",
+          STB, i);
+      queryDB(taos, qstr, false);
+    }
+  }
+
+  if (isDropDb) {
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    sleep(86400);
+  }
+
+  int32_t     colLen = type == CHECK_ALTER_STABLE_MODIFY_COL ? 455 : 115;
+  const char *colName = type == CHECK_ALTER_STABLE_MODIFY_COL ? "column c_" : "tag t_";
+
+  // alter stb cols
+  for (int i = 0; i < STB_NUM_MODIFY; ++i) {
+    for (int c = 0; c < 36; ++c) {
+      sprintf(qstr, "alter table %s_%d modify %s%d NCHAR(%d);", STB, i, colName, c, colLen);
+      queryDB(taos, qstr, true);
+      //   usleep(1000);
+    }
+    sprintf(qstr, "desc %s_%d;", STB, i);
+    queryDB(taos, qstr, false);
+  }
+
+  // check
+  for (int i = 0; i < STB_NUM_MODIFY; ++i) {
+    sprintf(qstr, "desc %s_%d;", STB, i);
+    queryDB(taos, qstr, false);
+  }
+}
+
+static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) {
+  // create ntb
+  if (isDropDb) {
+    for (int i = 0; i < NTB_NUM; ++i) {
+      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32));", NTB, i, COL, 0);
+      queryDB(taos, qstr, false);
+    }
+  }
+
+  if (isDropDb) {
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    sleep(86400);
+  }
+
+  // alter ntb cols
+  for (int i = 0; i < NTB_NUM; ++i) {
+    for (int c = 1; c < COL_NUM; ++c) {
+      sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", NTB, i, c, COL_NCHAR_LEN);
+      queryDB(taos, qstr, true);
+    }
+    sprintf(qstr, "desc %s_%d;", NTB, i);
+    queryDB(taos, qstr, false);
+  }
+
+  // check
+  for (int i = 0; i < NTB_NUM; ++i) {
+    sprintf(qstr, "desc %s_%d;", NTB, i);
+    queryDB(taos, qstr, false);
+  }
+}
+
+static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) {
+  // create stb
+  if (isDropDb) {
+    for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
+      sprintf(
+          qstr,
+          "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 "
+          "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 "
+          "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 "
+          "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 "
+          "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 "
+          "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 "
+          "NCHAR(160),c_35 NCHAR(160));",
+          NTB, i);
+      queryDB(taos, qstr, false);
+    }
+  }
+
+  if (isDropDb) {
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    sleep(86400);
+  }
+
+  // alter ntb cols
+  for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
+    for (int c = 0; c < 36; ++c) {
+      sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455);
+      queryDB(taos, qstr, true);
+      //   usleep(1000);
+    }
+    sprintf(qstr, "desc %s_%d;", NTB, i);
+    queryDB(taos, qstr, false);
+  }
+
+  // check
+  for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
+    sprintf(qstr, "desc %s_%d;", NTB, i);
+    queryDB(taos, qstr, false);
+  }
+}
+
 int main(int argc, char *argv[]) {
   char qstr[1024];
 
   // connect to server
   if (argc < 2) {
-    printf("please input server-ip \n");
+    printf("please input server-ip \n");  // e.g. localhost
     return 0;
   }
 
   if (argc < 3) {
-    printf("please specify if drop DB\n");
+    printf("please specify if drop DB to clear env\n");  // 0 not drop, 1 drop
     return 0;
   }
 
   isDropDb = atoi(argv[2]);
 
+  if (argc < 4) {
+    printf("please specify check type\n");  // enum of ENUM_CHECK_ALTER_TYPE
+    return 0;
+  }
+
+  checkType = atoi(argv[3]);
+
   TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
   if (taos == NULL) {
     printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/);
     exit(1);
   }
-  createDatabaseAlterStbColumns(taos, argv[1], qstr, 0);
+
+  if (isDropDb) {
+    sprintf(qstr, "drop database if exists %s", DB);
+    queryDB(taos, qstr, false);
+    sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER);
+    queryDB(taos, qstr, false);
+  }
+  sprintf(qstr, "use %s", DB);
+  queryDB(taos, qstr, false);
+
+  switch (checkType) {
+    case CHECK_ALTER_STABLE_ADD_COL:  // reproduced in 3.0.7.1
+      createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL);
+      break;
+    case CHECK_ALTER_STABLE_ADD_TAG:  // reproduced in 3.0.7.1
+      createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG);
+      break;
+    case CHECK_ALTER_STABLE_MODIFY_COL:  // not reproduced in 3.0.7.1 since already checked in mnode
+      createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL);
+      break;
+    case CHECK_ALTER_STABLE_MODIFY_TAG:  // reproduced in 3.0.7.1
+      createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG);
+      break;
+    case CHECK_ALTER_NTABLE_ADD_COL:  // not reproduced in 3.0.7.1
+      createDatabaseAlterNtbAddColumn(taos, argv[1], qstr);
+      break;
+    case CHECK_ALTER_NTABLE_MODIFY_COL:  // not reproduced in 3.0.7.1
+      createDatabaseAlterNtbModifyColumn(taos, argv[1], qstr);
+      break;
+    default:
+      printf("unkown check type:%d\n", checkType);
+      break;
+  }
 
   taos_close(taos);
   taos_cleanup();
diff --git a/tests/script/api/makefile b/tests/script/api/makefile
index d848dca194..2bc127d995 100644
--- a/tests/script/api/makefile
+++ b/tests/script/api/makefile
@@ -16,6 +16,7 @@ exe:
 	gcc $(CFLAGS) ./dbTableRoute.c  -o $(ROOT)dbTableRoute $(LFLAGS)
 	gcc $(CFLAGS) ./insertSameTs.c  -o $(ROOT)insertSameTs $(LFLAGS)
 	gcc $(CFLAGS) ./passwdTest.c  -o $(ROOT)passwdTest $(LFLAGS)
+	gcc $(CFLAGS) ./alterTableTest.c  -o $(ROOT)alterTableTest $(LFLAGS)
 	gcc $(CFLAGS) ./whiteListTest.c  -o $(ROOT)whiteListTest $(LFLAGS)
 	gcc $(CFLAGS) ./insert_stb.c  -o $(ROOT)insert_stb $(LFLAGS)
 	gcc $(CFLAGS) ./tmqViewTest.c  -o $(ROOT)tmqViewTest $(LFLAGS)
@@ -27,6 +28,7 @@ clean:
 	rm $(ROOT)dbTableRoute
 	rm $(ROOT)insertSameTs
 	rm $(ROOT)passwdTest
+	rm $(ROOT)alterTableTest
 	rm $(ROOT)whiteListTest
 	rm $(ROOT)insert_stb
 	rm $(ROOT)tmqViewTest

From 56c570cdcabdd22bc84edfcd850dc426f8059610 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 23 Nov 2023 19:51:48 +0800
Subject: [PATCH 31/43] test: add alter table test

---
 tests/script/api/alterTableTest.c | 37 +++++++++++++++----------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c
index 402cdb347d..e1cf3f1b9a 100644
--- a/tests/script/api/alterTableTest.c
+++ b/tests/script/api/alterTableTest.c
@@ -24,7 +24,7 @@
  * Usage Example: check add column for stable
  *  step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables.
  *  step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously.
- * 
+ *
  *  Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..."
  */
 
@@ -97,11 +97,11 @@ static int32_t queryDB(TAOS *taos, char *command, bool skipError) {
   taos_free_result(pSql);
 }
 
-static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
+static void checkAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
   // create stb
   if (isDropDb) {
     for (int i = 0; i < STB_NUM; ++i) {
-      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(16));", STB, i, COL,
+      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(32));", STB, i, COL,
               0);
       queryDB(taos, qstr, false);
       // create ctb
@@ -113,7 +113,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *
   }
 
   if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
     sleep(86400);
   }
 
@@ -122,7 +122,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *
 
   // alter stb cols
   for (int i = 0; i < STB_NUM; ++i) {
-    for (int c = 1; c < COL_NUM; ++c) {
+    for (int c = 1; c < colNum; ++c) {
       sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN);
       queryDB(taos, qstr, true);
     }
@@ -137,7 +137,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *
   }
 }
 
-static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
+static void checkAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
   // create stb
   if (isDropDb) {
     for (int i = 0; i < STB_NUM_MODIFY; ++i) {
@@ -161,7 +161,7 @@ static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, cha
   }
 
   if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
     sleep(86400);
   }
 
@@ -186,7 +186,7 @@ static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, cha
   }
 }
 
-static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) {
+static void checkAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) {
   // create ntb
   if (isDropDb) {
     for (int i = 0; i < NTB_NUM; ++i) {
@@ -196,7 +196,7 @@ static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *
   }
 
   if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
     sleep(86400);
   }
 
@@ -217,8 +217,8 @@ static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *
   }
 }
 
-static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) {
-  // create stb
+static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) {
+  // create ntb
   if (isDropDb) {
     for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
       sprintf(
@@ -236,7 +236,7 @@ static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, cha
   }
 
   if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n");
+    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
     sleep(86400);
   }
 
@@ -245,7 +245,6 @@ static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, cha
     for (int c = 0; c < 36; ++c) {
       sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455);
       queryDB(taos, qstr, true);
-      //   usleep(1000);
     }
     sprintf(qstr, "desc %s_%d;", NTB, i);
     queryDB(taos, qstr, false);
@@ -298,22 +297,22 @@ int main(int argc, char *argv[]) {
 
   switch (checkType) {
     case CHECK_ALTER_STABLE_ADD_COL:  // reproduced in 3.0.7.1
-      createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL);
+      checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL);
       break;
     case CHECK_ALTER_STABLE_ADD_TAG:  // reproduced in 3.0.7.1
-      createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG);
+      checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG);
       break;
     case CHECK_ALTER_STABLE_MODIFY_COL:  // not reproduced in 3.0.7.1 since already checked in mnode
-      createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL);
+      checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL);
       break;
     case CHECK_ALTER_STABLE_MODIFY_TAG:  // reproduced in 3.0.7.1
-      createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG);
+      checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG);
       break;
     case CHECK_ALTER_NTABLE_ADD_COL:  // not reproduced in 3.0.7.1
-      createDatabaseAlterNtbAddColumn(taos, argv[1], qstr);
+      checkAlterNtbAddColumn(taos, argv[1], qstr);
       break;
     case CHECK_ALTER_NTABLE_MODIFY_COL:  // not reproduced in 3.0.7.1
-      createDatabaseAlterNtbModifyColumn(taos, argv[1], qstr);
+      checkAlterNtbModifyColumn(taos, argv[1], qstr);
       break;
     default:
       printf("unkown check type:%d\n", checkType);

From 9d9a88b3edc2bdd04e4cc7923c5ac5d1a18cb3b7 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Nov 2023 19:52:32 +0800
Subject: [PATCH 32/43] set datablock table name

---
 source/libs/executor/src/scanoperator.c | 36 ++++++++++++++-----------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 28832ffec8..448c585869 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1525,6 +1525,17 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
+  SExprSupp*    pTbNameCalSup = &pInfo->tbnameCalSup;
+  blockDataCleanup(pInfo->pCreateTbRes);
+  if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) {
+    pBlock->info.parTbName[0] = 0;
+  } else {
+    appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
+                         pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore);
+  }
+}
+
 static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
   blockDataCleanup(pDestBlock);
   int32_t rows = pSrcBlock->info.rows;
@@ -1549,15 +1560,21 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS
   for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
     uint64_t srcUid = srcUidData[i];
     uint64_t groupId = srcGp[i];
-    char*    tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0};
+    char     tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0};
     if (groupId == 0) {
       groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver);
     }
     if (pInfo->tbnameCalSup.pExprInfo) {
       void* parTbname = NULL;
-      pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
-
-      memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
+      code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
+      if (code != TSDB_CODE_SUCCESS) {
+        SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver);
+        printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo));
+        calBlockTbName(pInfo, pPreRes);
+        memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName));
+      } else {
+        memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
+      }
       varDataSetLen(tbname, strlen(varDataVal(tbname)));
       pInfo->stateStore.streamStateFreeVal(parTbname);
     }
@@ -1583,17 +1600,6 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
   return code;
 }
 
-static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
-  SExprSupp*    pTbNameCalSup = &pInfo->tbnameCalSup;
-  blockDataCleanup(pInfo->pCreateTbRes);
-  if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) {
-    pBlock->info.parTbName[0] = 0;
-  } else {
-    appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
-                         pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore);
-  }
-}
-
 void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
                                       uint64_t* pGp, void* pTbName) {
   SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

From e1eda752e7c8d45cc3d99d666cc17329355c6805 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 23 Nov 2023 20:44:10 +0800
Subject: [PATCH 33/43] fix:[TS-4242] tmq status is always RECOVER

---
 source/client/src/clientTmq.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 6aebba7c02..d05cdc0156 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1216,7 +1216,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   }
 
   int32_t retryCnt = 0;
-  while (TSDB_CODE_MND_CONSUMER_NOT_READY == syncAskEp(tmq)) {
+  while (syncAskEp(tmq) != 0) {
     if (retryCnt++ > MAX_RETRY_COUNT) {
       tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
       code = TSDB_CODE_MND_CONSUMER_NOT_READY;
@@ -1454,6 +1454,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
   if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) {
     tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d",
             tmq->consumerId, tmq->epoch, epoch, topicNumGet);
+    if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){
+      atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
+    }
     return false;
   }
 
@@ -1965,9 +1968,14 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
     return NULL;
   }
 
-  while (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
+  while (1) {
+    if(atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__RECOVER){
+      break;
+    }
+    tscInfo("consumer:0x%" PRIx64 " tmq status is recover", tmq->consumerId);
+
     int32_t retryCnt = 0;
-    while (TSDB_CODE_MND_CONSUMER_NOT_READY == syncAskEp(tmq)) {
+    while (syncAskEp(tmq) != 0) {
       if (retryCnt++ > 40) {
         return NULL;
       }

From 9ce28483a001160aae2d7642385c182520549d28 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 23 Nov 2023 21:22:41 +0800
Subject: [PATCH 34/43] fix: add check in mnode/vnode when alter table col/tag

---
 source/dnode/vnode/src/meta/metaTable.c | 33 ++++++++++++++++++-------
 tests/script/api/alterTableTest.c       |  8 +++---
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index f600925a0b..688adbf64d 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1233,6 +1233,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
   if (pAlterTbReq->colName == NULL) {
     terrno = TSDB_CODE_INVALID_MSG;
+    metaError("meta/table: null pAlterTbReq->colName");
     return -1;
   }
 
@@ -1300,20 +1301,26 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid};
   oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols;
 
-  int32_t iCol = 0;
+  int32_t rowLen = -1;
+  if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ||
+      pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) {
+    rowLen = 0;
+  }
+
+  int32_t  iCol = 0;
+  SSchema *qColumn = NULL;
   for (;;) {
-    pColumn = NULL;
+    qColumn = NULL;
 
     if (iCol >= pSchema->nCols) break;
-    pColumn = &pSchema->pSchema[iCol];
+    qColumn = &pSchema->pSchema[iCol];
 
-    if (NULL == pAlterTbReq->colName) {
-      metaError("meta/table: null pAlterTbReq->colName");
-      return -1;
+    if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) {
+      pColumn = qColumn;
+      if (rowLen < 0) break;
     }
-
-    if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
-    iCol++;
+    rowLen += qColumn->bytes;
+    ++iCol;
   }
 
   entry.version = version;
@@ -1328,6 +1335,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
       if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) {
         goto _err;
       }
+      if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) {
+        terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH;
+        goto _err;
+      }
       pSchema->version++;
       pSchema->nCols++;
       pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols);
@@ -1373,6 +1384,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
         goto _err;
       }
+      if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) {
+        terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH;
+        goto _err;
+      }
       if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) {
         terrno = TSDB_CODE_VND_COL_SUBSCRIBED;
         goto _err;
diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c
index e1cf3f1b9a..7d7df340e8 100644
--- a/tests/script/api/alterTableTest.c
+++ b/tests/script/api/alterTableTest.c
@@ -246,8 +246,8 @@ static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr)
       sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455);
       queryDB(taos, qstr, true);
     }
-    sprintf(qstr, "desc %s_%d;", NTB, i);
-    queryDB(taos, qstr, false);
+    // sprintf(qstr, "desc %s_%d;", NTB, i);
+    // queryDB(taos, qstr, false);
   }
 
   // check
@@ -308,10 +308,10 @@ int main(int argc, char *argv[]) {
     case CHECK_ALTER_STABLE_MODIFY_TAG:  // reproduced in 3.0.7.1
       checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG);
       break;
-    case CHECK_ALTER_NTABLE_ADD_COL:  // not reproduced in 3.0.7.1
+    case CHECK_ALTER_NTABLE_ADD_COL:  // reproduced in 3.0.7.1
       checkAlterNtbAddColumn(taos, argv[1], qstr);
       break;
-    case CHECK_ALTER_NTABLE_MODIFY_COL:  // not reproduced in 3.0.7.1
+    case CHECK_ALTER_NTABLE_MODIFY_COL:  // not reproduced in 3.0.7.1(should reproduced)
       checkAlterNtbModifyColumn(taos, argv[1], qstr);
       break;
     default:

From dc977ca2007698dfcbb31165dd67ac542737f1d2 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 23 Nov 2023 23:09:49 +0800
Subject: [PATCH 35/43] fix: stash drop column

---
 source/dnode/vnode/src/meta/metaTable.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 688adbf64d..eaaa9882ac 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1307,7 +1307,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
     rowLen = 0;
   }
 
-  int32_t  iCol = 0;
+  int32_t  iCol = 0, jCol = 0;
   SSchema *qColumn = NULL;
   for (;;) {
     qColumn = NULL;
@@ -1317,6 +1317,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
     if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) {
       pColumn = qColumn;
+      jCol = iCol;
       if (rowLen < 0) break;
     }
     rowLen += qColumn->bytes;
@@ -1367,7 +1368,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         goto _err;
       }
       pSchema->version++;
-      tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
+      tlen = (pSchema->nCols - jCol - 1) * sizeof(SSchema);
       if (tlen) {
         memmove(pColumn, pColumn + 1, tlen);
       }

From c4f1babdcf4a7d46951f45fe312812112770cf4f Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 24 Nov 2023 06:42:30 +0800
Subject: [PATCH 36/43] enh: naming optimize

---
 source/dnode/vnode/src/meta/metaTable.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index eaaa9882ac..6967d79d12 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1312,16 +1312,16 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   for (;;) {
     qColumn = NULL;
 
-    if (iCol >= pSchema->nCols) break;
-    qColumn = &pSchema->pSchema[iCol];
+    if (jCol >= pSchema->nCols) break;
+    qColumn = &pSchema->pSchema[jCol];
 
     if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) {
       pColumn = qColumn;
-      jCol = iCol;
+      iCol = jCol;
       if (rowLen < 0) break;
     }
     rowLen += qColumn->bytes;
-    ++iCol;
+    ++jCol;
   }
 
   entry.version = version;
@@ -1368,7 +1368,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         goto _err;
       }
       pSchema->version++;
-      tlen = (pSchema->nCols - jCol - 1) * sizeof(SSchema);
+      tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
       if (tlen) {
         memmove(pColumn, pColumn + 1, tlen);
       }

From ac47d3ea7df72e61d1a4976c1f3f020d119adf46 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 24 Nov 2023 06:51:57 +0800
Subject: [PATCH 37/43] enh: more check for column bytes

---
 source/dnode/vnode/src/meta/metaTable.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 6967d79d12..a4e665e225 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1381,7 +1381,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         terrno = TSDB_CODE_VND_COL_NOT_EXISTS;
         goto _err;
       }
-      if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes > pAlterTbReq->colModBytes) {
+      if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) {
         terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
         goto _err;
       }

From b1841ee97b57d83bc298391ac9c731c70b6c1c7e Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 24 Nov 2023 09:06:43 +0800
Subject: [PATCH 38/43] tsdb/reader-writer: fix pgno iteration with cache
 setting

---
 source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index abcba75b53..e6f419362c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -341,7 +341,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
     memcpy(pBuf + n, pFD->pBuf + bOffset, nRead);
 
     n += nRead;
-    pgno++;
+    ++pgno;
     bOffset = 0;
   }
 
@@ -367,6 +367,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
       tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
 
       if (szHint > 0 && n >= size) {
+        ++pgno;
         continue;
       }
       memcpy(pFD->pBuf, pBlock + i * pFD->szPage, pFD->szPage);
@@ -383,7 +384,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
       memcpy(pBuf + n, pFD->pBuf + bOffset, nRead);
 
       n += nRead;
-      pgno++;
+      ++pgno;
       bOffset = 0;
     }
 

From 6694bf86c607cd0313cbac049aed3a715e14ed95 Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Fri, 24 Nov 2023 11:14:24 +0800
Subject: [PATCH 39/43] fix test case

---
 source/libs/parser/test/parSelectTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp
index 68ded3afdd..53d97d0699 100644
--- a/source/libs/parser/test/parSelectTest.cpp
+++ b/source/libs/parser/test/parSelectTest.cpp
@@ -383,7 +383,7 @@ TEST_F(ParserSelectTest, semanticCheck) {
 
   run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE);
 
-  run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE);
+  run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_FUNC_FUNTION_PARA_NUM);
 
   // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION
   run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION);

From ff894fd70ba2574119f1d2648008f136dc4e14bd Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 24 Nov 2023 16:32:13 +0800
Subject: [PATCH 40/43] fix: sim error message issue

---
 utils/tsim/src/simExe.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c
index ac17fad36b..394b168b08 100644
--- a/utils/tsim/src/simExe.c
+++ b/utils/tsim/src/simExe.c
@@ -693,8 +693,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
       ret = 0;
       break;
     } else if (ret != 0) {
-      simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
-               tstrerror(ret), taos_errstr(pSql));
+      simDebug("script:%s, taos:%p, %s failed, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
+               tstrerror(ret));
 
       if (line->errorJump == SQL_JUMP_TRUE) {
         script->linePos = line->jump;

From ff0828fcc2c0f04563a5a8d2f33ea9d7a7d98875 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 24 Nov 2023 18:30:58 +0800
Subject: [PATCH 41/43] fix:add O_CLOSEXEC for .running & process snode when
 drop stream

---
 include/os/osFile.h                      |  1 +
 source/dnode/mgmt/node_util/src/dmFile.c |  2 +-
 source/dnode/mnode/impl/src/mndStream.c  | 13 ++++++++++---
 source/os/src/osFile.c                   |  1 +
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/include/os/osFile.h b/include/os/osFile.h
index e409936468..e9d685ed69 100644
--- a/include/os/osFile.h
+++ b/include/os/osFile.h
@@ -64,6 +64,7 @@ typedef struct TdFile *TdFilePtr;
 #define TD_FILE_EXCL          0x0080
 #define TD_FILE_STREAM        0x0100  // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
 #define TD_FILE_WRITE_THROUGH 0x0200
+#define TD_FILE_CLOEXEC       0x0400
 TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions);
 TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions);
 
diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c
index 03c6734e0c..5cbeeebc14 100644
--- a/source/dnode/mgmt/node_util/src/dmFile.c
+++ b/source/dnode/mgmt/node_util/src/dmFile.c
@@ -149,7 +149,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to open file:%s since %s", filepath, terrstr());
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index db013017e3..235703428f 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -18,6 +18,7 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndSnode.h"
 #include "mndPrivilege.h"
 #include "mndScheduler.h"
 #include "mndShow.h"
@@ -697,9 +698,15 @@ static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
-  SEpSet  epset = mndGetVgroupEpset(pMnode, pVgObj);
-  mndReleaseVgroup(pMnode, pVgObj);
+  SEpSet  epset = {0};
+  if(pTask->info.nodeId == SNODE_HANDLE){
+    SSnodeObj* pObj = mndAcquireSnode(pMnode, pTask->info.nodeId);
+    addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port);
+  }else{
+    SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
+    epset = mndGetVgroupEpset(pMnode, pVgObj);
+    mndReleaseVgroup(pMnode, pVgObj);
+  }
 
   // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
   initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index cf3bce1ad4..229387a1b6 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -647,6 +647,7 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
   access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0;
   access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
   access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
+  access |= (tdFileOptions & TD_FILE_CLOEXEC) ? O_CLOEXEC : 0;
   int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
   return fd;
 }

From 0b27cb487b75912d34d1334a6e4504a1991f50eb Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 25 Nov 2023 00:31:07 +0800
Subject: [PATCH 42/43] test: adjust test case for alter table

---
 tests/script/api/alterTableTest.c          | 324 ---------------------
 tests/script/api/makefile                  |   2 -
 tests/system-test/1-insert/alter_stable.py | 120 ++++++++
 3 files changed, 120 insertions(+), 326 deletions(-)
 delete mode 100644 tests/script/api/alterTableTest.c

diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c
deleted file mode 100644
index 7d7df340e8..0000000000
--- a/tests/script/api/alterTableTest.c
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-// TAOS standard API example. The same syntax as MySQL, but only a subset
-// to compile: gcc -o demo demo.c -ltaos
-
-/**
- *  alterTableTest.c
- *   - for JIRA: PI-23
- *   - Run the test case in clear TDengine environment with default root passwd 'taosdata'
- *
- * Usage Example: check add column for stable
- *  step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables.
- *  step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously.
- *
- *  Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..."
- */
-
-#include <inttypes.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "taos.h"  // TAOS header file
-
-typedef enum {
-  CHECK_ALTER_STABLE_ADD_COL = 0,
-  CHECK_ALTER_STABLE_ADD_TAG = 1,
-  CHECK_ALTER_STABLE_MODIFY_COL = 2,
-  CHECK_ALTER_STABLE_MODIFY_TAG = 3,
-  CHECK_ALTER_NTABLE_ADD_COL = 4,
-  CHECK_ALTER_NTABLE_MODIFY_COL = 5,
-} ENUM_CHECK_ALTER_TYPE;
-
-#define nDup           3
-#define USER_LEN       24
-#define BUF_LEN        1024
-#define DB             "d0"
-#define DB_BUFFER      32
-#define STB            "stb"
-#define NTB            "ntb"
-#define CTB            "ctb"
-#define COL            "c"
-#define STB_NUM        10
-#define NTB_NUM        20
-#define CTB_NUM        1
-#define COL_NUM        505
-#define TAG_NUM        127
-#define STB_NUM_MODIFY 100  // for modify columns/tags(increase the number if not easy to reproduced)
-#define NTB_NUM_MODIFY 500
-#define COL_NCHAR_LEN  32
-
-int32_t isDropDb = 0;
-int32_t checkType = 0;
-
-static int32_t queryDB(TAOS *taos, char *command, bool skipError) {
-  int       i;
-  TAOS_RES *pSql = NULL;
-  int32_t   code = -1;
-
-  for (i = 0; i < nDup; ++i) {
-    if (NULL != pSql) {
-      taos_free_result(pSql);
-      pSql = NULL;
-    }
-
-    pSql = taos_query(taos, command);
-    code = taos_errno(pSql);
-    if (0 == code) {
-      break;
-    }
-  }
-
-  if (code != 0) {
-    fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql));
-    if (!skipError) {
-      taos_free_result(pSql);
-      taos_close(taos);
-      exit(EXIT_FAILURE);
-    }
-  } else {
-    fprintf(stderr, "success to run: %s\n", command);
-  }
-
-  taos_free_result(pSql);
-}
-
-static void checkAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
-  // create stb
-  if (isDropDb) {
-    for (int i = 0; i < STB_NUM; ++i) {
-      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(32));", STB, i, COL,
-              0);
-      queryDB(taos, qstr, false);
-      // create ctb
-      for (int j = 0; j < CTB_NUM; ++j) {
-        sprintf(qstr, "CREATE table %s_%d_%s_%d using %s_%d tags('%d_%d');", STB, i, CTB, j, STB, i, i, j);
-        queryDB(taos, qstr, false);
-      }
-    }
-  }
-
-  if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
-    sleep(86400);
-  }
-
-  int32_t     colNum = type == CHECK_ALTER_STABLE_ADD_COL ? COL_NUM : TAG_NUM;
-  const char *colName = type == CHECK_ALTER_STABLE_ADD_COL ? "column" : "tag";
-
-  // alter stb cols
-  for (int i = 0; i < STB_NUM; ++i) {
-    for (int c = 1; c < colNum; ++c) {
-      sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN);
-      queryDB(taos, qstr, true);
-    }
-    sprintf(qstr, "desc %s_%d;", STB, i);
-    queryDB(taos, qstr, false);
-  }
-
-  // check
-  for (int i = 0; i < STB_NUM; ++i) {
-    sprintf(qstr, "desc %s_%d;", STB, i);
-    queryDB(taos, qstr, false);
-  }
-}
-
-static void checkAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) {
-  // create stb
-  if (isDropDb) {
-    for (int i = 0; i < STB_NUM_MODIFY; ++i) {
-      sprintf(
-          qstr,
-          "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 "
-          "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 "
-          "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 "
-          "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 "
-          "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 "
-          "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 "
-          "NCHAR(160),c_35 NCHAR(160)) tags(t_0 NCHAR(80), t_1 NCHAR(80), t_2 NCHAR(80), t_3 NCHAR(80),t_4 "
-          "NCHAR(80),t_5 NCHAR(80),t_6 NCHAR(80),t_7 NCHAR(80),t_8 NCHAR(80),t_9 NCHAR(80),t_10 NCHAR(80),t_11 "
-          "NCHAR(80),t_12 NCHAR(80),t_13 NCHAR(80),t_14 NCHAR(80),t_15 NCHAR(80),t_16 NCHAR(80),t_17 NCHAR(80),t_18 "
-          "NCHAR(80),t_19 NCHAR(80),t_20 NCHAR(80),t_21 NCHAR(80),t_22 NCHAR(80),t_23 NCHAR(80),t_24 NCHAR(80),t_25 "
-          "NCHAR(80),t_26 NCHAR(80),t_27 NCHAR(80),t_28 NCHAR(80),t_29 NCHAR(80),t_30 NCHAR(80),t_31 NCHAR(80),t_32 "
-          "NCHAR(80),t_33 NCHAR(80),t_34 NCHAR(80),t_35 NCHAR(80));",
-          STB, i);
-      queryDB(taos, qstr, false);
-    }
-  }
-
-  if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
-    sleep(86400);
-  }
-
-  int32_t     colLen = type == CHECK_ALTER_STABLE_MODIFY_COL ? 455 : 115;
-  const char *colName = type == CHECK_ALTER_STABLE_MODIFY_COL ? "column c_" : "tag t_";
-
-  // alter stb cols
-  for (int i = 0; i < STB_NUM_MODIFY; ++i) {
-    for (int c = 0; c < 36; ++c) {
-      sprintf(qstr, "alter table %s_%d modify %s%d NCHAR(%d);", STB, i, colName, c, colLen);
-      queryDB(taos, qstr, true);
-      //   usleep(1000);
-    }
-    sprintf(qstr, "desc %s_%d;", STB, i);
-    queryDB(taos, qstr, false);
-  }
-
-  // check
-  for (int i = 0; i < STB_NUM_MODIFY; ++i) {
-    sprintf(qstr, "desc %s_%d;", STB, i);
-    queryDB(taos, qstr, false);
-  }
-}
-
-static void checkAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) {
-  // create ntb
-  if (isDropDb) {
-    for (int i = 0; i < NTB_NUM; ++i) {
-      sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32));", NTB, i, COL, 0);
-      queryDB(taos, qstr, false);
-    }
-  }
-
-  if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
-    sleep(86400);
-  }
-
-  // alter ntb cols
-  for (int i = 0; i < NTB_NUM; ++i) {
-    for (int c = 1; c < COL_NUM; ++c) {
-      sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", NTB, i, c, COL_NCHAR_LEN);
-      queryDB(taos, qstr, true);
-    }
-    sprintf(qstr, "desc %s_%d;", NTB, i);
-    queryDB(taos, qstr, false);
-  }
-
-  // check
-  for (int i = 0; i < NTB_NUM; ++i) {
-    sprintf(qstr, "desc %s_%d;", NTB, i);
-    queryDB(taos, qstr, false);
-  }
-}
-
-static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) {
-  // create ntb
-  if (isDropDb) {
-    for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
-      sprintf(
-          qstr,
-          "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 "
-          "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 "
-          "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 "
-          "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 "
-          "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 "
-          "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 "
-          "NCHAR(160),c_35 NCHAR(160));",
-          NTB, i);
-      queryDB(taos, qstr, false);
-    }
-  }
-
-  if (isDropDb) {
-    printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n");
-    sleep(86400);
-  }
-
-  // alter ntb cols
-  for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
-    for (int c = 0; c < 36; ++c) {
-      sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455);
-      queryDB(taos, qstr, true);
-    }
-    // sprintf(qstr, "desc %s_%d;", NTB, i);
-    // queryDB(taos, qstr, false);
-  }
-
-  // check
-  for (int i = 0; i < NTB_NUM_MODIFY; ++i) {
-    sprintf(qstr, "desc %s_%d;", NTB, i);
-    queryDB(taos, qstr, false);
-  }
-}
-
-int main(int argc, char *argv[]) {
-  char qstr[1024];
-
-  // connect to server
-  if (argc < 2) {
-    printf("please input server-ip \n");  // e.g. localhost
-    return 0;
-  }
-
-  if (argc < 3) {
-    printf("please specify if drop DB to clear env\n");  // 0 not drop, 1 drop
-    return 0;
-  }
-
-  isDropDb = atoi(argv[2]);
-
-  if (argc < 4) {
-    printf("please specify check type\n");  // enum of ENUM_CHECK_ALTER_TYPE
-    return 0;
-  }
-
-  checkType = atoi(argv[3]);
-
-  TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
-  if (taos == NULL) {
-    printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/);
-    exit(1);
-  }
-
-  if (isDropDb) {
-    sprintf(qstr, "drop database if exists %s", DB);
-    queryDB(taos, qstr, false);
-    sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER);
-    queryDB(taos, qstr, false);
-  }
-  sprintf(qstr, "use %s", DB);
-  queryDB(taos, qstr, false);
-
-  switch (checkType) {
-    case CHECK_ALTER_STABLE_ADD_COL:  // reproduced in 3.0.7.1
-      checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL);
-      break;
-    case CHECK_ALTER_STABLE_ADD_TAG:  // reproduced in 3.0.7.1
-      checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG);
-      break;
-    case CHECK_ALTER_STABLE_MODIFY_COL:  // not reproduced in 3.0.7.1 since already checked in mnode
-      checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL);
-      break;
-    case CHECK_ALTER_STABLE_MODIFY_TAG:  // reproduced in 3.0.7.1
-      checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG);
-      break;
-    case CHECK_ALTER_NTABLE_ADD_COL:  // reproduced in 3.0.7.1
-      checkAlterNtbAddColumn(taos, argv[1], qstr);
-      break;
-    case CHECK_ALTER_NTABLE_MODIFY_COL:  // not reproduced in 3.0.7.1(should reproduced)
-      checkAlterNtbModifyColumn(taos, argv[1], qstr);
-      break;
-    default:
-      printf("unkown check type:%d\n", checkType);
-      break;
-  }
-
-  taos_close(taos);
-  taos_cleanup();
-}
\ No newline at end of file
diff --git a/tests/script/api/makefile b/tests/script/api/makefile
index 2bc127d995..d848dca194 100644
--- a/tests/script/api/makefile
+++ b/tests/script/api/makefile
@@ -16,7 +16,6 @@ exe:
 	gcc $(CFLAGS) ./dbTableRoute.c  -o $(ROOT)dbTableRoute $(LFLAGS)
 	gcc $(CFLAGS) ./insertSameTs.c  -o $(ROOT)insertSameTs $(LFLAGS)
 	gcc $(CFLAGS) ./passwdTest.c  -o $(ROOT)passwdTest $(LFLAGS)
-	gcc $(CFLAGS) ./alterTableTest.c  -o $(ROOT)alterTableTest $(LFLAGS)
 	gcc $(CFLAGS) ./whiteListTest.c  -o $(ROOT)whiteListTest $(LFLAGS)
 	gcc $(CFLAGS) ./insert_stb.c  -o $(ROOT)insert_stb $(LFLAGS)
 	gcc $(CFLAGS) ./tmqViewTest.c  -o $(ROOT)tmqViewTest $(LFLAGS)
@@ -28,7 +27,6 @@ clean:
 	rm $(ROOT)dbTableRoute
 	rm $(ROOT)insertSameTs
 	rm $(ROOT)passwdTest
-	rm $(ROOT)alterTableTest
 	rm $(ROOT)whiteListTest
 	rm $(ROOT)insert_stb
 	rm $(ROOT)tmqViewTest
diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py
index 52f185a868..201610abf0 100644
--- a/tests/system-test/1-insert/alter_stable.py
+++ b/tests/system-test/1-insert/alter_stable.py
@@ -13,6 +13,7 @@
 
 import random
 import string
+import threading
 from util.log import *
 from util.cases import *
 from util.sql import *
@@ -25,10 +26,24 @@ class TDTestCase:
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
         self.setsql = TDSetSql()
+        self.fname = __file__ + '.tmp.sql'
+        self.dbname = 'db1'
         self.ntbname = 'ntb'
         self.stbname = 'stb'
+        self.stbnum = 10
+        self.ntbnum = 10
+        self.colnum = 52
+        self.tagnum = 15
+        self.collen = 320
+        self.colnum_modify = 40
+        self.tagnum_modify = 40
+        self.collen_old_modify = 160
+        self.collen_new_modify = 455
+        self.taglen_old_modify = 80
+        self.taglen_new_modify = 155
         self.binary_length = 20 # the length of binary for column_dict
         self.nchar_length = 20  # the length of nchar for column_dict
+        self.threadnum = 2
         self.column_dict = {
             'ts'  : 'timestamp',
             'col1': 'tinyint',
@@ -183,9 +198,114 @@ class TDTestCase:
         tdLog.info(res)
         assert(res[1][2] == 39001)
 
+    def prepareAlterEnv(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+        tdSql.execute(f'create database if not exists {self.dbname} vgroups 2')
+        tdSql.execute(f'use {self.dbname}')
+
+    def destroyAlterEnv(self):
+        tdSql.execute(f'drop database if exists {self.dbname}')
+
+    def alterTableTask(self, i):
+        os.system(f'taos -f {self.fname}.{i};')
+
+    def alterTableCheck(self, opt):
+        if opt in ["stb_add_col", "stb_add_tag"]:
+            for i in range(self.stbnum):
+                tdSql.execute(f'desc {self.stbname}_{i}')
+        elif opt in ["stb_modify_col", "stb_modify_tag"]:
+            for i in range(self.stbnum):
+                tdSql.execute(f'desc {self.stbname}_{i}')
+        elif opt in ["ntb_add_col", "ntb_modify_col"]:
+            for i in range(self.ntbnum):
+                tdSql.execute(f'desc {self.ntbname}_{i}')
+
+    def executeAlterTableAndCheck(self, opt):
+        threads = []
+        for i in range(self.threadnum):
+            thread = threading.Thread(target=self.alterTableTask, args=(i,))
+            threads.append(thread)
+            thread.start()            
+        for i in range(self.threadnum):
+            threads[i].join()
+        self.alterTableCheck(opt)
+
+    def destroyAlterTableEnv(self):
+        for i in range(self.threadnum):
+            if os.path.isfile(f'{self.fname}.{i}'):
+                os.remove(f'{self.fname}.{i}')
+    
+    def prepareAlterTableEnv(self, opt):
+        self.destroyAlterTableEnv()
+        lines = [f'use {self.dbname};\n']
+        if opt in ["stb_add_col", "stb_add_tag"]:
+            for i in range(self.stbnum):
+                tdSql.execute(f'create table if not exists {self.stbname}_{i} (ts timestamp, c_0 NCHAR({self.collen})) tags(t0 nchar({self.collen}));')
+            for i in range(self.stbnum):
+                if opt == 'stb_add_col': 
+                    for c in range(1, self.colnum):
+                        lines.append(f'alter table {self.stbname}_{i} add column c_{c} NCHAR({self.collen});\n')
+                else:
+                    for c in range(1, self.tagnum):
+                        lines.append(f'alter table {self.stbname}_{i} add tag t_{c} NCHAR({self.collen});\n')
+        elif opt in ["stb_modify_col", "stb_modify_tag"]:
+            for i in range(self.stbnum):
+                createTbSql = f'CREATE table if not exists {self.stbname}_{i} (ts timestamp'
+                for j in range(self.colnum_modify):
+                    createTbSql += f',c_{j} NCHAR({self.collen_old_modify})'
+                createTbSql += f') tags(t_0 NCHAR({self.taglen_old_modify})'
+                for k in range(1,self.tagnum_modify):
+                    createTbSql += f',t_{k} NCHAR({self.taglen_old_modify})'
+                createTbSql += f');'
+                tdLog.info(createTbSql)
+                tdSql.execute(createTbSql)
+            for i in range(self.stbnum):
+                if opt == 'stb_modify_col': 
+                    for c in range(self.colnum_modify):
+                        lines.append(f'alter table {self.stbname}_{i} modify column c_{c} NCHAR({self.collen_new_modify});\n')
+                else:
+                    for c in range(self.tagnum_modify):
+                        lines.append(f'alter table {self.stbname}_{i} modify tag t_{c} NCHAR({self.taglen_new_modify});\n')
+        elif opt in ['ntb_add_col']:
+            for i in range(self.ntbnum):
+                tdSql.execute(f'create table if not exists {self.ntbname}_{i} (ts timestamp, c_0 NCHAR({self.collen}));')
+            for i in range(self.ntbnum):
+                for c in range(1, self.colnum):
+                    lines.append(f'alter table {self.ntbname}_{i} add column c_{c} NCHAR({self.collen});\n')
+        elif opt in ['ntb_modify_col']:
+            for i in range(self.ntbnum):
+                createTbSql = f'CREATE table if not exists {self.ntbname}_{i} (ts timestamp'
+                for j in range(self.colnum_modify):
+                    createTbSql += f',c_{j} NCHAR({self.collen_old_modify})'
+                createTbSql += f');'
+                tdLog.info(createTbSql)
+                tdSql.execute(createTbSql)
+            for i in range(self.ntbnum):
+                for c in range(self.colnum_modify):
+                    lines.append(f'alter table {self.ntbname}_{i} modify column c_{c} NCHAR({self.collen_new_modify});\n')
+        # generate sql file
+        with open(f'{self.fname}.0', "a") as f:
+            f.writelines(lines)
+        # clone sql file in case of race condition
+        for i in range(1, self.threadnum):
+            shutil.copy(f'{self.fname}.0', f'{self.fname}.{i}')
+
+    def alter_stable_multi_client_check(self):
+        """Check alter stable/ntable var type column/tag(PI-23)
+        """
+        alter_table_check_type = ["stb_add_col", "stb_add_tag", "stb_modify_col", "stb_modify_tag", "ntb_add_col", "ntb_modify_col"]
+
+        for opt in alter_table_check_type:
+            self.prepareAlterEnv()
+            self.prepareAlterTableEnv(opt)
+            self.executeAlterTableAndCheck(opt)
+            self.destroyAlterTableEnv()
+        self.destroyAlterEnv()
+
     def run(self):
         self.alter_stable_check()
         self.alter_stable_column_varchar_39001()
+        self.alter_stable_multi_client_check()
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

From 9ceddd3f60467d2be931179b67962495131e098e Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 25 Nov 2023 00:49:49 +0800
Subject: [PATCH 43/43] test: adjust test case for alter table

---
 tests/system-test/1-insert/alter_stable.py | 34 +++++++++++-----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py
index 201610abf0..40d7f04ceb 100644
--- a/tests/system-test/1-insert/alter_stable.py
+++ b/tests/system-test/1-insert/alter_stable.py
@@ -209,7 +209,16 @@ class TDTestCase:
     def alterTableTask(self, i):
         os.system(f'taos -f {self.fname}.{i};')
 
-    def alterTableCheck(self, opt):
+    def executeAlterTable(self, opt):
+        threads = []
+        for i in range(self.threadnum):
+            thread = threading.Thread(target=self.alterTableTask, args=(i,))
+            threads.append(thread)
+            thread.start()
+        for i in range(self.threadnum):
+            threads[i].join()
+
+    def checkAlterTable(self, opt):
         if opt in ["stb_add_col", "stb_add_tag"]:
             for i in range(self.stbnum):
                 tdSql.execute(f'desc {self.stbname}_{i}')
@@ -220,23 +229,13 @@ class TDTestCase:
             for i in range(self.ntbnum):
                 tdSql.execute(f'desc {self.ntbname}_{i}')
 
-    def executeAlterTableAndCheck(self, opt):
-        threads = []
-        for i in range(self.threadnum):
-            thread = threading.Thread(target=self.alterTableTask, args=(i,))
-            threads.append(thread)
-            thread.start()            
-        for i in range(self.threadnum):
-            threads[i].join()
-        self.alterTableCheck(opt)
-
-    def destroyAlterTableEnv(self):
+    def destroyAlterTable(self):
         for i in range(self.threadnum):
             if os.path.isfile(f'{self.fname}.{i}'):
                 os.remove(f'{self.fname}.{i}')
     
-    def prepareAlterTableEnv(self, opt):
-        self.destroyAlterTableEnv()
+    def prepareAlterTable(self, opt):
+        self.destroyAlterTable()
         lines = [f'use {self.dbname};\n']
         if opt in ["stb_add_col", "stb_add_tag"]:
             for i in range(self.stbnum):
@@ -297,9 +296,10 @@ class TDTestCase:
 
         for opt in alter_table_check_type:
             self.prepareAlterEnv()
-            self.prepareAlterTableEnv(opt)
-            self.executeAlterTableAndCheck(opt)
-            self.destroyAlterTableEnv()
+            self.prepareAlterTable(opt)
+            self.executeAlterTable(opt)
+            self.checkAlterTable(opt)
+            self.destroyAlterTable()
         self.destroyAlterEnv()
 
     def run(self):