From 4bc0d33db31401cab51317d4962b3df2870bab01 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Fri, 30 Sep 2022 21:14:09 -0700
Subject: [PATCH 01/44] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fe9bb49ed8..08ae84dcb9 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
 
-English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengine.com/tsdb/)
+English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/tsdb/)
 
 # What is TDengine？
 
@@ -33,7 +33,7 @@ TDengine is an open source, high-performance, cloud native [time-series database
 
 - **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
 
-For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/)
+For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/). The easiest way to experience TDengine is through [TDengine Cloud](https://cloud.tdengine.com). 
 
 # Documentation
 

From 64e453c4a6924b93fe3efc4a6f84ed067ac1a802 Mon Sep 17 00:00:00 2001
From: jiacy-jcy <714897623@qq.com>
Date: Mon, 10 Oct 2022 10:01:50 +0800
Subject: [PATCH 02/44] test:add test case for show into ci

---
 tests/system-test/0-others/show.py | 60 ++++++++++++++++++++++++++++++
 tests/system-test/fulltest.sh      |  2 +-
 2 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 tests/system-test/0-others/show.py

diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py
new file mode 100644
index 0000000000..673e795297
--- /dev/null
+++ b/tests/system-test/0-others/show.py
@@ -0,0 +1,60 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+import subprocess
+from util.common import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.dbname = 'db'
+        self.ins_param_list = ['dnodes','mnodes','qnodes','cluster','functions','users','grants','topics','subscriptions','streams']
+        self.perf_param = ['apps','connections','consumers','queries','transactions']
+        self.perf_param_list = ['apps','connections','consumers','queries','trans']
+
+    def ins_check(self):
+        for param in self.ins_param_list:
+            tdSql.query(f'show {param}')
+            show_result = tdSql.queryResult
+            tdSql.query(f'select * from information_schema.ins_{param}')
+            select_result = tdSql.queryResult
+            tdSql.checkEqual(show_result,select_result)
+
+    def perf_check(self):
+        for param in range(len(self.perf_param_list)):
+            tdSql.query(f'show {self.perf_param[param]}')
+            if len(tdSql.queryResult) != 0:
+                show_result = tdSql.queryResult[0][0]
+                tdSql.query(f'select * from performance_schema.perf_{self.perf_param_list[param]}')
+                select_result = tdSql.queryResult[0][0]
+                tdSql.checkEqual(show_result,select_result)
+            else :
+                continue
+    def run(self):
+        tdSql.prepare()
+        self.ins_check()
+        self.perf_check()
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index d4ac3e2844..1c6aef6286 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -37,7 +37,7 @@ python3 ./test.py -f 1-insert/table_param_ttl.py -R
 python3 ./test.py -f 1-insert/update_data_muti_rows.py
 python3 ./test.py -f 1-insert/db_tb_name_check.py
 python3 ./test.py -f 1-insert/database_pre_suf.py
-
+python3 ./test.py -f 0-others/show.py
 python3 ./test.py -f 2-query/abs.py
 python3 ./test.py -f 2-query/abs.py -R
 python3 ./test.py -f 2-query/and_or_for_byte.py

From fe4c7dee7c4dd35ad1c192e89c5366a7eb2ac289 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 10 Oct 2022 14:45:49 +0800
Subject: [PATCH 03/44] fix: add logs

---
 source/dnode/mnode/impl/src/mndMain.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index b3b9927a34..275fc76e36 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -814,9 +814,11 @@ int32_t mndAcquireRpcRef(SMnode *pMnode) {
   int32_t code = 0;
   taosThreadRwlockRdlock(&pMnode->lock);
   if (pMnode->stopped) {
+    mTrace("mnode not running");
     terrno = TSDB_CODE_APP_NOT_READY;
     code = -1;
   } else if (!mndIsMaster(pMnode)) {
+    mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
     code = -1;
   } else {
     int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);

From 68842b9c641507e71e793bcf62e27e2ae818ae50 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 10 Oct 2022 14:55:58 +0800
Subject: [PATCH 04/44] refactor: do some internal refactor.

---
 include/libs/function/function.h            |  2 +-
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 47 +++++++++++++++------
 source/libs/function/src/builtins.c         |  2 +-
 3 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 60c7b18367..1796956f96 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -54,7 +54,7 @@ typedef struct SFuncExecFuncs {
   FExecCombine  combine;
 } SFuncExecFuncs;
 
-#define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
+#define MAX_INTERVAL_TIME_WINDOW 10000000  // maximum allowed time windows in final results
 
 #define TOP_BOTTOM_QUERY_LIMIT    100
 #define FUNCTIONS_NAME_MAX_LENGTH 16
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index bf94ae68ff..4a6f677787 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -265,24 +265,43 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
     // only apply to the child tables, ordinary tables will not incur this filter procedure.
     size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
-    SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk));
-    for (int32_t i = 0; i < size; ++i) {
-      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
-      uint64_t s = p->suid;
-      if (s < suid) {
-        continue;
-      }
 
-      if (s == suid) {
-        taosArrayPush(pTmp, p);
-      } else if (s > suid) {
-        break;
+    if (size > 1) {
+      SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
+      SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);
+
+      // all identical
+      if (pStart->suid == pEnd->suid) {
+        if (pStart->suid == suid) {
+          // do nothing
+        } else if (pStart->suid != suid) {
+          // no qualified stt block existed
+          (*pIter)->iSttBlk = -1;
+          double el = (taosGetTimestampUs() - st)/1000.0;
+          tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
+          return code;
+        }
+      } else {
+        SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk));
+        for (int32_t i = 0; i < size; ++i) {
+          SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
+          uint64_t s = p->suid;
+          if (s < suid) {
+            continue;
+          }
+
+          if (s == suid) {
+            taosArrayPush(pTmp, p);
+          } else if (s > suid) {
+            break;
+          }
+        }
+
+        taosArrayDestroy(pBlockLoadInfo->aSttBlk);
+        pBlockLoadInfo->aSttBlk = pTmp;
       }
     }
 
-    taosArrayDestroy(pBlockLoadInfo->aSttBlk);
-    pBlockLoadInfo->aSttBlk = pTmp;
-
     double el = (taosGetTimestampUs() - st)/1000.0;
     tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
   }
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index e9415dc879..2308aaf214 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -662,7 +662,7 @@ static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  if (pValue->datum.i < 1 || pValue->datum.i > 100) {
+  if (pValue->datum.i < 1 || pValue->datum.i > TOP_BOTTOM_QUERY_LIMIT) {
     return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
   }
 

From 9a410b96efd9d216a666c24ef89236b5d25aeac1 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 10 Oct 2022 15:04:33 +0800
Subject: [PATCH 05/44] fix(query): new META_READER_NOLOCK flag for
 metaReaderInit

---
 source/dnode/vnode/inc/vnode.h          | 2 ++
 source/dnode/vnode/src/meta/metaQuery.c | 6 ++++--
 source/libs/executor/src/scanoperator.c | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index ba16bad7cf..9926c5a8c4 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -91,6 +91,8 @@ typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;
 typedef struct SMetaEntry  SMetaEntry;
 
+#define META_READER_NOLOCK 0x1
+
 void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
 void        metaReaderClear(SMetaReader *pReader);
 int32_t     metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index d6c8b12f87..1b5d25974e 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -19,11 +19,13 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
   memset(pReader, 0, sizeof(*pReader));
   pReader->flags = flags;
   pReader->pMeta = pMeta;
-  metaRLock(pMeta);
+  if (!(flags & META_READER_NOLOCK)) {
+    metaRLock(pMeta);
+  }
 }
 
 void metaReaderClear(SMetaReader *pReader) {
-  if (pReader->pMeta) {
+  if (pReader->pMeta && !(pReader->flags & META_READER_NOLOCK)) {
     metaULock(pReader->pMeta);
   }
   tDecoderClear(&pReader->coder);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b055ba1ce5..76f45a1cef 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2757,7 +2757,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
         colDataAppend(pColInfoData, numOfRows, (char*)&ts, false);
 
         SMetaReader mr = {0};
-        metaReaderInit(&mr, pInfo->readHandle.meta, 0);
+        metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK);
 
         uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
         int32_t  code = metaGetTableEntryByUid(&mr, suid);

From 8686e7d2e02c9f613f152f0306e1ee3878589a88 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 10 Oct 2022 16:05:56 +0800
Subject: [PATCH 06/44] chore: update taos-tools 8dea29a for 3.0 (#17249)

* chore: update taos-tools 8dea29a for 3.0

* chore: update taos-tools 85b582b for 3.0
---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 packaging/tools/makepkg.sh        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 2d90d8c49f..cb9502b2b2 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 85179e9
+        GIT_TAG 85b582b
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index fb0ef4f9a3..60d7440487 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -47,7 +47,7 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
   cd ${top_dir}/tools/taos-tools/packaging/deb
   [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
-  taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}')
+  taostools_ver=$(git tag |grep -v taos | sort | tail -1)
   taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}"
 
   cd ${curr_dir}

From 5ccc7aa98e06fd498e7f8df153c8041c4f991dc3 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Mon, 10 Oct 2022 16:16:54 +0800
Subject: [PATCH 07/44] fix: fix no stop error when interval fill order by desc

---
 source/libs/executor/src/tfill.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index cf0bc078d7..74d5401a3b 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -562,10 +562,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma
   int32_t  numOfRows = taosNumOfRemainRows(pFillInfo);
 
   TSKEY ekey1 = ekey;
-  if (!FILL_IS_ASC_FILL(pFillInfo)) {
-    pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->interval.precision);
-  }
-
+  
   int64_t numOfRes = -1;
   if (numOfRows > 0) {  // still fill gap within current data block, not generating data after the result set.
     TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];

From 9324ad288e466149e09ce6f80dbafd892fa81678 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Mon, 10 Oct 2022 16:53:17 +0800
Subject: [PATCH 08/44] fix: fix _wend computation error

---
 source/libs/executor/src/tfill.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 74d5401a3b..85c1d63868 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -105,9 +105,8 @@ static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, S
     } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_END) {
       // TODO: include endpoint
       SInterval* pInterval = &pFillInfo->interval;
-      int32_t    step = (pFillInfo->order == TSDB_ORDER_ASC) ? 1 : -1;
       int64_t    windowEnd =
-          taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision);
+          taosTimeAdd(pFillInfo->currentKey, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
       colDataAppend(pDstColInfoData, rowIndex, (const char*)&windowEnd, false);
       return true;
     } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_DURATION) {

From 2f9048b2ae3f92c5963a01c06c99206a02ca157a Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Mon, 10 Oct 2022 17:02:48 +0800
Subject: [PATCH 09/44] feat(shell): lastWord function less add one

---
 tools/shell/src/shellAuto.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c
index 5db9b47a5f..506897cf5b 100644
--- a/tools/shell/src/shellAuto.c
+++ b/tools/shell/src/shellAuto.c
@@ -1268,7 +1268,7 @@ char* lastWord(char* p) {
   char* p2 = strrchr(p, ',');
 
   if (p1 && p2) {
-    return p1 > p2 ? p1 : p2 + 1;
+    return p1 > p2 ? p1 + 1 : p2 + 1;
   } else if (p1) {
     return p1 + 1;
   } else if (p2) {

From 021a9d216e3cc7b81a7901c6886c4c45ad00b6f3 Mon Sep 17 00:00:00 2001
From: Zhiqiang Wang <1296468573@qq.com>
Date: Mon, 10 Oct 2022 17:42:08 +0800
Subject: [PATCH 10/44] fix: link libtaosws error (#17266)

---
 packaging/tools/make_install.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh
index a71614ad6e..b8796ddf09 100755
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@@ -317,10 +317,6 @@ function install_lib() {
             ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
 
         ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
-
-        if [ -d "${lib64_link_dir}" ]; then
-            ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
-        fi
     fi
   else
     ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \

From 579af54c492bfc9124a415dc8d3e2f06de635eb7 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 10 Oct 2022 17:45:10 +0800
Subject: [PATCH 11/44] more code

---
 include/util/taoserror.h | 4 ++++
 source/util/src/terror.c | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index f6940b2895..add8f33cc2 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -632,6 +632,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TMQ_CONSUMER_MISMATCH          TAOS_DEF_ERROR_CODE(0, 0x4001)
 #define TSDB_CODE_TMQ_CONSUMER_CLOSED            TAOS_DEF_ERROR_CODE(0, 0x4002)
 
+// TDLite
+#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x4100)
+#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR           TAOS_DEF_ERROR_CODE(0, 0x4101)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 1906a77127..eb13a08be4 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -633,6 +633,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG,                "Invalid message")
 TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH,          "Consumer mismatch")
 TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED,            "Consumer closed")
 
+// TDLite
+TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS,         "Invalid TDLite open flags")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR,           "Invalid TDLite open directory")
+
 #ifdef TAOS_ERROR_C
 };
 #endif

From 4eecd7fd4a250262ce1d696165c1749b8117fc5f Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 10 Oct 2022 18:17:58 +0800
Subject: [PATCH 12/44] more code

---
 source/dnode/vnode/src/vnd/vnodeOpen.c | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 616aa39bdf..ddd0f36194 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -28,13 +28,24 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
   }
 
   // create vnode env
-  if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) {
-    vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno));
-    return -1;
+  if (pTfs) {
+    if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) {
+      vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno));
+      return -1;
+    }
+    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
+  } else {
+    if (taosMkDir(path)) {
+      return TAOS_SYSTEM_ERROR(errno);
+    }
+    strcpy(dir, path);
   }
 
-  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
-  info.config = *pCfg;
+  if (pCfg) {
+    info.config = *pCfg;
+  } else {
+    info.config = vnodeCfgDefault;
+  }
   info.state.committed = -1;
   info.state.applied = -1;
   info.state.commitID = 0;
@@ -44,7 +55,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
     return -1;
   }
 
-  vInfo("vgId:%d, vnode is created", pCfg->vgId);
+  vInfo("vgId:%d, vnode is created", info.config.vgId);
 
   return 0;
 }

From fe115ab02e4b41a98e6e7b0686d8277b8ee7050d Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Mon, 10 Oct 2022 18:20:53 +0800
Subject: [PATCH 13/44] fix: support statistics of insert_req

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |  9 +++++
 source/dnode/vnode/src/inc/vnodeInt.h       | 10 ++++++
 source/dnode/vnode/src/vnd/vnodeCommit.c    | 39 +++++++++++++++++++++
 source/dnode/vnode/src/vnd/vnodeOpen.c      |  4 +++
 source/dnode/vnode/src/vnd/vnodeQuery.c     |  8 ++---
 source/dnode/vnode/src/vnd/vnodeSvr.c       |  9 +++--
 6 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 4047bc2340..5451fe33f1 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -78,6 +78,15 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
   pMgmt->state.numOfBatchInsertReqs = numOfBatchInsertReqs;
   pMgmt->state.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
 
+  printf("%s:%d: Info: nInsert:%" PRIi64 ", nInsertSuccess:%" PRIi64 ", nBatch:%" PRIi64 ", nBatchSuccess:%" PRIi64
+         "\n",
+         __func__, __LINE__, pInfo->vstat.numOfInsertReqs, pInfo->vstat.numOfInsertSuccessReqs,
+         pInfo->vstat.numOfBatchInsertReqs, pInfo->vstat.numOfBatchInsertSuccessReqs);
+  printf("%s:%d: Mgmt: nInsert:%" PRIi64 ", nInsertSuccess:%" PRIi64 ", nBatch:%" PRIi64 ", nBatchSuccess:%" PRIi64
+         "\n",
+         __func__, __LINE__, pMgmt->state.numOfInsertReqs, pMgmt->state.numOfInsertSuccessReqs,
+         pMgmt->state.numOfBatchInsertReqs, pMgmt->state.numOfBatchInsertSuccessReqs);
+
   tfsGetMonitorInfo(pMgmt->pTfs, &pInfo->tfs);
   taosArrayDestroy(pVloads);
 }
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 2a8a74d297..d4b88abb1d 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -56,6 +56,7 @@ typedef struct SSma               SSma;
 typedef struct STsdb              STsdb;
 typedef struct STQ                STQ;
 typedef struct SVState            SVState;
+typedef struct SVStatis           SVStatis;
 typedef struct SVBufPool          SVBufPool;
 typedef struct SQWorker           SQHandle;
 typedef struct STsdbKeepCfg       STsdbKeepCfg;
@@ -284,9 +285,17 @@ struct SVState {
   int64_t commitTerm;
 };
 
+struct SVStatis {
+  int64_t nInsert;
+  int64_t nInsertSuccess;
+  int64_t nBatchInsert;
+  int64_t nBatchInsertSuccess;
+};
+
 struct SVnodeInfo {
   SVnodeCfg config;
   SVState   state;
+  SVStatis  statis;
 };
 
 typedef enum {
@@ -309,6 +318,7 @@ struct SVnode {
   char*         path;
   SVnodeCfg     config;
   SVState       state;
+  SVStatis      statis;
   STfs*         pTfs;
   SMsgCb        msgCb;
   TdThreadMutex mutex;
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 6dc3ef86a7..ae55d50456 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -227,6 +227,11 @@ int vnodeCommit(SVnode *pVnode) {
   info.state.committed = pVnode->state.applied;
   info.state.commitTerm = pVnode->state.applyTerm;
   info.state.commitID = pVnode->state.commitID;
+  info.statis.nInsert = pVnode->statis.nInsert;
+  info.statis.nInsertSuccess = pVnode->statis.nInsertSuccess;
+  info.statis.nBatchInsert = pVnode->statis.nBatchInsert;
+  info.statis.nBatchInsertSuccess = pVnode->statis.nBatchInsertSuccess;
+
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   if (vnodeSaveInfo(dir, &info) < 0) {
     ASSERT(0);
@@ -352,6 +357,32 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   return 0;
 }
 
+static int vnodeEncodeStatis(const void *pObj, SJson *pJson) {
+  const SVStatis *pStatis = (SVStatis *)pObj;
+
+  if (tjsonAddIntegerToObject(pJson, "insert", pStatis->nInsert) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "insert success", pStatis->nInsertSuccess) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "batch insert", pStatis->nBatchInsert) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "batch insert success", pStatis->nBatchInsertSuccess) < 0) return -1;
+
+  return 0;
+}
+
+static int vnodeDecodeStatis(const SJson *pJson, void *pObj) {
+  SVStatis *pStatis = (SVStatis *)pObj;
+
+  int32_t code;
+  tjsonGetNumberValue(pJson, "insert", pStatis->nInsert, code);
+  if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "insert success", pStatis->nInsertSuccess, code);
+  if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "batch insert", pStatis->nBatchInsert, code);
+  if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "batch insert success", pStatis->nBatchInsertSuccess, code);
+  if (code < 0) return -1;
+  return 0;
+}
+
 static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
   SJson *pJson;
   char  *pData;
@@ -371,6 +402,10 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
     goto _err;
   }
 
+  if (tjsonAddObject(pJson, "statis", vnodeEncodeStatis, (void *)&pInfo->statis) < 0) {
+    goto _err;
+  }
+
   pData = tjsonToString(pJson);
   if (pData == NULL) {
     goto _err;
@@ -402,6 +437,10 @@ static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
     goto _err;
   }
 
+  if (tjsonToObject(pJson, "statis", vnodeDecodeStatis, (void *)&pInfo->statis) < 0) {
+    goto _err;
+  }
+
   tjsonDelete(pJson);
 
   return 0;
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 616aa39bdf..cd87784732 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -85,6 +85,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->state.applied = info.state.committed;
   pVnode->state.commitID = info.state.commitID;
   pVnode->state.commitTerm = info.state.commitTerm;
+  pVnode->statis.nInsert = info.statis.nInsert;
+  pVnode->statis.nInsertSuccess = info.statis.nInsertSuccess;
+  pVnode->statis.nBatchInsert = info.statis.nBatchInsert;
+  pVnode->statis.nBatchInsertSuccess = info.statis.nBatchInsertSuccess;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
   taosThreadMutexInit(&pVnode->lock, NULL);
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 0d57c7bb74..ba8fbaf553 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -375,10 +375,10 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   pLoad->compStorage = (int64_t)2 * 1073741824;
   pLoad->pointsWritten = 100;
   pLoad->numOfSelectReqs = 1;
-  pLoad->numOfInsertReqs = 3;
-  pLoad->numOfInsertSuccessReqs = 2;
-  pLoad->numOfBatchInsertReqs = 5;
-  pLoad->numOfBatchInsertSuccessReqs = 4;
+  pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
+  pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
+  pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
+  pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 5c8590c7c9..bff836faf5 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -812,10 +812,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
   SSubmitReq    *pSubmitReq = (SSubmitReq *)pReq;
   SSubmitRsp     submitRsp = {0};
   SSubmitMsgIter msgIter = {0};
-  SSubmitBlk    *pBlock;
+  SSubmitBlk    *pBlock = NULL;
   SVCreateTbReq  createTbReq = {0};
   SDecoder       decoder = {0};
-  int32_t        nRows;
+  int32_t        nRows = 0;
   int32_t        tsize, ret;
   SEncoder       encoder = {0};
   SArray        *newTbUids = NULL;
@@ -823,6 +823,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 
   pRsp->code = 0;
   pSubmitReq->version = version;
+  atomic_fetch_add_64(&pVnode->statis.nBatchInsert, 1);
 
 #ifdef TD_DEBUG_PRINT_ROW
   vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__);
@@ -942,12 +943,16 @@ _exit:
 
   taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp);
 
+  atomic_fetch_add_64(&pVnode->statis.nInsert, submitRsp.numOfRows);
+  atomic_fetch_add_64(&pVnode->statis.nInsertSuccess, submitRsp.affectedRows);
+
   // TODO: the partial success scenario and the error case
   // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level
   // 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
+    atomic_fetch_add_64(&pVnode->statis.nBatchInsertSuccess, 1);
   }
 
   vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version);

From 6c097be04ee45f1e0610c699f49e522fe19160c9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 10 Oct 2022 18:29:14 +0800
Subject: [PATCH 14/44] more code

---
 source/dnode/vnode/src/vnd/vnodeOpen.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index ddd0f36194..414834e2eb 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -69,7 +69,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   char       tdir[TSDB_FILENAME_LEN * 2];
   int        ret;
 
-  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
+  if (pTfs) {
+    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
+  } else {
+    snprintf(dir, TSDB_FILENAME_LEN, "%s", path);
+  }
 
   info.config = vnodeCfgDefault;
 

From 73b5a001d6de35f640fe554f5a61d4a2836b5367 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 10 Oct 2022 19:15:52 +0800
Subject: [PATCH 15/44] fix(query): fix invalid type convert in cast function.

---
 source/libs/scalar/src/sclfunc.c | 243 ++++++++++++++++---------------
 1 file changed, 126 insertions(+), 117 deletions(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 4ead1147e4..50f6b16168 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -709,8 +709,15 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
   int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
   int64_t outputLen  = GET_PARAM_BYTES(&pOutput[0]);
 
-  char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows + 1, 1);
-  char *output = outputBuf;
+  int32_t code = TSDB_CODE_SUCCESS;
+  char *  convBuf = taosMemoryMalloc(inputLen);
+  char *  output = taosMemoryCalloc(1, outputLen);
+  char    buf[400] = {0};
+
+  if (convBuf == NULL || output == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _end;
+  }
 
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
     if (colDataIsNull_s(pInput[0].columnData, i)) {
@@ -723,17 +730,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
     switch(outputType) {
       case TSDB_DATA_TYPE_TINYINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(int8_t *)output = taosStr2Int8(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+
+          convBuf[len] = 0;
+          *(int8_t *)output = taosStr2Int8(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input);
         }
@@ -741,17 +749,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_SMALLINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(int16_t *)output = taosStr2Int16(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(int16_t *)output = taosStr2Int16(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input);
         }
@@ -759,17 +767,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_INT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(int32_t *)output = taosStr2Int32(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+
+          convBuf[len] = 0;
+          *(int32_t *)output = taosStr2Int32(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input);
         }
@@ -777,17 +786,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_BIGINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(int64_t *)output = taosStr2Int64(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(int64_t *)output = taosStr2Int64(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input);
         }
@@ -795,17 +804,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_UTINYINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(uint8_t *)output = taosStr2UInt8(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(uint8_t *)output = taosStr2UInt8(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input);
         }
@@ -813,17 +822,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_USMALLINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(uint16_t *)output = taosStr2UInt16(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(uint16_t *)output = taosStr2UInt16(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input);
         }
@@ -831,17 +840,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_UINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(uint32_t *)output = taosStr2UInt32(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(uint32_t *)output = taosStr2UInt32(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input);
         }
@@ -849,17 +858,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_UBIGINT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(uint64_t *)output = taosStr2UInt64(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+
+          convBuf[len] = 0;
+          *(uint64_t *)output = taosStr2UInt64(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input);
         }
@@ -867,17 +877,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_FLOAT: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(float *)output = taosStr2Float(varDataVal(input), NULL);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(float *)output = taosStr2Float(buf, NULL);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(float *)output = taosStr2Float(newBuf, NULL);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(float *)output = taosStr2Float(convBuf, NULL);
         } else {
           GET_TYPED_DATA(*(float *)output, float, inputType, input);
         }
@@ -885,17 +895,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_DOUBLE: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(double *)output = taosStr2Double(varDataVal(input), NULL);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(double *)output = taosStr2Double(buf, NULL);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(double *)output = taosStr2Double(newBuf, NULL);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(double *)output = taosStr2Double(convBuf, NULL);
         } else {
           GET_TYPED_DATA(*(double *)output, double, inputType, input);
         }
@@ -903,17 +913,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_BOOL: {
         if (inputType == TSDB_DATA_TYPE_BINARY) {
-          *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10);
+          memcpy(buf, varDataVal(input), varDataLen(input));
+          buf[varDataLen(input)] = 0;
+          *(bool *)output = taosStr2Int8(buf, NULL, 10);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
-          newBuf[len] = 0;
-          *(bool *)output = taosStr2Int8(newBuf, NULL, 10);
-          taosMemoryFree(newBuf);
+          convBuf[len] = 0;
+          *(bool *)output = taosStr2Int8(convBuf, NULL, 10);
         } else {
           GET_TYPED_DATA(*(bool *)output, bool, inputType, input);
         }
@@ -944,22 +954,19 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
           len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input));
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          char *newBuf = taosMemoryCalloc(1, inputLen);
-          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf);
+          int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
           if (len < 0) {
-            taosMemoryFree(newBuf);
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
           len = TMIN(len, outputLen - VARSTR_HEADER_SIZE);
-          memcpy(varDataVal(output), newBuf, len);
+          memcpy(varDataVal(output), convBuf, len);
           varDataSetLen(output, len);
-          taosMemoryFree(newBuf);
         } else {
-          char tmp[400] = {0};
-          NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
-          int32_t len = (int32_t)strlen(tmp);
+          NUM_TO_STRING(inputType, input, sizeof(buf), buf);
+          int32_t len = (int32_t)strlen(buf);
           len = (outputLen - VARSTR_HEADER_SIZE) > len ? len : (outputLen - VARSTR_HEADER_SIZE);
-          memcpy(varDataVal(output), tmp, len);
+          memcpy(varDataVal(output), buf, len);
           varDataSetLen(output, len);
         }
         break;
@@ -972,14 +979,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
           len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" );
           bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
+
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_BINARY) {
           len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
           bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
@@ -987,38 +997,39 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
           memcpy(output, input, len + VARSTR_HEADER_SIZE);
           varDataSetLen(output, len);
         } else {
-          char tmp[400] = {0};
-          NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
-          len = (int32_t)strlen(tmp);
+          NUM_TO_STRING(inputType, input, sizeof(buf), buf);
+          len = (int32_t)strlen(buf);
           len = outputCharLen > len ? len : outputCharLen;
-          bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
+          bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
-            return TSDB_CODE_FAILED;
+            code = TSDB_CODE_FAILED;
+            goto _end;
           }
           varDataSetLen(output, len);
         }
+
         //for constant conversion, need to set proper length of pOutput description
         if (len < outputLen) {
           pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE;
         }
+
         break;
       }
       default: {
-        return TSDB_CODE_FAILED;
+        code = TSDB_CODE_FAILED;
+        goto _end;
       }
     }
 
     colDataAppend(pOutput->columnData, i, output, false);
-    if (IS_VAR_DATA_TYPE(outputType)) {
-      output += varDataTLen(output);
-    } else {
-      output += tDataTypes[outputType].bytes;
-    }
   }
 
   pOutput->numOfRows = pInput->numOfRows;
-  taosMemoryFree(outputBuf);
-  return TSDB_CODE_SUCCESS;
+
+  _end:
+  taosMemoryFree(output);
+  taosMemoryFree(buf);
+  return code;
 }
 
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
@@ -1400,8 +1411,6 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
       } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/
         GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]);
         if (type == TSDB_DATA_TYPE_TIMESTAMP) {
-          int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
-                           (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
           int64_t timeValSec = timeVal[k] / factor;
           if (timeValSec < 1000000000) {
             timeVal[k] = timeValSec;

From c889dcd6fd8504a1908149321a2f4865cafbc0b5 Mon Sep 17 00:00:00 2001
From: Xuefeng Tan <1172915550@qq.com>
Date: Mon, 10 Oct 2022 21:34:32 +0800
Subject: [PATCH 16/44] enh(taosAdapter):  support user specify timezone and
 return corresponding time (#17272)

---
 cmake/taosadapter_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index 16444c07f2..a9f8868f50 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG be729ab
+        GIT_TAG cc43ef0
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 2dcb5a9c29d1bebe630145564936beadf6b57315 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 10 Oct 2022 22:00:36 +0800
Subject: [PATCH 17/44] fix(query): fix an typo.

---
 source/libs/scalar/src/sclfunc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 50f6b16168..7279307d5f 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1028,7 +1028,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 
   _end:
   taosMemoryFree(output);
-  taosMemoryFree(buf);
+  taosMemoryFree(convBuf);
   return code;
 }
 

From dc15ab84d2952c1c173d695d3501f8225deccac6 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 10 Oct 2022 22:05:50 +0800
Subject: [PATCH 18/44] feat: taosbenchmark supplement insert (#17278)

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index cb9502b2b2..7bde332c8c 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 85b582b
+        GIT_TAG d58230c
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From fbfae317b32d3c34611a23b168548f5938f22151 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 11 Oct 2022 02:09:15 +0800
Subject: [PATCH 19/44] refactor

---
 include/common/tcommon.h                |  3 +-
 source/common/src/tdatablock.c          |  5 +-
 source/dnode/mnode/impl/src/mndStream.c |  1 -
 source/dnode/vnode/src/tq/tqSink.c      | 78 +++++++++++++++----------
 source/libs/executor/src/scanoperator.c | 74 ++++++++++++++++-------
 source/libs/wal/src/walMeta.c           | 11 ++--
 6 files changed, 111 insertions(+), 61 deletions(-)

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 8f7808cf02..eaddf4e983 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -171,7 +171,8 @@ typedef struct SDataBlockInfo {
   STimeWindow calWin;     // used for stream, do not serialize
   TSKEY       watermark;  // used for stream
 
-  char parTbName[TSDB_TABLE_NAME_LEN];  // used for stream
+  char  parTbName[TSDB_TABLE_NAME_LEN];  // used for stream partition
+  STag* pTag;                            // used for stream partition
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index fb99ba5361..1028a899b6 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1235,6 +1235,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
   taosArrayDestroy(pBlock->pDataBlock);
   pBlock->pDataBlock = NULL;
   taosMemoryFreeClear(pBlock->pBlockAgg);
+  taosMemoryFree(pBlock->info.pTag);
   memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
 }
 
@@ -1317,7 +1318,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) {
   pBlock->info.rows = 0;
   pBlock->info.type = type;
   pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) +
-                         sizeof(TSKEY) + TSDB_TABLE_NAME_LEN;
+                         sizeof(TSKEY) + VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
   pBlock->info.watermark = INT64_MIN;
 
   pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData));
@@ -1345,7 +1346,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) {
 
   // table name
   infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
-  infoData.info.bytes = TSDB_TABLE_NAME_LEN;
+  infoData.info.bytes = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
   taosArrayPush(pBlock->pDataBlock, &infoData);
 
   return pBlock;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index fa291177c9..ce6b3b0656 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -677,7 +677,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   // build stream obj from request
   if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
-    /*ASSERT(0);*/
     mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
     goto _OVER;
   }
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 7230b6232f..521d12fdab 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -84,26 +84,6 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
         continue;
       }
 
-      STagVal tagVal = {
-          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
-          .type = TSDB_DATA_TYPE_UBIGINT,
-          .i64 = (int64_t)pDataBlock->info.groupId,
-      };
-      STag* pTag = NULL;
-      taosArrayClear(tagArray);
-      taosArrayPush(tagArray, &tagVal);
-      tTagNew(tagArray, 1, false, &pTag);
-      if (pTag == NULL) {
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        taosArrayDestroy(tagArray);
-        return NULL;
-      }
-
-      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
-      strcpy(tagNameStr, "group_id");
-      taosArrayPush(tagName, tagNameStr);
-
       //      STag* pTag = NULL;
       //      taosArrayClear(tagArray);
       //      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
@@ -126,42 +106,76 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       //      }
 
       SVCreateTbReq createTbReq = {0};
+
+      // set const
       createTbReq.flags = 0;
       createTbReq.type = TSDB_CHILD_TABLE;
+      createTbReq.ctb.suid = suid;
 
+      // set super table name
       SName name = {0};
       tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
       createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name));  // strdup(stbFullName);
 
+      // set tag content
+      taosArrayClear(tagArray);
+      STagVal tagVal = {
+          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
+          .type = TSDB_DATA_TYPE_UBIGINT,
+          .i64 = (int64_t)pDataBlock->info.groupId,
+      };
+      taosArrayPush(tagArray, &tagVal);
+      createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
+
+      STag* pTag = NULL;
+      tTagNew(tagArray, 1, false, &pTag);
+      if (pTag == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        taosArrayDestroy(tagArray);
+        taosArrayDestroyP(schemaReqs, taosMemoryFree);
+        taosArrayDestroy(schemaReqSz);
+        return NULL;
+      }
+      createTbReq.ctb.pTag = (uint8_t*)pTag;
+
+      // set tag name
+      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
+      strcpy(tagNameStr, "group_id");
+      taosArrayPush(tagName, tagNameStr);
+      createTbReq.ctb.tagName = tagName;
+
+      // set table name
       if (pDataBlock->info.parTbName[0]) {
         createTbReq.name = strdup(pDataBlock->info.parTbName);
       } else {
         createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
       }
 
-      createTbReq.ctb.suid = suid;
-      createTbReq.ctb.pTag = (uint8_t*)pTag;
-      createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
-      createTbReq.ctb.tagName = tagName;
-
+      // save schema len
       int32_t code;
       int32_t schemaLen;
       tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
       if (code < 0) {
         tdDestroySVCreateTbReq(&createTbReq);
         taosArrayDestroy(tagArray);
-        taosMemoryFreeClear(ret);
+        taosArrayDestroyP(schemaReqs, taosMemoryFree);
+        taosArrayDestroy(schemaReqSz);
         return NULL;
       }
+      taosArrayPush(schemaReqSz, &schemaLen);
 
+      // save schema str
       void* schemaStr = taosMemoryMalloc(schemaLen);
       if (schemaStr == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        taosArrayDestroyP(schemaReqs, taosMemoryFree);
+        taosArrayDestroy(schemaReqSz);
         return NULL;
       }
       taosArrayPush(schemaReqs, &schemaStr);
-      taosArrayPush(schemaReqSz, &schemaLen);
 
       SEncoder encoder = {0};
       tEncoderInit(&encoder, schemaStr, schemaLen);
@@ -169,6 +183,10 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
       if (code < 0) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        taosArrayDestroyP(schemaReqs, taosMemoryFree);
+        taosArrayDestroy(schemaReqSz);
+        tEncoderClear(&encoder);
         return NULL;
       }
       tEncoderClear(&encoder);
@@ -221,10 +239,8 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
     tqDebug("tq sink, convert block %d, rows: %d", i, rows);
 
     int32_t dataLen = 0;
-
-    void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
-
     int32_t schemaLen = 0;
+    void*   blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
     if (createTb) {
       schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
       void* schemaStr = taosArrayGetP(schemaReqs, i);
@@ -262,7 +278,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
 
   ret->length = htonl(ret->length);
 
-  if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree);
+  taosArrayDestroyP(schemaReqs, taosMemoryFree);
   taosArrayDestroy(schemaReqSz);
 
   return ret;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b055ba1ce5..cd870a930a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1300,6 +1300,56 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
   return code;
 }
 
+static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) {
+  if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return;
+  if (pBlock == NULL || pBlock->info.rows == 0) return;
+
+  SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
+  ASSERT(pSrcBlock->info.rows == 1);
+
+  blockDataEnsureCapacity(pResBlock, 1);
+
+  projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL);
+  ASSERT(pResBlock->info.rows == 1);
+
+  // build tagArray
+  // build STag
+  // set STag
+
+  blockDataDestroy(pSrcBlock);
+}
+
+static void calBlockTbName(SExprSupp* pTbNameCalSup, SSDataBlock* pBlock) {
+  if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return;
+  if (pBlock == NULL || pBlock->info.rows == 0) return;
+
+  SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
+  ASSERT(pSrcBlock->info.rows == 1);
+
+  SSDataBlock* pResBlock = createDataBlock();
+  pResBlock->info.rowSize = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN;
+  SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0);
+  taosArrayPush(pResBlock->pDataBlock, &data);
+  blockDataEnsureCapacity(pResBlock, 1);
+
+  projectApplyFunctions(pTbNameCalSup->pExprInfo, pResBlock, pSrcBlock, pTbNameCalSup->pCtx, 1, NULL);
+  ASSERT(pResBlock->info.rows == 1);
+  ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1);
+  SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0);
+  ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR);
+
+  void* pData = colDataGetData(pCol, 0);
+  // TODO check tbname validation
+  if (pData != (void*)-1 && pData != NULL) {
+    memcpy(pBlock->info.parTbName, varDataVal(pData), varDataLen(pData));
+  } else {
+    pBlock->info.parTbName[0] = 0;
+  }
+
+  blockDataDestroy(pSrcBlock);
+  blockDataDestroy(pResBlock);
+}
+
 void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
                                       uint64_t* pGp, void* pTbName) {
   SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@@ -1421,28 +1471,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
   blockDataFreeRes((SSDataBlock*)pBlock);
 
-  if (pInfo->tbnameCalSup.numOfExprs > 0 && pInfo->pRes->info.rows > 0) {
-    SSDataBlock* pTmpBlock = blockCopyOneRow(pInfo->pRes, 0);
-    SSDataBlock* pResBlock = createDataBlock();
-    pResBlock->info.rowSize = TSDB_TABLE_NAME_LEN;
-    SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0);
-    taosArrayPush(pResBlock->pDataBlock, &data);
-    blockDataEnsureCapacity(pResBlock, 1);
-    projectApplyFunctions(pInfo->tbnameCalSup.pExprInfo, pResBlock, pTmpBlock, pInfo->tbnameCalSup.pCtx, 1, NULL);
-    ASSERT(pResBlock->info.rows == 1);
-    ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1);
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0);
-    ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR);
-    void* pData = colDataGetData(pCol, 0);
-    // TODO check tbname validation
-    if (pData != (void*)-1) {
-      memcpy(pInfo->pRes->info.parTbName, varDataVal(pData), varDataLen(pData));
-    } else {
-      pInfo->pRes->info.parTbName[0] = 0;
-    }
-    blockDataDestroy(pTmpBlock);
-    blockDataDestroy(pResBlock);
-  }
+  calBlockTbName(&pInfo->tbnameCalSup, pInfo->pRes);
 
   return 0;
 }
@@ -1780,6 +1809,7 @@ FETCH_NEXT_BLOCK:
           pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
           checkUpdateData(pInfo, true, pSDB, false);
           // printDataBlock(pSDB, "stream scan update");
+          calBlockTbName(&pInfo->tbnameCalSup, pSDB);
           return pSDB;
         }
         blockDataCleanup(pInfo->pUpdateDataRes);
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 5e70dce72d..8f97f78556 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -492,18 +492,21 @@ int walSaveMeta(SWal* pWal) {
   int  metaVer = walFindCurMetaVer(pWal);
   char fnameStr[WAL_FILE_LEN];
   walBuildMetaName(pWal, metaVer + 1, fnameStr);
-  TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
-  if (pMataFile == NULL) {
+  TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
+  if (pMetaFile == NULL) {
     return -1;
   }
   char* serialized = walMetaSerialize(pWal);
   int   len = strlen(serialized);
-  if (len != taosWriteFile(pMataFile, serialized, len)) {
+  if (len != taosWriteFile(pMetaFile, serialized, len)) {
     // TODO:clean file
+
+    taosCloseFile(&pMetaFile);
+    taosRemoveFile(fnameStr);
     return -1;
   }
 
-  taosCloseFile(&pMataFile);
+  taosCloseFile(&pMetaFile);
   // delete old file
   if (metaVer > -1) {
     walBuildMetaName(pWal, metaVer, fnameStr);

From 8862a31fb1c28571a4258c06dd355e4dacfe674d Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 11 Oct 2022 09:23:47 +0800
Subject: [PATCH 20/44] fix: coverity issues

---
 source/dnode/mgmt/mgmt_vnode/src/vmFile.c |  13 ++-
 source/dnode/mnode/impl/src/mndMnode.c    |   2 +-
 source/dnode/mnode/impl/src/mndVgroup.c   | 108 +++++++++++-----------
 3 files changed, 60 insertions(+), 63 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c
index b882e02e94..a49e855e39 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c
@@ -201,14 +201,13 @@ _OVER:
   taosCloseFile(&pFile);
   taosMemoryFree(content);
 
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    SVnodeObj *pVnode = ppVnodes[i];
-    if (pVnode != NULL) {
-      vmReleaseVnode(pMgmt, pVnode);
-    }
-  }
-
   if (ppVnodes != NULL) {
+    for (int32_t i = 0; i < numOfVnodes; ++i) {
+      SVnodeObj *pVnode = ppVnodes[i];
+      if (pVnode != NULL) {
+        vmReleaseVnode(pMgmt, pVnode);
+      }
+    }
     taosMemoryFree(ppVnodes);
   }
 
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index 4b76909a96..826e1d2fd0 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -87,7 +87,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
 
   SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
   if (pRaw == NULL) return -1;
-  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+  (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
 
   mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);
 
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index a31bf6b8e2..4a7221accb 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -1532,65 +1532,63 @@ _OVER:
 
 int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
   if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) {
+    return mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG);
+  }
+
+  SVgObj newVgroup = {0};
+  memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
+  mndTransSetSerial(pTrans);
+
+  if (newVgroup.replica < pDb->cfg.replications) {
+    mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
+          pVgroup->vnodeGid[0].dnodeId);
+
+    if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+
+    if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+  } else if (newVgroup.replica > pDb->cfg.replications) {
+    mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
+
+    SVnodeGid del1 = {0};
+    if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
+    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+
+    SVnodeGid del2 = {0};
+    if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
+    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+  } else {
+  }
+
+  {
+    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+    if (pVgRaw == NULL) return -1;
+    if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+      sdbFreeRaw(pVgRaw);
       return -1;
     }
-  } else {
-    SVgObj newVgroup = {0};
-    memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
-    mndTransSetSerial(pTrans);
+    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+  }
 
-    if (newVgroup.replica < pDb->cfg.replications) {
-      mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
-            pVgroup->vnodeGid[0].dnodeId);
-
-      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-
-      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-    } else if (newVgroup.replica > pDb->cfg.replications) {
-      mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
-
-      SVnodeGid del1 = {0};
-      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
-      if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-
-      SVnodeGid del2 = {0};
-      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
-      if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-    } else {
-    }
-
-    {
-      SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
-      if (pVgRaw == NULL) return -1;
-      if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
-        sdbFreeRaw(pVgRaw);
-        return -1;
-      }
-      (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
-    }
-
-    {
-      SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
-      if (pVgRaw == NULL) return -1;
-      if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
-        sdbFreeRaw(pVgRaw);
-        return -1;
-      }
-      (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+  {
+    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+    if (pVgRaw == NULL) return -1;
+    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
+      sdbFreeRaw(pVgRaw);
+      return -1;
     }
+    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
   }
 
   return 0;

From 187735dede3d9d01028032836b444223e2bb2d66 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Oct 2022 10:49:19 +0800
Subject: [PATCH 21/44] fix(query): fix  the  invalid write.

---
 source/libs/scalar/src/sclfunc.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 7279307d5f..6060f8cd92 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -711,7 +711,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 
   int32_t code = TSDB_CODE_SUCCESS;
   char *  convBuf = taosMemoryMalloc(inputLen);
-  char *  output = taosMemoryCalloc(1, outputLen);
+  char *  output = taosMemoryCalloc(1, outputLen + TSDB_NCHAR_SIZE);
   char    buf[400] = {0};
 
   if (convBuf == NULL || output == NULL) {
@@ -947,11 +947,12 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_BINARY: {
         if (inputType == TSDB_DATA_TYPE_BOOL) {
+          // NOTE: sprintf will append '\0' at the end of string
           int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false");
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_BINARY) {
           int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE);
-          len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input));
+          memcpy(varDataVal(output), varDataVal(input), len);
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
           int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);

From 5583ee42c0a7adc08e46ebbd8b09943d9acac79b Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 22/44] fix: coverity issues

CID: 400233
---
 source/common/src/ttszip.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c
index c86bf08e81..35ffe04e1d 100644
--- a/source/common/src/ttszip.c
+++ b/source/common/src/ttszip.c
@@ -35,7 +35,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
     // tscError("tmp file created failed since %s", terrstr());
     return NULL;
   }
-  
+
   STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf));
   if (pTSBuf == NULL) {
     return NULL;
@@ -52,10 +52,13 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
   }
 
   if (!autoDelete) {
-    taosRemoveFile(pTSBuf->path);
+    if (taosRemoveFile(pTSBuf->path) == NULL) {
+      taosMemoryFree(pTSBuf);
+      return NULL;
+    }
   }
 
-  if (NULL == allocResForTSBuf(pTSBuf)) {
+  if (allocResForTSBuf(pTSBuf) == NULL) {
     return NULL;
   }
 

From 445596011e020dfc637f94fdf5c88f62e306e401 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Tue, 11 Oct 2022 14:12:11 +0800
Subject: [PATCH 23/44] fix: win compile error

---
 include/os/os.h          |  3 +++
 source/os/src/osSocket.c | 14 ++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/include/os/os.h b/include/os/os.h
index 71966061a1..d646ffe4fd 100644
--- a/include/os/os.h
+++ b/include/os/os.h
@@ -51,6 +51,9 @@ extern "C" {
 #endif
 #else
 
+#ifndef __func__
+#define __func__ __FUNCTION__ 
+#endif
 #include <malloc.h>
 #include <time.h>
 #ifndef TD_USE_WINSOCK
diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index c1e02b48aa..bfeef248cd 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -267,21 +267,29 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void
     return -1;
   }
 #ifdef WINDOWS
+#ifdef TCP_KEEPCNT
   if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
     return 0;
   }
+#endif
 
+#ifdef TCP_KEEPIDLE
   if (level == SOL_TCP && optname == TCP_KEEPIDLE) {
     return 0;
   }
+#endif
 
+#ifdef TCP_KEEPINTVL
   if (level == SOL_TCP && optname == TCP_KEEPINTVL) {
     return 0;
   }
+#endif
 
+#ifdef TCP_KEEPCNT
   if (level == SOL_TCP && optname == TCP_KEEPCNT) {
     return 0;
   }
+#endif
 
   return setsockopt(pSocket->fd, level, optname, optval, optlen);
 #else
@@ -601,26 +609,32 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) {
 
 #ifndef __APPLE__
   // all fails on macosx
+#ifdef TCP_KEEPCNT
   int32_t probes = 3;
   if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
     // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
     taosCloseSocket(&pSocket);
     return -1;
   }
+#endif
 
+#ifdef TCP_KEEPIDLE
   int32_t alivetime = 10;
   if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
     // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
     taosCloseSocket(&pSocket);
     return -1;
   }
+#endif
 
+#ifdef TCP_KEEPINTVL
   int32_t interval = 3;
   if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
     // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
     taosCloseSocket(&pSocket);
     return -1;
   }
+#endif
 #endif  // __APPLE__
 
   int32_t nodelay = 1;

From fc598288bead276c69b290c4be84efff0d4c89f5 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 24/44] fix: coverity issues

CID: 400233
---
 source/common/src/ttszip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c
index 35ffe04e1d..3dd865e91f 100644
--- a/source/common/src/ttszip.c
+++ b/source/common/src/ttszip.c
@@ -52,7 +52,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
   }
 
   if (!autoDelete) {
-    if (taosRemoveFile(pTSBuf->path) == NULL) {
+    if (taosRemoveFile(pTSBuf->path) != 0) {
       taosMemoryFree(pTSBuf);
       return NULL;
     }

From c58ed31ad69990de77b4ca95110a94d13f6340f5 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 25/44] fix: coverity issues

CID: 400220
---
 docs/examples/c/tmq_example.c | 48 +++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/docs/examples/c/tmq_example.c b/docs/examples/c/tmq_example.c
index 19adaad116..a3bade308a 100644
--- a/docs/examples/c/tmq_example.c
+++ b/docs/examples/c/tmq_example.c
@@ -184,22 +184,54 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
 tmq_t* build_consumer() {
   tmq_conf_res_t code;
   tmq_conf_t*    conf = tmq_conf_new();
+
   code = tmq_conf_set(conf, "enable.auto.commit", "true");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "group.id", "cgrpName");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "client.id", "user defined name");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "td.connect.user", "root");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "auto.offset.reset", "earliest");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
+
   code = tmq_conf_set(conf, "experimental.snapshot.enable", "false");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
 
   tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
 

From b2f19def371da278b74c7dda97954363abf1c166 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 26/44] fix: coverity issues

CID: 400186
---
 source/common/src/tdataformat.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 110c3bc8f4..2a40859400 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -411,7 +411,7 @@ _exit:
 
 int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) {
   int32_t code = 0;
-  int32_t rLen;
+  int32_t rLen = 0;
 
   TSROW_LEN(pRow, rLen);
   (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen);
@@ -1711,4 +1711,4 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
 
 _exit:
   return code;
-}
\ No newline at end of file
+}

From 47f286869c18fc55877ec0d1294dcaba3a2d1f54 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 11 Oct 2022 11:19:53 +0800
Subject: [PATCH 27/44] fix(tdb): add cell header size while decoding left
 bytes

---
 source/libs/tdb/src/db/tdbBtree.c                         | 2 +-
 .../6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py  | 2 +-
 tests/system-test/6-cluster/clusterCommonCheck.py         | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index f663f4363d..a148c1e36b 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1485,7 +1485,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
     if (dropOfp) {
       int    ret = 0;
       SPgno  pgno = *(SPgno *)(pCell + nLocal - sizeof(SPgno));
-      int    nLeft = nPayload - nLocal + sizeof(SPgno);
+      int    nLeft = nPayload - nLocal + sizeof(SPgno) + nHeader;
       SPage *ofp;
       int    bytes;
 
diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py
index 371d147efc..9a94632a12 100644
--- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py
+++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py
@@ -132,7 +132,7 @@ class TDTestCase:
         tdSql.error("create mnode on dnode 2")
         tdSql.query("select * from information_schema.ins_dnodes;")
         print(tdSql.queryResult)
-        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDnodes(dnodeNumbers, 60)
 
         # create database and stable
         clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py
index c37e3541d4..28c4810833 100644
--- a/tests/system-test/6-cluster/clusterCommonCheck.py
+++ b/tests/system-test/6-cluster/clusterCommonCheck.py
@@ -37,10 +37,10 @@ class ClusterComCheck:
         tdSql.init(conn.cursor())
         # tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
-    def checkDnodes(self,dnodeNumbers):
+    def checkDnodes(self,dnodeNumbers, timeout=30):
         count=0
         # print(tdSql)
-        while count < 30:
+        while count < timeout:
             tdSql.query("select * from information_schema.ins_dnodes")
             # tdLog.debug(tdSql.queryResult)
             status=0
@@ -50,14 +50,14 @@ class ClusterComCheck:
             tdLog.info(status)
 
             if status == dnodeNumbers:
-                tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 30s! " %dnodeNumbers)
+                tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count))
                 return True
             count+=1
             time.sleep(1)
         else:
             tdSql.query("select * from information_schema.ins_dnodes")
             tdLog.debug(tdSql.queryResult)
-            tdLog.exit("it find cluster with %d dnodes but  check that there dnodes are not ready within 30s ! "%dnodeNumbers)
+            tdLog.exit("it find cluster with %d dnodes but  check that there dnodes are not ready within %ds ! "% (dnodeNumbers, timeout))
 
     def checkDbRows(self,dbNumbers):
         dbNumbers=int(dbNumbers)

From 37a9143fcbea4711d2b3ecbe98120eb27079d7bd Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 28/44] fix: coverity issues

CID: 400170
---
 source/libs/scalar/src/sclfunc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 4ead1147e4..86456a6ae1 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1992,7 +1992,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
     }
   }
 
-  if (hasNull) {
+  if (hasNull || (count == 0)) {
     colDataAppendNULL(pOutputData, 0);
   } else {
     if (IS_SIGNED_NUMERIC_TYPE(type)) {

From 26fae13c61c4307fd2daa55fd9839d44f3ed9500 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 11 Oct 2022 15:27:56 +0800
Subject: [PATCH 29/44] refactor(stream): state/session map id to tbname

---
 source/dnode/mnode/impl/src/mndDef.c          |  6 ++-
 source/dnode/vnode/src/tq/tqMeta.c            |  1 +
 source/dnode/vnode/src/tq/tqOffsetSnapshot.c  |  2 +-
 source/libs/executor/inc/executorimpl.h       |  6 +++
 source/libs/executor/src/executorimpl.c       | 40 +++++++++++++++++++
 source/libs/executor/src/timewindowoperator.c | 34 +++++++++++++---
 source/libs/wal/src/walMeta.c                 |  3 +-
 7 files changed, 83 insertions(+), 9 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index 059bfdeeb3..462b068a73 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -117,9 +117,13 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
       SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
       for (int32_t j = 0; j < innerSz; j++) {
         SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
-        if (pTask == NULL) return -1;
+        if (pTask == NULL) {
+          taosArrayDestroy(pArray);
+          return -1;
+        }
         if (tDecodeSStreamTask(pDecoder, pTask) < 0) {
           taosMemoryFree(pTask);
+          taosArrayDestroy(pArray);
           return -1;
         }
         taosArrayPush(pArray, &pTask);
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index c29541873e..1a57a391b1 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -180,6 +180,7 @@ int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
       return -1;
     }
   }
+  tdbFree(pKey);
   tdbTbcClose(pCur);
   return 0;
 }
diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c
index 084959af65..b63ff8af1d 100644
--- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c
+++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c
@@ -54,7 +54,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
 
   char*     fname = tqOffsetBuildFName(pReader->pTq->path, 0);
   TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
-  if (pFile != NULL) {
+  if (pFile == NULL) {
     taosMemoryFree(fname);
     return 0;
   }
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index de4e4dd1f8..1a4072f2cb 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -767,6 +767,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   SPhysiNode*         pPhyNode;   // create new child
   bool                isFinal;
   bool                ignoreExpiredData;
+  SHashObj*           pGroupIdTbNameMap;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct SStreamPartitionOperatorInfo {
@@ -844,6 +845,7 @@ typedef struct SStreamStateAggOperatorInfo {
   void*               pDelIterator;
   SArray*             pChildren;  // cache for children's result;
   bool                ignoreExpiredData;
+  SHashObj*           pGroupIdTbNameMap;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SSortOperatorInfo {
@@ -898,8 +900,12 @@ void    destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
 int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
                     const char* pkey);
 void    initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
+
+void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
+                           SDiskbasedBuf* pBuf);
 void    doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                                SDiskbasedBuf* pBuf);
+
 int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf);
 bool    hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
 void    initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 2f191db74d..8c1104f519 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1389,6 +1389,46 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
   return 0;
 }
 
+void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
+                           SDiskbasedBuf* pBuf) {
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SSDataBlock*   pBlock = pbInfo->pRes;
+
+  // set output datablock version
+  pBlock->info.version = pTaskInfo->version;
+
+  blockDataCleanup(pBlock);
+  if (!hasRemainResults(pGroupResInfo)) {
+    return;
+  }
+
+  // clear the existed group id
+  pBlock->info.groupId = 0;
+  ASSERT(!pbInfo->mergeResultBlock);
+  doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
+  if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
+    SStreamStateAggOperatorInfo* pInfo = pOperator->info;
+
+    char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
+    if (tbname != NULL) {
+      memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
+    } else {
+      pBlock->info.parTbName[0] = 0;
+    }
+  } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
+             pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
+             pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
+    SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
+
+    char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
+    if (tbname != NULL) {
+      memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
+    } else {
+      pBlock->info.parTbName[0] = 0;
+    }
+  }
+}
+
 void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                             SDiskbasedBuf* pBuf) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 69e1b66ecf..fa3171a8c1 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -3712,6 +3712,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->pPhyNode = pPhyNode;
   pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
 
+  pInfo->pGroupIdTbNameMap =
+      taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
+
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
   pOperator->blocking = true;
@@ -4335,7 +4338,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
       printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
       return pInfo->pDelRes;
     }
-    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+    doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
@@ -4354,6 +4357,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
     }
     printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
 
+    if (pBlock->info.parTbName[0]) {
+      taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
+                  TSDB_TABLE_NAME_LEN);
+      /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
+    }
+
     if (pBlock->info.type == STREAM_CLEAR) {
       SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
       doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX,
@@ -4435,7 +4444,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
     printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
     return pInfo->pDelRes;
   }
-  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
   printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
@@ -4466,7 +4475,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
   }
 
   {
-    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+    doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows > 0) {
       printDataBlock(pBInfo->pRes, "semi session");
       return pBInfo->pRes;
@@ -4507,6 +4516,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
     }
     printDataBlock(pBlock, "semi session recv");
 
+    if (pBlock->info.parTbName[0]) {
+      taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
+                  TSDB_TABLE_NAME_LEN);
+      /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
+    }
+
     if (pBlock->info.type == STREAM_CLEAR) {
       SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
       doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins);
@@ -4550,7 +4565,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
 
-  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
   if (pBInfo->pRes->info.rows > 0) {
     printDataBlock(pBInfo->pRes, "semi session");
     return pBInfo->pRes;
@@ -4594,6 +4609,10 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
         createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL,
                             destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
   }
+
+  pInfo->pGroupIdTbNameMap =
+      taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
+
   pOperator->operatorType = pPhyNode->type;
   if (numOfChild > 0) {
     pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
@@ -4894,7 +4913,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
       printDataBlock(pInfo->pDelRes, "single state");
       return pInfo->pDelRes;
     }
-    doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+    doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
     }
@@ -4956,7 +4975,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
     printDataBlock(pInfo->pDelRes, "single state");
     return pInfo->pDelRes;
   }
-  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
   printDataBlock(pBInfo->pRes, "single state");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
@@ -5029,6 +5048,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pChildren = NULL;
   pInfo->ignoreExpiredData = pStateNode->window.igExpired;
 
+  pInfo->pGroupIdTbNameMap =
+      taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
+
   pOperator->name = "StreamStateAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
   pOperator->blocking = true;
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 8f97f78556..233d6a87b8 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -150,7 +150,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
   const char* idxPattern = "^[0-9]+.idx$";
   regex_t     logRegPattern;
   regex_t     idxRegPattern;
-  SArray*     actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
 
   regcomp(&logRegPattern, logPattern, REG_EXTENDED);
   regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
@@ -163,6 +162,8 @@ int walCheckAndRepairMeta(SWal* pWal) {
     return -1;
   }
 
+  SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
+
   // scan log files and build new meta
   TdDirEntryPtr pDirEntry;
   while ((pDirEntry = taosReadDir(pDir)) != NULL) {

From ccdd3ff8c25ae02e00f7683165d89ecb9e04c5ae Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 11 Oct 2022 15:32:07 +0800
Subject: [PATCH 30/44] fix: unit test case for tdb ovfl pages

---
 source/libs/tdb/test/tdbExOVFLTest.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp
index 58ea6147ef..d98c271edb 100644
--- a/source/libs/tdb/test/tdbExOVFLTest.cpp
+++ b/source/libs/tdb/test/tdbExOVFLTest.cpp
@@ -238,8 +238,8 @@ TEST(TdbOVFLPagesTest, TbGetTest) {
   }
 }
 
-TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
-  // TEST(TdbOVFLPagesTest, TbDeleteTest) {
+// TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
+TEST(TdbOVFLPagesTest, TbDeleteTest) {
   int ret = 0;
 
   taosRemoveDir("tdb");
@@ -267,7 +267,8 @@ TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
   tdbBegin(pEnv, &txn);
 
   // generate value payload
-  char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  char val[((4083 - 4 - 3 - 2) + 1) * 2];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
   int  valLen = sizeof(val) / sizeof(val[0]);
   generateBigVal(val, valLen);
 
@@ -340,8 +341,8 @@ tdbBegin(pEnv, &txn);
   tdbTxnClose(&txn);
 }
 
-TEST(tdb_test, DISABLED_simple_insert1) {
-  // TEST(tdb_test, simple_insert1) {
+// TEST(tdb_test, DISABLED_simple_insert1) {
+TEST(tdb_test, simple_insert1) {
   int           ret;
   TDB          *pEnv;
   TTB          *pDb;

From 7da3ed2748b9dd80726006c1ae2b5c5247b43836 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 11 Oct 2022 15:36:35 +0800
Subject: [PATCH 31/44] fix: support statistics of insert_req

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |  8 ++--
 source/dnode/vnode/src/vnd/vnodeCommit.c    | 38 ------------------
 source/dnode/vnode/src/vnd/vnodeOpen.c      |  4 --
 source/dnode/vnode/src/vnd/vnodeQuery.c     | 44 ++++++++++++++-------
 source/dnode/vnode/src/vnd/vnodeSvr.c       |  2 +-
 5 files changed, 34 insertions(+), 62 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 5451fe33f1..094d0559d5 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -66,10 +66,10 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
   pInfo->vstat.totalVnodes = totalVnodes;
   pInfo->vstat.masterNum = masterNum;
   pInfo->vstat.numOfSelectReqs = numOfSelectReqs - pMgmt->state.numOfSelectReqs;
-  pInfo->vstat.numOfInsertReqs = numOfInsertReqs - pMgmt->state.numOfInsertReqs;
-  pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs - pMgmt->state.numOfInsertSuccessReqs;
-  pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs - pMgmt->state.numOfBatchInsertReqs;
-  pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs - pMgmt->state.numOfBatchInsertSuccessReqs;
+  pInfo->vstat.numOfInsertReqs = numOfInsertReqs;
+  pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs;
+  pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs;
+  pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
   pMgmt->state.totalVnodes = totalVnodes;
   pMgmt->state.masterNum = masterNum;
   pMgmt->state.numOfSelectReqs = numOfSelectReqs;
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index ae55d50456..922b5ac5d8 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -227,10 +227,6 @@ int vnodeCommit(SVnode *pVnode) {
   info.state.committed = pVnode->state.applied;
   info.state.commitTerm = pVnode->state.applyTerm;
   info.state.commitID = pVnode->state.commitID;
-  info.statis.nInsert = pVnode->statis.nInsert;
-  info.statis.nInsertSuccess = pVnode->statis.nInsertSuccess;
-  info.statis.nBatchInsert = pVnode->statis.nBatchInsert;
-  info.statis.nBatchInsertSuccess = pVnode->statis.nBatchInsertSuccess;
 
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   if (vnodeSaveInfo(dir, &info) < 0) {
@@ -357,32 +353,6 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   return 0;
 }
 
-static int vnodeEncodeStatis(const void *pObj, SJson *pJson) {
-  const SVStatis *pStatis = (SVStatis *)pObj;
-
-  if (tjsonAddIntegerToObject(pJson, "insert", pStatis->nInsert) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "insert success", pStatis->nInsertSuccess) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "batch insert", pStatis->nBatchInsert) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "batch insert success", pStatis->nBatchInsertSuccess) < 0) return -1;
-
-  return 0;
-}
-
-static int vnodeDecodeStatis(const SJson *pJson, void *pObj) {
-  SVStatis *pStatis = (SVStatis *)pObj;
-
-  int32_t code;
-  tjsonGetNumberValue(pJson, "insert", pStatis->nInsert, code);
-  if (code < 0) return -1;
-  tjsonGetNumberValue(pJson, "insert success", pStatis->nInsertSuccess, code);
-  if (code < 0) return -1;
-  tjsonGetNumberValue(pJson, "batch insert", pStatis->nBatchInsert, code);
-  if (code < 0) return -1;
-  tjsonGetNumberValue(pJson, "batch insert success", pStatis->nBatchInsertSuccess, code);
-  if (code < 0) return -1;
-  return 0;
-}
-
 static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
   SJson *pJson;
   char  *pData;
@@ -402,10 +372,6 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
     goto _err;
   }
 
-  if (tjsonAddObject(pJson, "statis", vnodeEncodeStatis, (void *)&pInfo->statis) < 0) {
-    goto _err;
-  }
-
   pData = tjsonToString(pJson);
   if (pData == NULL) {
     goto _err;
@@ -437,10 +403,6 @@ static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
     goto _err;
   }
 
-  if (tjsonToObject(pJson, "statis", vnodeDecodeStatis, (void *)&pInfo->statis) < 0) {
-    goto _err;
-  }
-
   tjsonDelete(pJson);
 
   return 0;
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index cd87784732..616aa39bdf 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -85,10 +85,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->state.applied = info.state.committed;
   pVnode->state.commitID = info.state.commitID;
   pVnode->state.commitTerm = info.state.commitTerm;
-  pVnode->statis.nInsert = info.statis.nInsert;
-  pVnode->statis.nInsertSuccess = info.statis.nInsertSuccess;
-  pVnode->statis.nBatchInsert = info.statis.nBatchInsert;
-  pVnode->statis.nBatchInsertSuccess = info.statis.nBatchInsertSuccess;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
   taosThreadMutexInit(&pVnode->lock, NULL);
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index ba8fbaf553..77499ee684 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -365,22 +365,36 @@ _exit:
   return code;
 }
 
+#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, type)                   \
+  do {                                                                \
+    if (oVal != atomic_val_compare_exchange_##type(&pVar, oVal, 0)) { \
+      int##type##_t tmpVal = atomic_sub_fetch_##type(&pVar, oVal);    \
+      ASSERT(tmpVal >= 0);                                            \
+    }                                                                 \
+  } while (0)
+
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
-  pLoad->vgId = TD_VID(pVnode);
-  pLoad->syncState = syncGetMyRole(pVnode->sync);
-  pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
-  pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
-  pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
-  pLoad->totalStorage = (int64_t)3 * 1073741824;
-  pLoad->compStorage = (int64_t)2 * 1073741824;
-  pLoad->pointsWritten = 100;
-  pLoad->numOfSelectReqs = 1;
-  pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
-  pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
-  pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
-  pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
-  return 0;
-}
+    pLoad->vgId = TD_VID(pVnode);
+    pLoad->syncState = syncGetMyRole(pVnode->sync);
+    pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
+    pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
+    pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
+    pLoad->totalStorage = (int64_t)3 * 1073741824;
+    pLoad->compStorage = (int64_t)2 * 1073741824;
+    pLoad->pointsWritten = 100;
+    pLoad->numOfSelectReqs = 1;
+    pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
+    pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
+    pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
+    pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
+
+    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64);
+    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
+    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
+    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
+
+    return 0;
+  }
 
 void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
   if (dbname) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index bff836faf5..6d72c0dec6 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -951,8 +951,8 @@ _exit:
   // 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
-    tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
     atomic_fetch_add_64(&pVnode->statis.nBatchInsertSuccess, 1);
+    tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
   }
 
   vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version);

From 18fa48ae57acc7d04fa099f8680ac7ee1f98e425 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 11 Oct 2022 15:54:01 +0800
Subject: [PATCH 32/44] feat: support batch loading of csv files

---
 include/common/tglobal.h                |   1 +
 include/libs/catalog/catalog.h          |   4 +
 include/libs/nodes/querynodes.h         |   1 -
 include/libs/parser/parser.h            |  11 ++
 source/client/inc/clientInt.h           |  25 ++-
 source/client/src/clientImpl.c          | 151 +++++++++-------
 source/client/src/clientMain.c          | 170 ++++++++++--------
 source/client/src/clientSml.c           |   2 +-
 source/common/src/tglobal.c             |  43 +++--
 source/libs/catalog/src/ctgUtil.c       | 228 ++++++++++++++++++++++++
 source/libs/parser/src/parInsert.c      | 108 ++++++++---
 source/libs/parser/src/parser.c         |   6 +
 source/libs/parser/test/mockCatalog.cpp |  20 +--
 source/libs/parser/test/parTestUtil.cpp |   1 -
 14 files changed, 564 insertions(+), 207 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index bd5e74387e..350cd785d9 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -103,6 +103,7 @@ extern bool    tsKeepColumnName;
 // client
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
+extern int32_t tsMaxMemUsedByInsert;
 
 // build info
 extern char version[];
diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h
index ee566d759a..c3caac00ad 100644
--- a/include/libs/catalog/catalog.h
+++ b/include/libs/catalog/catalog.h
@@ -303,6 +303,10 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re
 
 int32_t catalogClearCache(void);
 
+SMetaData* catalogCloneMetaData(SMetaData* pData);
+
+void catalogFreeMetaData(SMetaData* pData);
+
 /**
  * Destroy catalog and relase all resources
  */
diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index 5ee097bd92..e1acf0dd6a 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -385,7 +385,6 @@ typedef struct SCmdMsgInfo {
   SEpSet  epSet;
   void*   pMsg;
   int32_t msgLen;
-  void*   pExtension;  // todo remove it soon
 } SCmdMsgInfo;
 
 typedef enum EQueryExecMode {
diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h
index b1a937910d..bcd2316baf 100644
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@@ -33,6 +33,13 @@ typedef struct SStmtCallback {
   int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
 } SStmtCallback;
 
+typedef struct SParseCsvCxt {
+  TdFilePtr   fp;           // last parsed file
+  int32_t     tableNo;      // last parsed table
+  SName       tableName;    // last parsed table
+  const char* pLastSqlPos;  // the location of the last parsed sql
+} SParseCsvCxt;
+
 typedef struct SParseContext {
   uint64_t         requestId;
   int64_t          requestRid;
@@ -57,6 +64,8 @@ typedef struct SParseContext {
   SArray*          pTableMetaPos;    // sql table pos => catalog data pos
   SArray*          pTableVgroupPos;  // sql table pos => catalog data pos
   int64_t          allocatorId;
+  bool             needMultiParse;
+  SParseCsvCxt     csvCxt;
 } SParseContext;
 
 int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@@ -67,6 +76,8 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq
 int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq,
                             const struct SMetaData* pMetaData, SQuery* pQuery);
 
+void qDestroyParseContext(SParseContext* pCxt);
+
 void qDestroyQuery(SQuery* pQueryNode);
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 535a436c6c..f815adfeaa 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -380,16 +380,25 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
 // --- mq
 void hbMgrInitMqHbRspHandle();
 
+typedef struct SSqlCallbackWrapper {
+  SParseContext* pParseCtx;
+  SCatalogReq*   pCatalogReq;
+  SMetaData*     pResultMeta;
+  SRequestObj*   pRequest;
+} SSqlCallbackWrapper;
+
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res);
 int32_t      scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
-void         launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta);
-int32_t      refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
-int32_t      updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
-void         doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
-int32_t      removeMeta(STscObj* pTscObj, SArray* tbList);
-int32_t      handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
-int32_t      handleCreateTbExecRes(void* res, SCatalog* pCatalog);
-bool         qnodeRequired(SRequestObj* pRequest);
+void    launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper);
+int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
+int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
+void    doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
+int32_t removeMeta(STscObj* pTscObj, SArray* tbList);
+int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
+int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
+bool    qnodeRequired(SRequestObj* pRequest);
+int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest);
+void    destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper);
 
 #ifdef __cplusplus
 }
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 8ffc88ec28..08e844dc2c 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -868,10 +868,11 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
   return code;
 }
 
-//todo refacto the error code  mgmt
+// todo refacto the error code  mgmt
 void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
-  SRequestObj* pRequest = (SRequestObj*)param;
-  STscObj* pTscObj = pRequest->pTscObj;
+  SSqlCallbackWrapper* pWrapper = param;
+  SRequestObj*         pRequest = pWrapper->pRequest;
+  STscObj*             pTscObj = pRequest->pTscObj;
 
   pRequest->code = code;
   if (pResult) {
@@ -882,7 +883,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
   int32_t type = pRequest->type;
   if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) {
     if (pResult) {
-      pRequest->body.resInfo.numOfRows = pResult->numOfRows;
+      pRequest->body.resInfo.numOfRows += pResult->numOfRows;
 
       // record the insert rows
       if (TDMT_VND_SUBMIT == type) {
@@ -899,12 +900,13 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
            pRequest->requestId);
 
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
-    tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64,
-             pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId);
+    tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
+             tstrerror(code), pRequest->retry, pRequest->requestId);
     pRequest->prevCode = code;
     schedulerFreeJob(&pRequest->body.queryJob, 0);
     qDestroyQuery(pRequest->pQuery);
     pRequest->pQuery = NULL;
+    destorySqlCallbackWrapper(pWrapper);
     doAsyncQuery(pRequest, true);
     return;
   }
@@ -920,6 +922,15 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
     pRequest->code = code1;
   }
 
+  if (pRequest->code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) {
+    code = continueInsertFromCsv(pWrapper, pRequest);
+    if (TSDB_CODE_SUCCESS == code) {
+      return;
+    }
+  }
+
+  destorySqlCallbackWrapper(pWrapper);
+
   // return to client
   pRequest->body.queryFp(pRequest->body.param, pRequest, code);
 }
@@ -1020,76 +1031,86 @@ SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool vali
   return launchQueryImpl(pRequest, pQuery, false, NULL);
 }
 
-void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
+static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta,
+                                 SSqlCallbackWrapper* pWrapper) {
+  pRequest->type = pQuery->msgType;
+
+  SArray*      pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
+  SPlanContext cxt = {.queryId = pRequest->requestId,
+                      .acctId = pRequest->pTscObj->acctId,
+                      .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
+                      .pAstRoot = pQuery->pRoot,
+                      .showRewrite = pQuery->showRewrite,
+                      .pMsg = pRequest->msgBuf,
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .pUser = pRequest->pTscObj->user,
+                      .sysInfo = pRequest->pTscObj->sysInfo,
+                      .allocatorId = pRequest->allocatorRefId};
+  SQueryPlan*  pDag = NULL;
+  int32_t      code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
+  if (code) {
+    tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
+             pRequest->requestId);
+  } else {
+    pRequest->body.subplanNum = pDag->numOfSubplans;
+  }
+
+  pRequest->metric.planEnd = taosGetTimestampUs();
+
+  if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
+    SArray* pNodeList = NULL;
+    buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
+
+    SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter,
+                             .requestId = pRequest->requestId,
+                             .requestObjRefId = pRequest->self};
+    SSchedulerReq    req = {
+           .syncReq = false,
+           .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT),
+           .pConn = &conn,
+           .pNodeList = pNodeList,
+           .pDag = pDag,
+           .allocatorRefId = pRequest->allocatorRefId,
+           .sql = pRequest->sqlstr,
+           .startTs = pRequest->metric.start,
+           .execFp = schedulerExecCb,
+           .cbParam = pWrapper,
+           .chkKillFp = chkRequestKilled,
+           .chkKillParam = (void*)pRequest->self,
+           .pExecRes = NULL,
+    };
+    code = schedulerExecJob(&req, &pRequest->body.queryJob);
+    taosArrayDestroy(pNodeList);
+  } else {
+    tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
+             pRequest->requestId);
+    destorySqlCallbackWrapper(pWrapper);
+    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+  }
+
+  // todo not to be released here
+  taosArrayDestroy(pMnodeList);
+
+  return code;
+}
+
+void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) {
   int32_t code = 0;
 
   pRequest->body.execMode = pQuery->execMode;
+  if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) {
+    destorySqlCallbackWrapper(pWrapper);
+  }
 
   switch (pQuery->execMode) {
     case QUERY_EXEC_MODE_LOCAL:
       asyncExecLocalCmd(pRequest, pQuery);
-      return;
+      break;
     case QUERY_EXEC_MODE_RPC:
       code = asyncExecDdlQuery(pRequest, pQuery);
       break;
     case QUERY_EXEC_MODE_SCHEDULE: {
-      SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
-
-      pRequest->type = pQuery->msgType;
-
-      SPlanContext cxt = {.queryId = pRequest->requestId,
-                          .acctId = pRequest->pTscObj->acctId,
-                          .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
-                          .pAstRoot = pQuery->pRoot,
-                          .showRewrite = pQuery->showRewrite,
-                          .pMsg = pRequest->msgBuf,
-                          .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
-                          .pUser = pRequest->pTscObj->user,
-                          .sysInfo = pRequest->pTscObj->sysInfo,
-                          .allocatorId = pRequest->allocatorRefId};
-
-      SAppInstInfo* pAppInfo = getAppInfo(pRequest);
-      SQueryPlan*   pDag = NULL;
-      code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
-      if (code) {
-        tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
-                 pRequest->requestId);
-      } else {
-        pRequest->body.subplanNum = pDag->numOfSubplans;
-      }
-
-      pRequest->metric.planEnd = taosGetTimestampUs();
-      if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
-        SArray* pNodeList = NULL;
-        buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
-
-        SRequestConnInfo conn = {
-            .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
-        SSchedulerReq req = {
-            .syncReq = false,
-            .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT),
-            .pConn = &conn,
-            .pNodeList = pNodeList,
-            .pDag = pDag,
-            .allocatorRefId = pRequest->allocatorRefId,
-            .sql = pRequest->sqlstr,
-            .startTs = pRequest->metric.start,
-            .execFp = schedulerExecCb,
-            .cbParam = pRequest,
-            .chkKillFp = chkRequestKilled,
-            .chkKillParam = (void*)pRequest->self,
-            .pExecRes = NULL,
-        };
-        code = schedulerExecJob(&req, &pRequest->body.queryJob);
-        taosArrayDestroy(pNodeList);
-      } else {
-        tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
-                 pRequest->requestId);
-        pRequest->body.queryFp(pRequest->body.param, pRequest, code);
-      }
-
-      // todo not to be released here
-      taosArrayDestroy(pMnodeList);
+      code = asyncExecSchQuery(pRequest, pQuery, pResultMeta, pWrapper);
       break;
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 5b255a28ea..6f8cef7c0d 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -667,43 +667,47 @@ const char *taos_get_server_info(TAOS *taos) {
   return pTscObj->sDetailVer;
 }
 
-typedef struct SqlParseWrapper {
-  SParseContext *pCtx;
-  SCatalogReq    catalogReq;
-  SRequestObj   *pRequest;
-} SqlParseWrapper;
-
 static void destoryTablesReq(void *p) {
   STablesReq *pRes = (STablesReq *)p;
   taosArrayDestroy(pRes->pTables);
 }
 
-static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
-  taosArrayDestroy(pWrapper->catalogReq.pDbVgroup);
-  taosArrayDestroy(pWrapper->catalogReq.pDbCfg);
-  taosArrayDestroy(pWrapper->catalogReq.pDbInfo);
-  taosArrayDestroyEx(pWrapper->catalogReq.pTableMeta, destoryTablesReq);
-  taosArrayDestroyEx(pWrapper->catalogReq.pTableHash, destoryTablesReq);
-  taosArrayDestroy(pWrapper->catalogReq.pUdf);
-  taosArrayDestroy(pWrapper->catalogReq.pIndex);
-  taosArrayDestroy(pWrapper->catalogReq.pUser);
-  taosArrayDestroy(pWrapper->catalogReq.pTableIndex);
-  taosArrayDestroy(pWrapper->pCtx->pTableMetaPos);
-  taosArrayDestroy(pWrapper->pCtx->pTableVgroupPos);
-  taosMemoryFree(pWrapper->pCtx);
+static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
+  if (NULL == pCatalogReq) {
+    return;
+  }
+  taosArrayDestroy(pCatalogReq->pDbVgroup);
+  taosArrayDestroy(pCatalogReq->pDbCfg);
+  taosArrayDestroy(pCatalogReq->pDbInfo);
+  taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq);
+  taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
+  taosArrayDestroy(pCatalogReq->pUdf);
+  taosArrayDestroy(pCatalogReq->pIndex);
+  taosArrayDestroy(pCatalogReq->pUser);
+  taosArrayDestroy(pCatalogReq->pTableIndex);
+  taosMemoryFree(pCatalogReq);
+}
+
+void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
+  if (NULL == pWrapper) {
+    return;
+  }
+  destoryCatalogReq(pWrapper->pCatalogReq);
+  qDestroyParseContext(pWrapper->pParseCtx);
+  catalogFreeMetaData(pWrapper->pResultMeta);
   taosMemoryFree(pWrapper);
 }
 
 void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
-  SqlParseWrapper *pWrapper = (SqlParseWrapper *)param;
-  SRequestObj     *pRequest = pWrapper->pRequest;
-  SQuery          *pQuery = pRequest->pQuery;
+  SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param;
+  SRequestObj         *pRequest = pWrapper->pRequest;
+  SQuery              *pQuery = pRequest->pQuery;
 
   pRequest->metric.ctgEnd = taosGetTimestampUs();
   qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
 
   if (code == TSDB_CODE_SUCCESS) {
-    code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
+    code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
     pRequest->stableQuery = pQuery->stableQuery;
     if (pQuery->pRoot) {
       pRequest->stmtType = pQuery->pRoot->type;
@@ -712,6 +716,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
 
   pRequest->metric.semanticEnd = taosGetTimestampUs();
 
+  if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) {
+    pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta);
+    if (NULL == pWrapper->pResultMeta) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
   if (code == TSDB_CODE_SUCCESS) {
     if (pQuery->haveResultSet) {
       setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
@@ -722,15 +733,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
     TSWAP(pRequest->tableList, (pQuery)->pTableList);
     TSWAP(pRequest->targetTableList, (pQuery)->pTargetTableList);
 
-    destorySqlParseWrapper(pWrapper);
-
-    double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0;
+    double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd) / 1000.0;
     tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64,
              pRequest->self, el, pRequest->requestId);
 
-    launchAsyncQuery(pRequest, pQuery, pResultMeta);
+    launchAsyncQuery(pRequest, pQuery, pResultMeta, pWrapper);
   } else {
-    destorySqlParseWrapper(pWrapper);
+    destorySqlCallbackWrapper(pWrapper);
     qDestroyQuery(pRequest->pQuery);
     pRequest->pQuery = NULL;
 
@@ -750,6 +759,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
   }
 }
 
+int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) {
+  qDestroyQuery(pRequest->pQuery);
+  pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
+  if (NULL == pRequest->pQuery) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS);
+  return TSDB_CODE_SUCCESS;
+}
+
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
   int64_t connId = *(int64_t *)taos;
   taosAsyncQueryImpl(connId, sql, fp, param, false);
@@ -786,37 +805,48 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
 }
 
 void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
-  SParseContext *pCxt = NULL;
-  STscObj       *pTscObj = pRequest->pTscObj;
-  int32_t        code = 0;
+  STscObj             *pTscObj = pRequest->pTscObj;
+  SSqlCallbackWrapper *pWrapper = NULL;
+  int32_t              code = TSDB_CODE_SUCCESS;
 
   if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
     code = pRequest->prevCode;
-    goto _error;
   }
 
-  code = createParseContext(pRequest, &pCxt);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
+  if (TSDB_CODE_SUCCESS == code) {
+    pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
+    if (pWrapper == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
+      pWrapper->pRequest = pRequest;
+    }
   }
 
-  pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
-  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createParseContext(pRequest, &pWrapper->pParseCtx);
   }
 
-  pRequest->metric.syntaxStart = taosGetTimestampUs();
-
-  SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)};
-  code = qParseSqlSyntax(pCxt, &pRequest->pQuery, &catalogReq);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
+  if (TSDB_CODE_SUCCESS == code) {
+    pWrapper->pParseCtx->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
+    code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pWrapper->pParseCtx->pCatalog);
   }
 
-  pRequest->metric.syntaxEnd = taosGetTimestampUs();
+  if (TSDB_CODE_SUCCESS == code) {
+    pRequest->metric.syntaxStart = taosGetTimestampUs();
 
-  if (!updateMetaForce) {
+    pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq));
+    if (pWrapper->pCatalogReq == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
+      pWrapper->pCatalogReq->forceUpdate = updateMetaForce;
+      pWrapper->pCatalogReq->qNodeRequired = qnodeRequired(pRequest);
+      code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq);
+    }
+
+    pRequest->metric.syntaxEnd = taosGetTimestampUs();
+  }
+
+  if (TSDB_CODE_SUCCESS == code && !updateMetaForce) {
     SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
     if (NULL == pRequest->pQuery->pRoot) {
       atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
@@ -825,38 +855,26 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
     }
   }
 
-  SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper));
-  if (pWrapper == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _error;
+  if (TSDB_CODE_SUCCESS == code) {
+    SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter,
+                             .requestId = pWrapper->pParseCtx->requestId,
+                             .requestObjRefId = pWrapper->pParseCtx->requestRid,
+                             .mgmtEps = pWrapper->pParseCtx->mgmtEpSet};
+
+    pRequest->metric.ctgStart = taosGetTimestampUs();
+
+    code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback,
+                                  pWrapper, &pRequest->body.queryJob);
   }
 
-  pWrapper->pCtx = pCxt;
-  pWrapper->pRequest = pRequest;
-  pWrapper->catalogReq = catalogReq;
-
-  SRequestConnInfo conn = {.pTrans = pCxt->pTransporter,
-                           .requestId = pCxt->requestId,
-                           .requestObjRefId = pCxt->requestRid,
-                           .mgmtEps = pCxt->mgmtEpSet};
-
-  pRequest->metric.ctgStart = taosGetTimestampUs();
-
-  code = catalogAsyncGetAllMeta(pCxt->pCatalog, &conn, &catalogReq, retrieveMetaCallback, pWrapper,
-                                &pRequest->body.queryJob);
-  pCxt = NULL;
-  if (code == TSDB_CODE_SUCCESS) {
-    return;
+  if (TSDB_CODE_SUCCESS != code) {
+    tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
+             pRequest->requestId);
+    destorySqlCallbackWrapper(pWrapper);
+    terrno = code;
+    pRequest->code = code;
+    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
   }
-
-_error:
-  tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
-           pRequest->requestId);
-  taosMemoryFree(pCxt);
-
-  terrno = code;
-  pRequest->code = code;
-  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
 }
 
 static void fetchCallback(void *pResult, void *param, int32_t code) {
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 195466061d..4eb1a3712c 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -2336,7 +2336,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
   SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
   atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
 
-  launchAsyncQuery(info->pRequest, info->pQuery, NULL);
+  launchAsyncQuery(info->pRequest, info->pQuery, NULL, NULL);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index ce9a9a7b50..88df54a9ea 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -85,7 +85,8 @@ uint16_t tsTelemPort = 80;
 char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
 char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = "";  // user defined child table name can be specified in tag value.
                                                      // If set to empty system will generate table name using MD5 hash.
-bool tsSmlDataFormat = false;  // true means that the name and order of cols in each line are the same(only for influx protocol)
+// true means that the name and order of cols in each line are the same(only for influx protocol)
+bool tsSmlDataFormat = false;
 
 // query
 int32_t tsQueryPolicy = 1;
@@ -125,6 +126,9 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000;
 // 1 database precision unit for interval time range, changed accordingly
 int32_t tsMinIntervalTime = 1;
 
+// maximum memory allowed to be allocated for a single csv load (in MB)
+int32_t tsMaxMemUsedByInsert = 1024;
+
 // the maximum allowed query buffer size during query processing for each data node.
 // -1 no limit (default)
 // 0  no query allowed, queries are disabled
@@ -296,6 +300,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 2;
   tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
@@ -374,8 +379,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1;
 
-//  tsNumOfVnodeFetchThreads = 1;
-//  if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1;
+  //  tsNumOfVnodeFetchThreads = 1;
+  //  if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1;
 
   tsNumOfVnodeWriteThreads = tsNumOfCores;
   tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
@@ -497,15 +502,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->stype = stype;
   }
 
-/*
-  pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads");
-  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsNumOfVnodeFetchThreads = numOfCores / 4;
-    tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
-    pItem->i32 = tsNumOfVnodeFetchThreads;
-    pItem->stype = stype;
-  }
-*/
+  /*
+    pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads");
+    if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+      tsNumOfVnodeFetchThreads = numOfCores / 4;
+      tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
+      pItem->i32 = tsNumOfVnodeFetchThreads;
+      pItem->stype = stype;
+    }
+  */
 
   pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
@@ -648,6 +653,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
   tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
 
+  tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
+
   tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
   tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
   tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
@@ -705,7 +712,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
   tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
   tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32;
-//  tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
+  //  tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
   tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
   tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
   tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
@@ -877,6 +884,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
             tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
           } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) {
             tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
+          } else if (strcasecmp("maxMemUsedByInsert", name) == 0) {
+            tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
           }
           break;
         }
@@ -955,10 +964,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
         tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
       } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) {
         tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
-/*
-      } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
-        tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
-*/
+        /*
+              } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
+                tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
+        */
       } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) {
         tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
       } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {
diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c
index 97b174de1c..0c0ca9649a 100644
--- a/source/libs/catalog/src/ctgUtil.c
+++ b/source/libs/catalog/src/ctgUtil.c
@@ -1193,4 +1193,232 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) {
   return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx);
 }
 
+static void* ctgCloneDbVgroup(void* pSrc) {
+  return taosArrayDup((const SArray*)pSrc);
+}
 
+static void ctgFreeDbVgroup(void* p) {
+  taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneDbCfgInfo(void* pSrc) {
+  SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(SDbCfgInfo));
+  return pDst;
+}
+
+static void ctgFreeDbCfgInfo(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneDbInfo(void* pSrc) {
+  SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(SDbInfo));
+  return pDst;
+}
+
+static void ctgFreeDbInfo(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneTableMeta(void* pSrc) {
+  STableMeta* pMeta = pSrc;
+  int32_t size = sizeof(STableMeta) + (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) * sizeof(SSchema);
+  STableMeta* pDst = taosMemoryMalloc(size);
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, size);
+  return pDst;
+}
+
+static void ctgFreeTableMeta(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneVgroupInfo(void* pSrc) {
+  SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(SVgroupInfo));
+  return pDst;
+}
+
+static void ctgFreeVgroupInfo(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneTableIndices(void* pSrc) {
+  return taosArrayDup((const SArray*)pSrc);
+}
+
+static void ctgFreeTableIndices(void* p) {
+  taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneFuncInfo(void* pSrc) {
+  SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(SFuncInfo));
+  return pDst;
+}
+
+static void ctgFreeFuncInfo(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneIndexInfo(void* pSrc) {
+  SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(SIndexInfo));
+  return pDst;
+}
+
+static void ctgFreeIndexInfo(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneUserAuth(void* pSrc) {
+  bool* pDst = taosMemoryMalloc(sizeof(bool));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  *pDst = *(bool*)pSrc;
+  return pDst;
+}
+
+static void ctgFreeUserAuth(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneQnodeList(void* pSrc) {
+  return taosArrayDup((const SArray*)pSrc);
+}
+
+static void ctgFreeQnodeList(void* p) {
+  taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneTableCfg(void* pSrc) {
+  STableCfg* pDst = taosMemoryMalloc(sizeof(STableCfg));
+  if (NULL == pDst) {
+    return NULL;
+  }
+  memcpy(pDst, pSrc, sizeof(STableCfg));
+  return pDst;
+}
+
+static void ctgFreeTableCfg(void* p) {
+  taosMemoryFree(((SMetaRes*)p)->pRes);
+}
+
+static void* ctgCloneDnodeList(void* pSrc) {
+  return taosArrayDup((const SArray*)pSrc);
+}
+
+static void ctgFreeDnodeList(void* p) {
+  taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
+}
+
+static int32_t ctgCloneMetaDataArray(SArray* pSrc, FCopy copyFunc, SArray** pDst) {
+  if (NULL == pSrc) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t size = taosArrayGetSize(pSrc);
+  *pDst = taosArrayInit(size, sizeof(SMetaRes));
+  if (NULL == *pDst) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  for (int32_t i = 0; i < size; ++i) {
+    SMetaRes* pRes = taosArrayGet(pSrc, i);
+    SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)};
+    if (NULL == res.pRes) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosArrayPush(*pDst, &res);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+SMetaData* catalogCloneMetaData(SMetaData* pData) {
+  SMetaData* pRes = taosMemoryCalloc(1, sizeof(SMetaData));
+  if (NULL == pRes) {
+    return NULL;
+  }
+
+  int32_t code = ctgCloneMetaDataArray(pData->pDbVgroup, ctgCloneDbVgroup, &pRes->pDbVgroup);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pDbCfg, ctgCloneDbCfgInfo, &pRes->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pDbInfo, ctgCloneDbInfo, &pRes->pDbInfo);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pTableMeta, ctgCloneTableMeta, &pRes->pTableMeta);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pTableHash, ctgCloneVgroupInfo, &pRes->pTableHash);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pTableIndex, ctgCloneTableIndices, &pRes->pTableIndex);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pUdfList, ctgCloneFuncInfo, &pRes->pUdfList);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pIndex, ctgCloneIndexInfo, &pRes->pIndex);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pUser, ctgCloneUserAuth, &pRes->pUser);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pQnodeList, ctgCloneQnodeList, &pRes->pQnodeList);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pTableCfg, ctgCloneTableCfg, &pRes->pTableCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = ctgCloneMetaDataArray(pData->pDnodeList, ctgCloneDnodeList, &pRes->pDnodeList);
+  }
+
+  if (TSDB_CODE_SUCCESS != code) {
+    catalogFreeMetaData(pRes);
+    return NULL;
+  }
+
+  return pRes;
+}
+
+void catalogFreeMetaData(SMetaData* pData) {
+  if (NULL == pData) {
+    return;
+  }
+
+  taosArrayDestroyEx(pData->pDbVgroup, ctgFreeDbVgroup);
+  taosArrayDestroyEx(pData->pDbCfg, ctgFreeDbCfgInfo);
+  taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo);
+  taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta);
+  taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo);
+  taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices);  
+  taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo); 
+  taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo);
+  taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth);
+  taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList);
+  taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg);
+  taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList);
+  taosMemoryFreeClear(pData->pSvrVer);
+  taosMemoryFree(pData);
+}
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 4fb55ed373..4e698fc999 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -266,7 +266,7 @@ static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pT
   return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg);
 }
 
-static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname, bool isStb) {
+static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) {
   CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta));
   if (!isStb) {
     SVgroupInfo vg;
@@ -276,12 +276,12 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName*
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) {
-  return getTableMetaImpl(pCxt, tbNo, name, dbFname, false);
+static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
+  return getTableMetaImpl(pCxt, tbNo, name, false);
 }
 
-static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) {
-  return getTableMetaImpl(pCxt, tbNo, name, dbFname, true);
+static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) {
+  return getTableMetaImpl(pCxt, tbNo, name, true);
 }
 
 static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) {
@@ -1178,7 +1178,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
   tNameGetFullDbName(&sname, dbFName);
   strcpy(pCxt->sTableName, sname.tname);
 
-  CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname, dbFName));
+  CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname));
   if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
     return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
   }
@@ -1385,6 +1385,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
       (*numOfRows)++;
     }
     pCxt->pSql = pRawSql;
+
+    if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) {
+      break;
+    }
   }
 
   if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
@@ -1393,23 +1397,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) {
-  char filePathStr[TSDB_FILENAME_LEN] = {0};
-  if (TK_NK_STRING == filePath.type) {
-    trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
-  } else {
-    strncpy(filePathStr, filePath.z, filePath.n);
-  }
-  TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
-  if (NULL == fp) {
-    return TAOS_SYSTEM_ERROR(errno);
-  }
-
+static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName,
+                                      STableDataBlocks* dataBuf) {
   int32_t maxNumOfRows;
   CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows));
 
   int32_t numOfRows = 0;
-  CHECK_CODE(parseCsvFile(pCxt, fp, dataBuf, maxNumOfRows, &numOfRows));
+  CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows));
 
   SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData);
   if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) {
@@ -1417,12 +1411,38 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa
                                     "too many rows in sql, total number of rows should be less than INT32_MAX");
   }
 
+  if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
+    pCxt->pComCxt->needMultiParse = true;
+    pCxt->pComCxt->csvCxt.tableNo = tableNo;
+    memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName));
+    pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql;
+  }
+
   dataBuf->numOfTables = 1;
   pCxt->totalNum += numOfRows;
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath,
+                                 STableDataBlocks* dataBuf) {
+  char filePathStr[TSDB_FILENAME_LEN] = {0};
+  if (TK_NK_STRING == filePath.type) {
+    trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
+  } else {
+    strncpy(filePathStr, filePath.z, filePath.n);
+  }
+  pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
+  if (NULL == pCxt->pComCxt->csvCxt.fp) {
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf);
+}
+
 static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
+  if (!pCxt->pComCxt->needMultiParse) {
+    taosCloseFile(&pCxt->pComCxt->csvCxt.fp);
+  }
   taosMemoryFreeClear(pCxt->pTableMeta);
   destroyBoundColumnInfo(&pCxt->tags);
   tdDestroySVCreateTbReq(&pCxt->createTblReq);
@@ -1481,7 +1501,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z);
       }
 
-      if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
+      if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) &&
+          !pCxt->pComCxt->needMultiParse) {
         return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
       }
       break;
@@ -1536,7 +1557,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
       NEXT_TOKEN(pCxt->pSql, sToken);
       autoCreateTbl = true;
     } else if (!existedUsing) {
-      CHECK_CODE(getTableMeta(pCxt, tbNum, &name, dbFName));
+      CHECK_CODE(getTableMeta(pCxt, tbNum, &name));
       if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) {
         return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported");
       }
@@ -1577,17 +1598,22 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
       if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) {
         return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
       }
-      CHECK_CODE(parseDataFromFile(pCxt, sToken, dataBuf));
+      CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf));
       pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT;
 
       tbNum++;
-      continue;
+      if (!pCxt->pComCxt->needMultiParse) {
+        continue;
+      } else {
+        parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
+        break;
+      }
     }
 
     return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
   }
 
-  qDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
+  parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
 
   if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
     SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
@@ -1612,6 +1638,26 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
   return buildOutput(pCxt);
 }
 
+static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) {
+  STableDataBlocks* dataBuf = NULL;
+  CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName));
+  CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid),
+                                  TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize,
+                                  pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq));
+  CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf));
+  if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) {
+    CHECK_CODE(parseInsertBody(pCxt));
+    pCxt->pComCxt->needMultiParse = false;
+    return TSDB_CODE_SUCCESS;
+  }
+  parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
+  // merge according to vgId
+  if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
+    CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks));
+  }
+  return buildOutput(pCxt);
+}
+
 // INSERT INTO
 //   tb_name
 //       [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
@@ -1621,7 +1667,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) {
   SInsertParseContext context = {
       .pComCxt = pContext,
-      .pSql = (char*)pContext->pSql,
+      .pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql,
       .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
       .pTableMeta = NULL,
       .createTblReq = {0},
@@ -1691,10 +1737,16 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
 
   context.pOutput->payloadType = PAYLOAD_TYPE_KV;
 
-  int32_t code = skipInsertInto(&context.pSql, &context.msg);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = parseInsertBody(&context);
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (!context.pComCxt->needMultiParse) {
+    code = skipInsertInto(&context.pSql, &context.msg);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = parseInsertBody(&context);
+    }
+  } else {
+    code = parseInsertBodyAgain(&context);
   }
+
   if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
     SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL);
     while (NULL != pTable) {
diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c
index 2fe6ebfb79..6b3d6c77c1 100644
--- a/source/libs/parser/src/parser.c
+++ b/source/libs/parser/src/parser.c
@@ -214,6 +214,12 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
   return code;
 }
 
+void qDestroyParseContext(SParseContext* pCxt) {
+  taosArrayDestroy(pCxt->pTableMetaPos);
+  taosArrayDestroy(pCxt->pTableVgroupPos);
+  taosMemoryFree(pCxt);
+}
+
 void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode((SNode*)pQueryNode); }
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) {
diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp
index fcaa5af05c..a9360f796c 100644
--- a/source/libs/parser/test/mockCatalog.cpp
+++ b/source/libs/parser/test/mockCatalog.cpp
@@ -130,16 +130,16 @@ void generatePerformanceSchema(MockCatalogService* mcs) {
  *          c5         |       column       |       DOUBLE       |    8     |
  */
 void generateTestTables(MockCatalogService* mcs, const std::string& db) {
-  ITableBuilder& builder = mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
-                               .setPrecision(TSDB_TIME_PRECISION_MILLI)
-                               .setVgid(1)
-                               .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
-                               .addColumn("c1", TSDB_DATA_TYPE_INT)
-                               .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
-                               .addColumn("c3", TSDB_DATA_TYPE_BIGINT)
-                               .addColumn("c4", TSDB_DATA_TYPE_DOUBLE)
-                               .addColumn("c5", TSDB_DATA_TYPE_DOUBLE);
-  builder.done();
+  mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
+      .setPrecision(TSDB_TIME_PRECISION_MILLI)
+      .setVgid(1)
+      .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
+      .addColumn("c1", TSDB_DATA_TYPE_INT)
+      .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
+      .addColumn("c3", TSDB_DATA_TYPE_BIGINT)
+      .addColumn("c4", TSDB_DATA_TYPE_DOUBLE)
+      .addColumn("c5", TSDB_DATA_TYPE_DOUBLE)
+      .done();
 }
 
 /*
diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp
index 14c991917b..bf27fd2e13 100644
--- a/source/libs/parser/test/parTestUtil.cpp
+++ b/source/libs/parser/test/parTestUtil.cpp
@@ -343,7 +343,6 @@ class ParserTestBaseImpl {
 
       unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
       doParseSql(&cxt, query.get());
-      SQuery* pQuery = *(query.get());
 
       if (g_dump) {
         dump();

From 936c7b6276cec5e516a4a756f55be39a938349f6 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 33/44] fix: coverity issues

CID: 400042
---
 source/libs/scalar/src/scalar.c    |  6 ++-
 source/libs/scalar/src/sclvector.c | 82 +++++++++++++++---------------
 2 files changed, 46 insertions(+), 42 deletions(-)

diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 9cba94d85a..b4b99be636 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -363,7 +363,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
       }
 
       SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
-      if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
+      if (NULL == block) {
+        SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+      }
+
+      if (ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
         sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
         SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index fe2a970aaa..781155e40c 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -281,7 +281,7 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r
   if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
     value = 0;
   }
-  
+
   colDataAppendInt64(pOut->columnData, rowIndex, &value);
 }
 
@@ -300,29 +300,29 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI
       *overflow = 0;
     }
   }
-      
+
   switch (pOut->columnData->info.type) {
     case TSDB_DATA_TYPE_TINYINT: {
       int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10);
-     
+
       colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_SMALLINT: {
       int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10);
       colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_INT: {
       int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10);
       colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_BIGINT: {
       int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10);
       colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
       break;
-    }   
+    }
   }
 }
 
@@ -347,22 +347,22 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro
       uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10);
       colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_USMALLINT: {
       uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10);
       colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_UINT: {
       uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10);
       colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
       break;
-    } 
+    }
     case TSDB_DATA_TYPE_UBIGINT: {
       uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10);
       colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
       break;
-    }   
+    }
   }
 }
 
@@ -372,7 +372,7 @@ static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIn
     colDataAppendFloat(pOut->columnData, rowIndex, &value);
     return;
   }
-  
+
   double value = taosStr2Double(buf, NULL);
   colDataAppendDouble(pOut->columnData, rowIndex, &value);
 }
@@ -488,11 +488,11 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
         tmp[len] = 0;
       }
     }
-    
+
     (*func)(tmp, pOut, i, overflow);
     taosMemoryFreeClear(tmp);
   }
-  
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -636,7 +636,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
-      
+
       int64_t value = 0;
       GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i));
       int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value);
@@ -653,7 +653,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
-      
+
       uint64_t value = 0;
       GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i));
       int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value);
@@ -670,7 +670,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1
         colDataAppendNULL(pOutputCol, i);
         continue;
       }
-      
+
       double value = 0;
       GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i));
       int32_t len = sprintf(varDataVal(tmp), "%lf", value);
@@ -698,7 +698,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
     sclError("input column is NULL, hashFilter %p", pIn->pHashFilter);
     return TSDB_CODE_APP_ERROR;
   }
-  
+
   int16_t inType   = pInputCol->info.type;
   int16_t outType  = pOutputCol->info.type;
 
@@ -710,14 +710,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
     ASSERT(1 == pIn->numOfRows);
 
     pOut->numOfRows = 0;
-    
+
     if (IS_SIGNED_NUMERIC_TYPE(outType)) {
       int64_t minValue = tDataTypes[outType].minValue;
       int64_t maxValue = tDataTypes[outType].maxValue;
-      
+
       double value = 0;
       GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0));
-      
+
       if (value > maxValue) {
         *overflow = 1;
         return TSDB_CODE_SUCCESS;
@@ -730,10 +730,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
     } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) {
       uint64_t minValue = (uint64_t)tDataTypes[outType].minValue;
       uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue;
-      
+
       double value = 0;
       GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0));
-      
+
       if (value > maxValue) {
         *overflow = 1;
         return TSDB_CODE_SUCCESS;
@@ -820,7 +820,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         uint8_t value = 0;
         GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i));
         colDataAppendInt8(pOutputCol, i, (int8_t *)&value);
@@ -833,7 +833,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         uint16_t value = 0;
         GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i));
         colDataAppendInt16(pOutputCol, i, (int16_t *)&value);
@@ -846,7 +846,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         uint32_t value = 0;
         GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i));
         colDataAppendInt32(pOutputCol, i, (int32_t *)&value);
@@ -859,7 +859,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         uint64_t value = 0;
         GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i));
         colDataAppendInt64(pOutputCol, i, (int64_t*)&value);
@@ -872,12 +872,12 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         float value = 0;
         GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i));
         colDataAppendFloat(pOutputCol, i, (float*)&value);
       }
-      break;  
+      break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
       for (int32_t i = 0; i < pIn->numOfRows; ++i) {
@@ -885,14 +885,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t*
           colDataAppendNULL(pOutputCol, i);
           continue;
         }
-        
+
         double value = 0;
         GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i));
         colDataAppendDouble(pOutputCol, i, (double*)&value);
       }
-      break;  
+      break;
     }
-    case TSDB_DATA_TYPE_BINARY: 
+    case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR: {
       return vectorConvertToVarData(pIn, pOut, inType, outType);
     }
@@ -966,10 +966,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
     return TSDB_CODE_SUCCESS;
   }
 
-  SScalarParam *param1 = NULL, *paramOut1 = NULL; 
+  SScalarParam *param1 = NULL, *paramOut1 = NULL;
   SScalarParam *param2 = NULL, *paramOut2 = NULL;
   int32_t code = 0;
-  
+
   if (leftType < rightType) {
     param1 = pLeft;
     param2 = pRight;
@@ -993,7 +993,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
       return code;
     }
   }
-  
+
   if (type != GET_PARAM_TYPE(param2)) {
     code = vectorConvertScalarParam(param2, paramOut2, type);
     if (code) {
@@ -1136,7 +1136,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
         }
         *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i);
       }
-    } 
+    }
   } else {
     double *output = (double *)pOutputCol->pData;
     _getDoubleValue_fn_t getVectorDoubleValueFnLeft  = getVectorDoubleValueFn(pLeftCol->info.type);
@@ -1198,7 +1198,7 @@ static void vectorMathTsSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pR
       }
       *output = taosTimeAdd(getVectorBigintValueFnLeft(pLeftCol->pData, i), -getVectorBigintValueFnRight(pRightCol->pData, 0),
                             pRightCol->info.scale, pRightCol->info.precision);
-      
+
     }
   }
 }
@@ -1706,7 +1706,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   int32_t       lType = GET_PARAM_TYPE(pLeft);
   int32_t       rType = GET_PARAM_TYPE(pRight);
   __compar_fn_t fp = NULL;
-  
+
   if (lType == rType) {
     fp = filterGetCompFunc(lType, optr);
   } else {
@@ -1736,9 +1736,9 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 }
 
 void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) {
-  SScalarParam pLeftOut = {0}; 
+  SScalarParam pLeftOut = {0};
   SScalarParam pRightOut = {0};
-  SScalarParam *param1 = NULL; 
+  SScalarParam *param1 = NULL;
   SScalarParam *param2 = NULL;
 
   if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
@@ -1762,7 +1762,7 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
   vectorCompareImpl(param1, param2, pOut, _ord, optr);
   sclFreeParam(&pLeftOut);
-  sclFreeParam(&pRightOut);  
+  sclFreeParam(&pRightOut);
 }
 
 void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {

From dbae9a47da57706ed977ab344856955070bbf610 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 11 Oct 2022 16:23:23 +0800
Subject: [PATCH 34/44] fix: check memory while alter db buffer

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 57 ++++++++------
 source/dnode/mnode/impl/inc/mndVgroup.h     |  3 +-
 source/dnode/mnode/impl/src/mndDb.c         |  6 +-
 source/dnode/mnode/impl/src/mndVgroup.c     | 87 +++++++++++++++------
 source/dnode/vnode/src/vnd/vnodeSvr.c       | 56 +++++++------
 5 files changed, 130 insertions(+), 79 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 4047bc2340..7d5a907fda 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -212,38 +212,47 @@ static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) {
 }
 
 int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
-  SCreateVnodeReq createReq = {0};
+  SCreateVnodeReq req = {0};
   SVnodeCfg       vnodeCfg = {0};
   SWrapperCfg     wrapperCfg = {0};
   int32_t         code = -1;
   char            path[TSDB_FILENAME_LEN] = {0};
 
-  if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
+  if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
 
   dInfo(
-      "vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d "
-      "tsdbPageSize:%d",
-      createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
-      createReq.sstTrigger, createReq.tsdbPageSize);
-  dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod,
-        createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
-  vmGenerateVnodeCfg(&createReq, &vnodeCfg);
+      "vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
+      " cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
+      "days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d, wal "
+      "fsync:%d level:%d retentionPeriod:%d retentionSize:%d rollPeriod:%d segSize:%d, hash method:%d begin:%u end:%u "
+      "prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d",
+      req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
+      req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid,
+      req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression,
+      req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
+      req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
+      req.replica, req.selfIndex, req.strict);
+  for (int32_t i = 0; i < req.replica; ++i) {
+    dInfo("vgId:%d, replica:%d fqdn:%s port:%u", req.vgId, req.replicas[i].id, req.replicas[i].fqdn,
+          req.replicas[i].port);
+  }
+  vmGenerateVnodeCfg(&req, &vnodeCfg);
 
-  if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {
-    dError("vgId:%d, failed to adjust tsma days since %s", createReq.vgId, terrstr());
+  if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
+    dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
     code = terrno;
     goto _OVER;
   }
 
-  vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
+  vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg);
 
-  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
   if (pVnode != NULL) {
-    dDebug("vgId:%d, already exist", createReq.vgId);
-    tFreeSCreateVnodeReq(&createReq);
+    dDebug("vgId:%d, already exist", req.vgId);
+    tFreeSCreateVnodeReq(&req);
     vmReleaseVnode(pMgmt, pVnode);
     terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
     code = terrno;
@@ -252,36 +261,36 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId);
   if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) {
-    tFreeSCreateVnodeReq(&createReq);
-    dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
+    tFreeSCreateVnodeReq(&req);
+    dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr());
     code = terrno;
     goto _OVER;
   }
 
   SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
   if (pImpl == NULL) {
-    dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
+    dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
     code = terrno;
     goto _OVER;
   }
 
   code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
   if (code != 0) {
-    dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
+    dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
     code = terrno;
     goto _OVER;
   }
 
-  code = vmTsmaProcessCreate(pImpl, &createReq);
+  code = vmTsmaProcessCreate(pImpl, &req);
   if (code != 0) {
-    dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr());
+    dError("vgId:%d, failed to create tsma since %s", req.vgId, terrstr());
     code = terrno;
     goto _OVER;
   }
 
   code = vnodeStart(pImpl);
   if (code != 0) {
-    dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
+    dError("vgId:%d, failed to start sync since %s", req.vgId, terrstr());
     goto _OVER;
   }
 
@@ -296,10 +305,10 @@ _OVER:
     vnodeClose(pImpl);
     vnodeDestroy(path, pMgmt->pTfs);
   } else {
-    dInfo("vgId:%d, vnode is created", createReq.vgId);
+    dInfo("vgId:%d, vnode is created", req.vgId);
   }
 
-  tFreeSCreateVnodeReq(&createReq);
+  tFreeSCreateVnodeReq(&req);
   terrno = code;
   return code;
 }
diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h
index c8237d17d8..ad5fbef34c 100644
--- a/source/dnode/mnode/impl/inc/mndVgroup.h
+++ b/source/dnode/mnode/impl/inc/mndVgroup.h
@@ -44,7 +44,8 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV
 int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
 int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray);
 int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId);
-int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray);
+int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
+                                  SArray *pArray);
 
 void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby);
 void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index adc5ff2b42..6670211d78 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -746,7 +746,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
+static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
   SArray *pArray = mndBuildDnodesArray(pMnode, 0);
@@ -756,8 +756,8 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
 
-    if (mndVgroupInDb(pVgroup, pNew->uid)) {
-      if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) {
+    if (mndVgroupInDb(pVgroup, pNewDb->uid)) {
+      if (mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray) != 0) {
         sdbCancelFetch(pSdb, pIter);
         sdbRelease(pSdb, pVgroup);
         taosArrayDestroy(pArray);
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index 4a7221accb..8e91ec90f5 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -405,7 +405,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
   pDnode->memUsed = mndGetVnodesMemory(pMnode, pDnode->id);
 
   mInfo("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id,
-         pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed);
+        pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed);
 
   if (isMnode) {
     pDnode->numOfVnodes++;
@@ -1293,7 +1293,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
       mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
              pVgroup->dbName, pVgroup->vgId, vgMem, pNew3->id, pNew3->memAvail, pNew3->memUsed);
       terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
-     goto _OVER;
+      goto _OVER;
     } else {
       pNew3->memUsed += vgMem;
     }
@@ -1530,44 +1530,81 @@ _OVER:
 #endif
 }
 
-int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
-  if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
-    return mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG);
+static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup,
+                                   SVgObj *pNewVgroup, SArray *pArray) {
+  for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, i);
+    bool       inVgroup = false;
+    for (int32_t j = 0; j < pOldVgroup->replica; ++j) {
+      SVnodeGid *pVgId = &pOldVgroup->vnodeGid[i];
+      if (pDnode->id == pVgId->dnodeId) {
+        pDnode->memUsed -= mndGetVgroupMemory(pMnode, pOldDb, pOldVgroup);
+        inVgroup = true;
+      }
+    }
+    for (int32_t j = 0; j < pNewVgroup->replica; ++j) {
+      SVnodeGid *pVgId = &pNewVgroup->vnodeGid[i];
+      if (pDnode->id == pVgId->dnodeId) {
+        pDnode->memUsed += mndGetVgroupMemory(pMnode, pNewDb, pNewVgroup);
+        inVgroup = true;
+      }
+    }
+    if (pDnode->memAvail - pDnode->memUsed <= 0) {
+      mError("db:%s, vgId:%d, no enough memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName,
+             pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed);
+      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
+      return -1;
+    } else if (inVgroup) {
+      mInfo("db:%s, vgId:%d, memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName,
+             pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed);
+    } else {
+    }
   }
+  return 0;
+}
 
+int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
+                                  SArray *pArray) {
   SVgObj newVgroup = {0};
   memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
+
+  if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) {
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) return -1;
+    if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1;
+    return 0;
+  }
+
   mndTransSetSerial(pTrans);
 
-  if (newVgroup.replica < pDb->cfg.replications) {
+  if (newVgroup.replica < pNewDb->cfg.replications) {
     mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
           pVgroup->vnodeGid[0].dnodeId);
 
     if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
 
     if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-  } else if (newVgroup.replica > pDb->cfg.replications) {
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
+  } else if (newVgroup.replica > pNewDb->cfg.replications) {
     mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
 
     SVnodeGid del1 = {0};
     if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
-    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1;
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
-    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del1, true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
 
     SVnodeGid del2 = {0};
     if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
-    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1;
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
-    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+    if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del2, true) != 0) return -1;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
   } else {
   }
 
@@ -1648,8 +1685,8 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
   if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER;
 
   // adjust vgroup
-  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER;
-  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER;
+  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
+  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
 
 _OVER:
   mndTransDrop(pTrans);
@@ -1782,7 +1819,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
     for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
       SDnodeObj *pDnode = taosArrayGet(pArray, i);
       mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
-             pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
+            pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
     }
 
     SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
@@ -1791,7 +1828,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
     float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
     float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
     mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore,
-           pDst->id, dstScore);
+          pDst->id, dstScore);
 
     if (srcScore > dstScore - 0.000001) {
       code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups);
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 5c8590c7c9..295e480c6a 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -1024,71 +1024,75 @@ static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, vo
 }
 
 static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  SAlterVnodeReq alterReq = {0};
+  SAlterVnodeReq req = {0};
   bool           walChanged = false;
   bool           tsdbChanged = false;
 
-  if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) {
+  if (tDeserializeSAlterVnodeReq(pReq, len, &req) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return TSDB_CODE_INVALID_MSG;
   }
 
-  vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast,
-        alterReq.cacheLastSize);
-  if (pVnode->config.cacheLastSize != alterReq.cacheLastSize) {
-    pVnode->config.cacheLastSize = alterReq.cacheLastSize;
+  vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
+        " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d strict:%d",
+        TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
+        req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
+        req.walFsyncPeriod, req.walLevel, req.strict);
+
+  if (pVnode->config.cacheLastSize != req.cacheLastSize) {
+    pVnode->config.cacheLastSize = req.cacheLastSize;
     tsdbCacheSetCapacity(pVnode, (size_t)pVnode->config.cacheLastSize * 1024 * 1024);
   }
 
-  if (pVnode->config.szBuf != alterReq.buffer * 1024LL * 1024LL) {
+  if (pVnode->config.szBuf != req.buffer * 1024LL * 1024LL) {
     vInfo("vgId:%d vnode buffer is changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pVnode->config.szBuf,
-          alterReq.buffer * 1024LL * 1024LL);
-    pVnode->config.szBuf = alterReq.buffer * 1024LL * 1024LL;
+          req.buffer * 1024LL * 1024LL);
+    pVnode->config.szBuf = req.buffer * 1024LL * 1024LL;
   }
 
-  if (pVnode->config.szCache != alterReq.pages) {
-    if (metaAlterCache(pVnode->pMeta, alterReq.pages) < 0) {
+  if (pVnode->config.szCache != req.pages) {
+    if (metaAlterCache(pVnode->pMeta, req.pages) < 0) {
       vError("vgId:%d failed to change vnode pages from %d to %d failed since %s", TD_VID(pVnode),
-             pVnode->config.szCache, alterReq.pages, tstrerror(errno));
+             pVnode->config.szCache, req.pages, tstrerror(errno));
       return errno;
     } else {
-      vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, alterReq.pages);
-      pVnode->config.szCache = alterReq.pages;
+      vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, req.pages);
+      pVnode->config.szCache = req.pages;
     }
   }
 
-  if (pVnode->config.cacheLast != alterReq.cacheLast) {
-    pVnode->config.cacheLast = alterReq.cacheLast;
+  if (pVnode->config.cacheLast != req.cacheLast) {
+    pVnode->config.cacheLast = req.cacheLast;
   }
 
-  if (pVnode->config.walCfg.fsyncPeriod != alterReq.walFsyncPeriod) {
-    pVnode->config.walCfg.fsyncPeriod = alterReq.walFsyncPeriod;
+  if (pVnode->config.walCfg.fsyncPeriod != req.walFsyncPeriod) {
+    pVnode->config.walCfg.fsyncPeriod = req.walFsyncPeriod;
 
     walChanged = true;
   }
 
-  if (pVnode->config.walCfg.level != alterReq.walLevel) {
-    pVnode->config.walCfg.level = alterReq.walLevel;
+  if (pVnode->config.walCfg.level != req.walLevel) {
+    pVnode->config.walCfg.level = req.walLevel;
 
     walChanged = true;
   }
 
-  if (pVnode->config.tsdbCfg.keep0 != alterReq.daysToKeep0) {
-    pVnode->config.tsdbCfg.keep0 = alterReq.daysToKeep0;
+  if (pVnode->config.tsdbCfg.keep0 != req.daysToKeep0) {
+    pVnode->config.tsdbCfg.keep0 = req.daysToKeep0;
     if (!VND_IS_RSMA(pVnode)) {
       tsdbChanged = true;
     }
   }
 
-  if (pVnode->config.tsdbCfg.keep1 != alterReq.daysToKeep1) {
-    pVnode->config.tsdbCfg.keep1 = alterReq.daysToKeep1;
+  if (pVnode->config.tsdbCfg.keep1 != req.daysToKeep1) {
+    pVnode->config.tsdbCfg.keep1 = req.daysToKeep1;
     if (!VND_IS_RSMA(pVnode)) {
       tsdbChanged = true;
     }
   }
 
-  if (pVnode->config.tsdbCfg.keep2 != alterReq.daysToKeep2) {
-    pVnode->config.tsdbCfg.keep2 = alterReq.daysToKeep2;
+  if (pVnode->config.tsdbCfg.keep2 != req.daysToKeep2) {
+    pVnode->config.tsdbCfg.keep2 = req.daysToKeep2;
     if (!VND_IS_RSMA(pVnode)) {
       tsdbChanged = true;
     }

From 32a36b66f123b4d31c833a4c05b5f5ee6e3a4c43 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Tue, 11 Oct 2022 16:57:47 +0800
Subject: [PATCH 35/44] fix(shell): remove TAB KEY usage introduction on
 windows

---
 tools/shell/src/shellEngine.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index abb6cf84d5..fd0ea60323 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -1112,7 +1112,9 @@ int32_t shellExecute() {
 #ifdef WEBSOCKET
   if (!shell.args.restful && !shell.args.cloud) {
 #endif
+#ifndef WINDOWS
   printfIntroduction();
+#endif  
 	shellGetGrantInfo();
 #ifdef WEBSOCKET
   }

From e5c3ff715828ccfcb10c2bb0ad989b7459275be7 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 11 Oct 2022 11:29:18 +0800
Subject: [PATCH 36/44] fix: coverity issues

CID: 399950
---
 source/common/src/tdataformat.c    |  4 +--
 source/libs/scalar/src/filter.c    |  2 +-
 source/libs/scalar/src/sclvector.c | 54 +++++++++++++++---------------
 3 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 2a40859400..ad5772e0fe 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1678,8 +1678,8 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
   int32_t size;
 
   ASSERT(pColDataSrc->nVal > 0);
-  ASSERT(pColDataDest->cid = pColDataSrc->cid);
-  ASSERT(pColDataDest->type = pColDataSrc->type);
+  ASSERT(pColDataDest->cid == pColDataSrc->cid);
+  ASSERT(pColDataDest->type == pColDataSrc->type);
 
   pColDataDest->smaOn = pColDataSrc->smaOn;
   pColDataDest->nVal = pColDataSrc->nVal;
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index ac063cb50d..dbb5214415 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -1939,7 +1939,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
       bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
 
       fi->data = taosMemoryCalloc(1, bytes);
-    } else{
+    } else {
       if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) {  //TIME RANGE
 /*
         fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 781155e40c..339b18bc06 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -529,24 +529,24 @@ void* ncharTobinary(void *buf){            // todo need to remove , if tobinary
 
 bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData,
                       void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){
-  if(optr == OP_TYPE_JSON_CONTAINS) {
+  if (optr == OP_TYPE_JSON_CONTAINS) {
     return true;
   }
 
-  if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
+  if (typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON) {
     return true;
   }
 
-  if(typeLeft == TSDB_DATA_TYPE_JSON){
-    if(tTagIsJson(*pLeftData)){
+  if (typeLeft == TSDB_DATA_TYPE_JSON) {
+    if (tTagIsJson(*pLeftData)) {
       terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
       return false;
     }
     typeLeft = **pLeftData;
     (*pLeftData) ++;
   }
-  if(typeRight == TSDB_DATA_TYPE_JSON){
-    if(tTagIsJson(*pLeftData)){
+  if (typeRight == TSDB_DATA_TYPE_JSON) {
+    if (tTagIsJson(*pLeftData)) {
       terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
       return false;
     }
@@ -554,71 +554,71 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     (*pRightData) ++;
   }
 
-  if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){
-    if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){
+  if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) {
+    if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY) {
       return false;
     }
   }
 
   // if types can not comparable
-  if((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) ||
-     (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) ||
-     (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) ||
-     (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) ||
-     ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) ||
-     ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL)))
+  if ((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) ||
+      (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) ||
+      (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) ||
+      (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) ||
+      ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) ||
+      ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL)))
     return false;
 
 
-  if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
+  if (typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL) {
     *isNull = true;
     return true;
   }
   int8_t type = vectorGetConvertType(typeLeft, typeRight);
 
-  if(type == 0) {
+  if (type == 0) {
     *fp = filterGetCompFunc(typeLeft, optr);
     return true;
   }
 
   *fp = filterGetCompFunc(type, optr);
 
-  if(IS_NUMERIC_TYPE(type)){
-    if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
+  if (IS_NUMERIC_TYPE(type)) {
+    if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
       ASSERT(0);
 //      convertNcharToDouble(*pLeftData, pLeftOut);
 //      *pLeftData = pLeftOut;
-    } else if(typeLeft == TSDB_DATA_TYPE_BINARY) {
+    } else if (typeLeft == TSDB_DATA_TYPE_BINARY) {
       ASSERT(0);
 //      convertBinaryToDouble(*pLeftData, pLeftOut);
 //      *pLeftData = pLeftOut;
-    } else if(typeLeft != type) {
+    } else if (typeLeft != type) {
       convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
       *pLeftData = pLeftOut;
     }
 
-    if(typeRight == TSDB_DATA_TYPE_NCHAR) {
+    if (typeRight == TSDB_DATA_TYPE_NCHAR) {
       ASSERT(0);
 //      convertNcharToDouble(*pRightData, pRightOut);
 //      *pRightData = pRightOut;
-    } else if(typeRight == TSDB_DATA_TYPE_BINARY) {
+    } else if (typeRight == TSDB_DATA_TYPE_BINARY) {
       ASSERT(0);
 //      convertBinaryToDouble(*pRightData, pRightOut);
 //      *pRightData = pRightOut;
-    } else if(typeRight != type) {
+    } else if (typeRight != type) {
       convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
       *pRightData = pRightOut;
     }
-  }else if(type == TSDB_DATA_TYPE_BINARY){
-    if(typeLeft == TSDB_DATA_TYPE_NCHAR){
+  } else if (type == TSDB_DATA_TYPE_BINARY) {
+    if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
       *pLeftData = ncharTobinary(*pLeftData);
       *freeLeft = true;
     }
-    if(typeRight == TSDB_DATA_TYPE_NCHAR){
+    if (typeRight == TSDB_DATA_TYPE_NCHAR) {
       *pRightData = ncharTobinary(*pRightData);
       *freeRight = true;
     }
-  }else{
+  } else {
     ASSERT(0);
   }
 

From 0563eb24757ae6ed4d49034da19ef0e248712147 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 11 Oct 2022 17:28:58 +0800
Subject: [PATCH 37/44] feat: support batch loading of csv files

---
 source/client/src/clientImpl.c  |   2 +-
 source/client/src/clientSml.c   | 245 +++++++++++++++++---------------
 source/libs/parser/src/parser.c |   4 +
 3 files changed, 134 insertions(+), 117 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 08e844dc2c..58bfcc8a09 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -922,7 +922,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
     pRequest->code = code1;
   }
 
-  if (pRequest->code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) {
+  if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) {
     code = continueInsertFromCsv(pWrapper, pRequest);
     if (TSDB_CODE_SUCCESS == code) {
       return;
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 4eb1a3712c..c5633e73d0 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -209,7 +209,7 @@ static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashOb
 }
 
 static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
-  if(pBuf->buf){
+  if (pBuf->buf) {
     memset(pBuf->buf, 0, pBuf->len);
     if (msg1) strncat(pBuf->buf, msg1, pBuf->len);
     int32_t left = pBuf->len - strlen(pBuf->buf);
@@ -256,15 +256,15 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
   while (result <= length) {
     result *= 2;
   }
-  if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
+  if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
     result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
-  } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+  } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
     result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
   }
 
-  if (type == TSDB_DATA_TYPE_NCHAR){
+  if (type == TSDB_DATA_TYPE_NCHAR) {
     result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
-  }else if (type == TSDB_DATA_TYPE_BINARY){
+  } else if (type == TSDB_DATA_TYPE_BINARY) {
     result = result + VARSTR_HEADER_SIZE;
   }
   return result;
@@ -274,7 +274,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
                                       ESchemaAction *action, bool isTag) {
   int32_t code = TSDB_CODE_SUCCESS;
   for (int j = 0; j < taosArrayGetSize(cols); ++j) {
-    if(j == 0 && !isTag) continue;
+    if (j == 0 && !isTag) continue;
     SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
     code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info);
     if (code != TSDB_CODE_SUCCESS) {
@@ -286,12 +286,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
 
 static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
   SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-  int32_t i = 0;
-  for ( ;i < length; i++) {
+  int32_t   i = 0;
+  for (; i < length; i++) {
     taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
   }
 
-  if (isTag){
+  if (isTag) {
     i = 0;
   } else {
     i = 1;
@@ -306,7 +306,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
   return 0;
 }
 
-static int32_t getBytes(uint8_t type, int32_t length){
+static int32_t getBytes(uint8_t type, int32_t length) {
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
     return smlFindNearestPowerOf2(length, type);
   } else {
@@ -314,21 +314,22 @@ static int32_t getBytes(uint8_t type, int32_t length){
   }
 }
 
-static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray* results, int32_t numOfCols, bool isTag) {
+static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
+                                  SArray *results, int32_t numOfCols, bool isTag) {
   for (int j = 0; j < taosArrayGetSize(cols); ++j) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
+    SSmlKv       *kv = (SSmlKv *)taosArrayGetP(cols, j);
     ESchemaAction action = SCHEMA_ACTION_NULL;
     smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
-    if(action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG){
+    if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
       SField field = {0};
       field.type = kv->type;
       field.bytes = getBytes(kv->type, kv->length);
       memcpy(field.name, kv->key, kv->keyLen);
       taosArrayPush(results, &field);
-    }else if(action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){
+    } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
       uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
-      uint16_t newIndex = *index;
-      if(isTag) newIndex -= numOfCols;
+      uint16_t  newIndex = *index;
+      if (isTag) newIndex -= numOfCols;
       SField *field = (SField *)taosArrayGet(results, newIndex);
       field->bytes = getBytes(kv->type, kv->length);
     }
@@ -336,12 +337,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-//static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                              int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
-static int32_t  smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, SArray* pTags,
-                               STableMeta *pTableMeta, ESchemaAction action){
-
-  SRequestObj*   pRequest = NULL;
+// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
+//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
+static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
+                              ESchemaAction action) {
+  SRequestObj   *pRequest = NULL;
   SMCreateStbReq pReq = {0};
   int32_t        code = TSDB_CODE_SUCCESS;
   SCmdMsgInfo    pCmdMsg = {0};
@@ -363,24 +363,24 @@ static int32_t  smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
     goto end;
   }
 
-  if (action == SCHEMA_ACTION_CREATE_STABLE){
+  if (action == SCHEMA_ACTION_CREATE_STABLE) {
     pReq.colVer = 1;
     pReq.tagVer = 1;
     pReq.suid = 0;
     pReq.source = TD_REQ_FROM_APP;
-  } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){
+  } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
     pReq.colVer = pTableMeta->sversion;
     pReq.tagVer = pTableMeta->tversion + 1;
     pReq.suid = pTableMeta->uid;
     pReq.source = TD_REQ_FROM_TAOX;
-  } else if (action == SCHEMA_ACTION_ADD_COLUMN  || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE){
+  } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) {
     pReq.colVer = pTableMeta->sversion + 1;
     pReq.tagVer = pTableMeta->tversion;
     pReq.suid = pTableMeta->uid;
     pReq.source = TD_REQ_FROM_TAOX;
   }
 
-  if (pReq.numOfTags == 0){
+  if (pReq.numOfTags == 0) {
     pReq.numOfTags = 1;
     SField field = {0};
     field.type = TSDB_DATA_TYPE_NCHAR;
@@ -412,7 +412,7 @@ static int32_t  smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
 
   launchQueryImpl(pRequest, &pQuery, true, NULL);
 
-  if(pRequest->code == TSDB_CODE_SUCCESS){
+  if (pRequest->code == TSDB_CODE_SUCCESS) {
     catalogRemoveTableMeta(info->pCatalog, pName);
   }
   code = pRequest->code;
@@ -425,11 +425,11 @@ end:
 }
 
 static int32_t smlModifyDBSchemas(SSmlHandle *info) {
-  int32_t         code = 0;
-  SHashObj       *hashTmp = NULL;
-  STableMeta     *pTableMeta = NULL;
+  int32_t     code = 0;
+  SHashObj   *hashTmp = NULL;
+  STableMeta *pTableMeta = NULL;
 
-  SName   pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
+  SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
   strcpy(pName.dbname, info->pRequest->pDb);
 
   SRequestConnInfo conn = {0};
@@ -451,8 +451,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
     code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
 
     if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
-      SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
-      SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
+      SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
+      SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
       smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true);
       smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false);
 
@@ -463,8 +463,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       }
       info->cost.numOfCreateSTables++;
     } else if (code == TSDB_CODE_SUCCESS) {
-      hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags,
-                                       taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+      hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
+                             HASH_NO_LOCK);
       for (uint16_t i = pTableMeta->tableInfo.numOfColumns;
            i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
         taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
@@ -475,22 +475,25 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       if (code != TSDB_CODE_SUCCESS) {
         goto end;
       }
-      if (action != SCHEMA_ACTION_NULL){
-        SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
-        SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
+      if (action != SCHEMA_ACTION_NULL) {
+        SArray *pColumns =
+            taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
+        SArray *pTags =
+            taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
 
         for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
           SField field = {0};
           field.type = pTableMeta->schema[i].type;
           field.bytes = pTableMeta->schema[i].bytes;
           strcpy(field.name, pTableMeta->schema[i].name);
-          if(i < pTableMeta->tableInfo.numOfColumns){
+          if (i < pTableMeta->tableInfo.numOfColumns) {
             taosArrayPush(pColumns, &field);
-          }else{
+          } else {
             taosArrayPush(pTags, &field);
           }
         }
-        smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true);
+        smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
+                           pTableMeta->tableInfo.numOfColumns, true);
 
         code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
         if (code != TSDB_CODE_SUCCESS) {
@@ -518,23 +521,26 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       if (code != TSDB_CODE_SUCCESS) {
         goto end;
       }
-      if (action != SCHEMA_ACTION_NULL){
-        SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
-        SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
+      if (action != SCHEMA_ACTION_NULL) {
+        SArray *pColumns =
+            taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
+        SArray *pTags =
+            taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
 
         for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
           SField field = {0};
           field.type = pTableMeta->schema[i].type;
           field.bytes = pTableMeta->schema[i].bytes;
           strcpy(field.name, pTableMeta->schema[i].name);
-          if(i < pTableMeta->tableInfo.numOfColumns){
+          if (i < pTableMeta->tableInfo.numOfColumns) {
             taosArrayPush(pColumns, &field);
-          }else{
+          } else {
             taosArrayPush(pTags, &field);
           }
         }
 
-        smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, pTableMeta->tableInfo.numOfColumns, false);
+        smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
+                           pTableMeta->tableInfo.numOfColumns, false);
 
         code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
         if (code != TSDB_CODE_SUCCESS) {
@@ -847,7 +853,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t
 static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) {
   int64_t ts = 0;
   if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
-//    uError("SML:data:%s,len:%d", data, len);
+    //    uError("SML:data:%s,len:%d", data, len);
     ts = smlParseInfluxTime(info, data, len);
   } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
     ts = smlParseOpenTsdbTime(info, data, len);
@@ -877,7 +883,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   if (smlIsBinary(pVal->value, pVal->length)) {
     pVal->type = TSDB_DATA_TYPE_BINARY;
     pVal->length -= BINARY_ADD_LEN;
-    if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
+    if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
       return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
     }
     pVal->value += (BINARY_ADD_LEN - 1);
@@ -887,7 +893,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   if (smlIsNchar(pVal->value, pVal->length)) {
     pVal->type = TSDB_DATA_TYPE_NCHAR;
     pVal->length -= NCHAR_ADD_LEN;
-    if(pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+    if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
       return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
     }
     pVal->value += (NCHAR_ADD_LEN - 1);
@@ -1063,7 +1069,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab
       continue;
     }
 
-    if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+    if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
       return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
     }
 
@@ -1224,7 +1230,7 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c
     kv->value = value;
     kv->length = valueLen;
     if (isTag) {
-      if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
         return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
       }
       kv->type = TSDB_DATA_TYPE_NCHAR;
@@ -1362,8 +1368,8 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) {
 static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
   SHashObj *s1 = *(SHashObj **)key1;
   SHashObj *s2 = *(SHashObj **)key2;
-  SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
-  SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
+  SSmlKv   *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
+  SSmlKv   *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
   ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP);
   ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP);
   if (kv1->i < kv2->i) {
@@ -1375,12 +1381,12 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
   }
 }
 
-static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){
-  if(dataFormat){
+static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) {
+  if (dataFormat) {
     void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT);
-    if(p == NULL){
+    if (p == NULL) {
       taosArrayPush(oneTable->cols, &cols);
-    }else{
+    } else {
       taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols);
     }
     return TSDB_CODE_SUCCESS;
@@ -1397,9 +1403,9 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
   }
 
   void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT);
-  if(p == NULL){
+  if (p == NULL) {
     taosArrayPush(oneTable->cols, &kvHash);
-  }else{
+  } else {
     taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash);
   }
   return TSDB_CODE_SUCCESS;
@@ -1488,15 +1494,15 @@ static void smlDestroyInfo(SSmlHandle *info) {
   taosMemoryFreeClear(info);
 }
 
-static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLProtocolType protocol, int8_t precision){
-  int32_t code = TSDB_CODE_SUCCESS;
-  SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle));
+static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) {
+  int32_t     code = TSDB_CODE_SUCCESS;
+  SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle));
   if (NULL == info) {
     return NULL;
   }
   info->id = smlGenId();
 
-  info->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
   if (NULL == info->pQuery) {
     uError("SML:0x%" PRIx64 " create info->pQuery error", info->id);
     goto cleanup;
@@ -1511,8 +1517,8 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
   }
   ((SVnodeModifOpStmt *)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
 
-  if (pTscObj){
-    info->taos        = pTscObj;
+  if (pTscObj) {
+    info->taos = pTscObj;
     code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code);
@@ -1528,7 +1534,7 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
     info->dataFormat = true;
   }
 
-  if(request){
+  if (request) {
     info->pRequest = request;
     info->msgBuf.buf = info->pRequest->msgBuf;
     info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
@@ -1827,10 +1833,11 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
   }
   pVal->length = (int16_t)strlen(value->valuestring);
 
-  if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){
+  if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
     return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
   }
-  if (pVal->type == TSDB_DATA_TYPE_NCHAR  && pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+  if (pVal->type == TSDB_DATA_TYPE_NCHAR &&
+      pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
     return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
   }
 
@@ -2058,7 +2065,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
   SSmlLineInfo elements = {0};
   uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql);
 
-  int          ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
+  int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id);
     return ret;
@@ -2314,7 +2321,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
     (*pMeta)->tableMeta->uid = tableData->uid;  // one table merge data block together according uid
 
     code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat,
-                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->msgBuf.buf, info->msgBuf.len);
+                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
+                       info->msgBuf.buf, info->msgBuf.len);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
       return code;
@@ -2336,7 +2344,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
   SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
   atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
 
-  launchAsyncQuery(info->pRequest, info->pQuery, NULL, NULL);
+  SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
+  if (pWrapper == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  pWrapper->pRequest = info->pRequest;
+  launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2417,41 +2430,41 @@ static int smlProcess(SSmlHandle *info, char *lines[], int numLines) {
 }
 
 static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) {
-//  SCatalog *catalog = NULL;
-//  int32_t   code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog);
-//  if (code != TSDB_CODE_SUCCESS) {
-//    uError("SML get catalog error %d", code);
-//    return code;
-//  }
-//
-//  SName name;
-//  tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db));
-//  char dbFname[TSDB_DB_FNAME_LEN] = {0};
-//  tNameGetFullDbName(&name, dbFname);
-//  SDbCfgInfo pInfo = {0};
-//
-//  SRequestConnInfo conn = {0};
-//  conn.pTrans = taos->pAppInfo->pTransporter;
-//  conn.requestId = request->requestId;
-//  conn.requestObjRefId = request->self;
-//  conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp);
-//
-//  code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo);
-//  if (code != TSDB_CODE_SUCCESS) {
-//    return code;
-//  }
-//  taosArrayDestroy(pInfo.pRetensions);
-//
-//  if (!pInfo.schemaless) {
-//    return TSDB_CODE_SML_INVALID_DB_CONF;
-//  }
+  //  SCatalog *catalog = NULL;
+  //  int32_t   code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog);
+  //  if (code != TSDB_CODE_SUCCESS) {
+  //    uError("SML get catalog error %d", code);
+  //    return code;
+  //  }
+  //
+  //  SName name;
+  //  tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db));
+  //  char dbFname[TSDB_DB_FNAME_LEN] = {0};
+  //  tNameGetFullDbName(&name, dbFname);
+  //  SDbCfgInfo pInfo = {0};
+  //
+  //  SRequestConnInfo conn = {0};
+  //  conn.pTrans = taos->pAppInfo->pTransporter;
+  //  conn.requestId = request->requestId;
+  //  conn.requestObjRefId = request->self;
+  //  conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp);
+  //
+  //  code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo);
+  //  if (code != TSDB_CODE_SUCCESS) {
+  //    return code;
+  //  }
+  //  taosArrayDestroy(pInfo.pRetensions);
+  //
+  //  if (!pInfo.schemaless) {
+  //    return TSDB_CODE_SML_INVALID_DB_CONF;
+  //  }
   return TSDB_CODE_SUCCESS;
 }
 
 static void smlInsertCallback(void *param, void *res, int32_t code) {
   SRequestObj *pRequest = (SRequestObj *)res;
   SSmlHandle  *info = (SSmlHandle *)param;
-  int32_t rows = taos_affected_rows(pRequest);
+  int32_t      rows = taos_affected_rows(pRequest);
 
   uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf);
   Params *pParam = info->params;
@@ -2461,7 +2474,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     pParam->request->code = code;
     pParam->request->body.resInfo.numOfRows += rows;
-  }else{
+  } else {
     pParam->request->body.resInfo.numOfRows += info->affectedRows;
   }
   if (pParam->cnt == pParam->total) {
@@ -2497,20 +2510,20 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
  *
  */
 
-TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
+TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
 
-  SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
-  if(!request){
+  SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT);
+  if (!request) {
     uError("SML:taos_schemaless_insert error request is null");
     return NULL;
   }
 
-  int    batchs = 0;
-  STscObj* pTscObj = request->pTscObj;
+  int      batchs = 0;
+  STscObj *pTscObj = request->pTscObj;
 
   pTscObj->schemalessType = 1;
   SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
@@ -2526,7 +2539,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
     goto end;
   }
 
-  if(isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS){
+  if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) {
     request->code = TSDB_CODE_SML_INVALID_DB_CONF;
     smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL);
     goto end;
@@ -2551,9 +2564,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
     goto end;
   }
 
-  if(protocol == TSDB_SML_JSON_PROTOCOL){
+  if (protocol == TSDB_SML_JSON_PROTOCOL) {
     numLines = 1;
-  }else if(numLines <= 0){
+  } else if (numLines <= 0) {
     request->code = TSDB_CODE_SML_INVALID_DATA;
     smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL);
     goto end;
@@ -2562,14 +2575,14 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
   batchs = ceil(((double)numLines) / LINE_BATCH);
   params.total = batchs;
   for (int i = 0; i < batchs; ++i) {
-    SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT);
-    if(!req){
+    SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT);
+    if (!req) {
       request->code = TSDB_CODE_OUT_OF_MEMORY;
       uError("SML:taos_schemaless_insert error request is null");
       goto end;
     }
-    SSmlHandle* info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision);
-    if(!info){
+    SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision);
+    if (!info) {
       request->code = TSDB_CODE_OUT_OF_MEMORY;
       uError("SML:taos_schemaless_insert error SSmlHandle is null");
       goto end;
@@ -2599,8 +2612,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
 end:
   taosThreadSpinDestroy(&params.lock);
   tsem_destroy(&params.sem);
-//  ((STscObj *)taos)->schemalessType = 0;
+  //  ((STscObj *)taos)->schemalessType = 0;
   pTscObj->schemalessType = 1;
   uDebug("resultend:%s", request->msgBuf);
-  return (TAOS_RES*)request;
+  return (TAOS_RES *)request;
 }
diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c
index 6b3d6c77c1..748478778a 100644
--- a/source/libs/parser/src/parser.c
+++ b/source/libs/parser/src/parser.c
@@ -215,6 +215,10 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
 }
 
 void qDestroyParseContext(SParseContext* pCxt) {
+  if (NULL == pCxt) {
+    return;
+  }
+
   taosArrayDestroy(pCxt->pTableMetaPos);
   taosArrayDestroy(pCxt->pTableVgroupPos);
   taosMemoryFree(pCxt);

From 02979f081d8bfc0cd11622c0e282f298769763b1 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 11 Oct 2022 17:47:23 +0800
Subject: [PATCH 38/44] fix: support statistics of insert_req

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 25 ++++-------
 source/dnode/mgmt/node_mgmt/inc/dmNodes.h   |  2 +-
 source/dnode/mgmt/node_mgmt/src/dmMonitor.c |  2 +-
 source/dnode/vnode/inc/vnode.h              |  2 +-
 source/dnode/vnode/src/inc/vnodeInt.h       |  8 ++--
 source/dnode/vnode/src/vnd/vnodeQuery.c     | 50 +++++++++++----------
 6 files changed, 41 insertions(+), 48 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 094d0559d5..7d8b8dc0f6 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
-void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
+void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
   pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad));
   if (pInfo->pVloads == NULL) return;
 
@@ -29,7 +29,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
 
     SVnodeObj *pVnode = *ppVnode;
     SVnodeLoad vload = {0};
-    vnodeGetLoad(pVnode->pImpl, &vload);
+    vnodeGetLoad(pVnode->pImpl, &vload, isReset);
     taosArrayPush(pInfo->pVloads, &vload);
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }
@@ -39,7 +39,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
 
 void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
   SMonVloadInfo vloads = {0};
-  vmGetVnodeLoads(pMgmt, &vloads);
+  vmGetVnodeLoads(pMgmt, &vloads, true);
 
   SArray *pVloads = vloads.pVloads;
   if (pVloads == NULL) return;
@@ -66,10 +66,10 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
   pInfo->vstat.totalVnodes = totalVnodes;
   pInfo->vstat.masterNum = masterNum;
   pInfo->vstat.numOfSelectReqs = numOfSelectReqs - pMgmt->state.numOfSelectReqs;
-  pInfo->vstat.numOfInsertReqs = numOfInsertReqs;
-  pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs;
-  pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs;
-  pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
+  pInfo->vstat.numOfInsertReqs = numOfInsertReqs;                          // delta
+  pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs;            // delta
+  pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs;                // delta
+  pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;  // delta
   pMgmt->state.totalVnodes = totalVnodes;
   pMgmt->state.masterNum = masterNum;
   pMgmt->state.numOfSelectReqs = numOfSelectReqs;
@@ -78,15 +78,6 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) {
   pMgmt->state.numOfBatchInsertReqs = numOfBatchInsertReqs;
   pMgmt->state.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
 
-  printf("%s:%d: Info: nInsert:%" PRIi64 ", nInsertSuccess:%" PRIi64 ", nBatch:%" PRIi64 ", nBatchSuccess:%" PRIi64
-         "\n",
-         __func__, __LINE__, pInfo->vstat.numOfInsertReqs, pInfo->vstat.numOfInsertSuccessReqs,
-         pInfo->vstat.numOfBatchInsertReqs, pInfo->vstat.numOfBatchInsertSuccessReqs);
-  printf("%s:%d: Mgmt: nInsert:%" PRIi64 ", nInsertSuccess:%" PRIi64 ", nBatch:%" PRIi64 ", nBatchSuccess:%" PRIi64
-         "\n",
-         __func__, __LINE__, pMgmt->state.numOfInsertReqs, pMgmt->state.numOfInsertSuccessReqs,
-         pMgmt->state.numOfBatchInsertReqs, pMgmt->state.numOfBatchInsertSuccessReqs);
-
   tfsGetMonitorInfo(pMgmt->pTfs, &pInfo->tfs);
   taosArrayDestroy(pVloads);
 }
@@ -118,7 +109,7 @@ int32_t vmProcessGetMonitorInfoReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
 int32_t vmProcessGetLoadsReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   SMonVloadInfo vloads = {0};
-  vmGetVnodeLoads(pMgmt, &vloads);
+  vmGetVnodeLoads(pMgmt, &vloads, false);
 
   int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads);
   if (rspLen < 0) {
diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h
index 8c2d57808f..d3f1044f88 100644
--- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h
+++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h
@@ -35,7 +35,7 @@ void qmGetMonitorInfo(void *pMgmt, SMonQmInfo *pInfo);
 void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo);
 void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo);
 
-void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo);
+void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo, bool isReset);
 void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo);
 void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo);
 
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
index ecad390ef9..50d6aca53e 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
@@ -152,7 +152,7 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
     if (tsMultiProcess) {
       dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo);
     } else if (pWrapper->pMgmt != NULL) {
-      vmGetVnodeLoads(pWrapper->pMgmt, pInfo);
+      vmGetVnodeLoads(pWrapper->pMgmt, pInfo, false);
     }
     dmReleaseWrapper(pWrapper);
   }
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index ba16bad7cf..ec300b5f9c 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -73,7 +73,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
 int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
 int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
 
-int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
+int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad, bool isReset);
 int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
 
 int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index d4b88abb1d..8fe20def62 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -286,10 +286,10 @@ struct SVState {
 };
 
 struct SVStatis {
-  int64_t nInsert;
-  int64_t nInsertSuccess;
-  int64_t nBatchInsert;
-  int64_t nBatchInsertSuccess;
+  int64_t nInsert;              // delta
+  int64_t nInsertSuccess;       // delta
+  int64_t nBatchInsert;         // delta
+  int64_t nBatchInsertSuccess;  // delta
 };
 
 struct SVnodeInfo {
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 77499ee684..545bacb673 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -15,6 +15,14 @@
 
 #include "vnd.h"
 
+#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType)                         \
+  do {                                                                       \
+    if ((oVal) != atomic_val_compare_exchange_##vType(&(pVar), (oVal), 0)) { \
+      int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal));     \
+      ASSERT(newVal >= 0);                                                   \
+    }                                                                        \
+  } while (0)
+
 int vnodeQueryOpen(SVnode *pVnode) {
   return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), (void **)&pVnode->pQuery, &pVnode->msgCb);
 }
@@ -365,37 +373,31 @@ _exit:
   return code;
 }
 
-#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, type)                   \
-  do {                                                                \
-    if (oVal != atomic_val_compare_exchange_##type(&pVar, oVal, 0)) { \
-      int##type##_t tmpVal = atomic_sub_fetch_##type(&pVar, oVal);    \
-      ASSERT(tmpVal >= 0);                                            \
-    }                                                                 \
-  } while (0)
-
-int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
-    pLoad->vgId = TD_VID(pVnode);
-    pLoad->syncState = syncGetMyRole(pVnode->sync);
-    pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
-    pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
-    pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
-    pLoad->totalStorage = (int64_t)3 * 1073741824;
-    pLoad->compStorage = (int64_t)2 * 1073741824;
-    pLoad->pointsWritten = 100;
-    pLoad->numOfSelectReqs = 1;
-    pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
-    pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
-    pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
-    pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
+int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad, bool isReset) {
+  pLoad->vgId = TD_VID(pVnode);
+  pLoad->syncState = syncGetMyRole(pVnode->sync);
+  pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
+  pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
+  pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
+  pLoad->totalStorage = (int64_t)3 * 1073741824;
+  pLoad->compStorage = (int64_t)2 * 1073741824;
+  pLoad->pointsWritten = 100;
+  pLoad->numOfSelectReqs = 1;
+  pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert);
+  pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
+  pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
+  pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
 
+  if (isReset) {
     VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64);
     VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
     VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
     VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
-
-    return 0;
   }
 
+  return 0;
+}
+
 void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
   if (dbname) {
     *dbname = pVnode->config.dbname;

From 6680ed32d2f0a9f10cf53df40aef50576b52d18c Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 11 Oct 2022 17:50:26 +0800
Subject: [PATCH 39/44] chore: code format revert

---
 source/dnode/vnode/src/vnd/vnodeCommit.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 922b5ac5d8..6dc3ef86a7 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -227,7 +227,6 @@ int vnodeCommit(SVnode *pVnode) {
   info.state.committed = pVnode->state.applied;
   info.state.commitTerm = pVnode->state.applyTerm;
   info.state.commitID = pVnode->state.commitID;
-
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   if (vnodeSaveInfo(dir, &info) < 0) {
     ASSERT(0);

From 114ae8dbdaa79628d23b7afe1303efbec6dcf7d7 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 11 Oct 2022 18:22:29 +0800
Subject: [PATCH 40/44] fix: support statistics of insert_req

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |  3 ++-
 source/dnode/vnode/inc/vnode.h              |  3 ++-
 source/dnode/vnode/src/vnd/vnodeQuery.c     | 23 +++++++++++++--------
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 7d8b8dc0f6..a9e83e7904 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -29,7 +29,8 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
 
     SVnodeObj *pVnode = *ppVnode;
     SVnodeLoad vload = {0};
-    vnodeGetLoad(pVnode->pImpl, &vload, isReset);
+    vnodeGetLoad(pVnode->pImpl, &vload);
+    if (isReset) vnodeResetLoad(pVnode->pImpl, &vload);
     taosArrayPush(pInfo->pVloads, &vload);
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index ec300b5f9c..70d55d0320 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -73,7 +73,8 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
 int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
 int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
 
-int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad, bool isReset);
+void    vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
+int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
 
 int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg);
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 545bacb673..d24439c850 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -373,7 +373,7 @@ _exit:
   return code;
 }
 
-int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad, bool isReset) {
+int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   pLoad->vgId = TD_VID(pVnode);
   pLoad->syncState = syncGetMyRole(pVnode->sync);
   pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
@@ -387,17 +387,22 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad, bool isReset) {
   pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess);
   pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert);
   pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess);
-
-  if (isReset) {
-    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64);
-    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
-    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
-    VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
-  }
-
   return 0;
 }
 
+/**
+ * @brief Reset the statistics value by monitor interval
+ *
+ * @param pVnode
+ * @param pLoad
+ */
+void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
+  VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64);
+  VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
+  VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
+  VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
+}
+
 void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
   if (dbname) {
     *dbname = pVnode->config.dbname;

From 3734dd32b07a64eb4e8d7416a68f3a6abddab37a Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 11 Oct 2022 18:32:02 +0800
Subject: [PATCH 41/44] chore: code optimization for insert_req statistics

---
 source/dnode/vnode/src/vnd/vnodeQuery.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index d24439c850..030ba65442 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -15,12 +15,10 @@
 
 #include "vnd.h"
 
-#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType)                         \
-  do {                                                                       \
-    if ((oVal) != atomic_val_compare_exchange_##vType(&(pVar), (oVal), 0)) { \
-      int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal));     \
-      ASSERT(newVal >= 0);                                                   \
-    }                                                                        \
+#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType)                   \
+  do {                                                                 \
+    int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \
+    ASSERT(newVal >= 0);                                               \
   } while (0)
 
 int vnodeQueryOpen(SVnode *pVnode) {
@@ -401,6 +399,8 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64);
   VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64);
   VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64);
+
+
 }
 
 void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {

From 9c458b68ee4c3532ed591333c7500f8f3323ca5e Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Tue, 11 Oct 2022 18:37:38 +0800
Subject: [PATCH 42/44] fix: win shell tab auto input

---
 tools/shell/src/shellAuto.c    | 8 ++++++++
 tools/shell/src/shellCommand.c | 3 ++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c
index 506897cf5b..8aafd747af 100644
--- a/tools/shell/src/shellAuto.c
+++ b/tools/shell/src/shellAuto.c
@@ -1133,6 +1133,10 @@ bool firstMatchCommand(TAOS* con, SShellCmd* cmd) {
 
   // print to screen
   printScreen(con, cmd, match);
+#ifdef WINDOWS
+  printf("\r");
+  shellShowOnScreen(cmd);
+#endif
   freeCommand(input);
   taosMemoryFree(input);
   return true;
@@ -1195,6 +1199,10 @@ bool nextMatchCommand(TAOS* con, SShellCmd* cmd, SWords* firstMatch) {
 
   // print to screen
   printScreen(con, cmd, match);
+#ifdef WINDOWS
+  printf("\r");
+  shellShowOnScreen(cmd);
+#endif
 
   // free
   if (input->source) {
diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c
index cffa02824e..0b26d685fd 100644
--- a/tools/shell/src/shellCommand.c
+++ b/tools/shell/src/shellCommand.c
@@ -417,8 +417,9 @@ char taosGetConsoleChar() {
   static char mbStr[5];
   static unsigned long bufLen = 0;
   static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0;
+  CONSOLE_READCONSOLE_CONTROL inputControl={ sizeof(CONSOLE_READCONSOLE_CONTROL), 0, 1<<TAB_KEY, 0 };
   while (bufLen == 0) {
-    ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL);
+    ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, &inputControl);
     if (bufLen > 0 && buf[0] == 0) bufLen = 0;
     bufIndex = 0;
   }

From 0b2ff85a1772d12febe7eec519a9486c8a0424ca Mon Sep 17 00:00:00 2001
From: Zhiqiang Wang <1296468573@qq.com>
Date: Tue, 11 Oct 2022 18:53:24 +0800
Subject: [PATCH 43/44] fix: jemalloc compile error (#17291)

* fix: jemalloc compile error

* fix: jemalloc compile error
---
 contrib/CMakeLists.txt           | 18 ++++++++++++++++++
 source/dnode/mgmt/CMakeLists.txt |  3 ---
 source/os/CMakeLists.txt         |  4 ++++
 tools/shell/CMakeLists.txt       |  3 ---
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 2dc7622f46..a1a6cd4519 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -37,6 +37,11 @@ if(${BUILD_WITH_ICONV})
     cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
+# jemalloc
+if(${JEMALLOC_ENABLED})
+    cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
 # msvc regex
 if(${BUILD_MSVCREGEX})
     cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -258,6 +263,19 @@ if(${BUILD_PTHREAD})
     target_link_libraries(pthread INTERFACE libpthreadVC3)
 endif()
 
+# jemalloc
+if(${JEMALLOC_ENABLED})
+    include(ExternalProject)
+    ExternalProject_Add(jemalloc
+        PREFIX        "jemalloc"
+        SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
+        BUILD_IN_SOURCE     1
+        CONFIGURE_COMMAND   ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
+        BUILD_COMMAND       ${MAKE}
+    )
+    INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)
+endif()
+
 # crashdump
 if(${BUILD_CRASHDUMP})
     add_executable(dumper "crashdump/dumper/dumper.c")
diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt
index 45bef7f98e..581686ba90 100644
--- a/source/dnode/mgmt/CMakeLists.txt
+++ b/source/dnode/mgmt/CMakeLists.txt
@@ -14,7 +14,4 @@ target_include_directories(
     taosd
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
 )
-IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
-    add_dependencies(taosd jemalloc)
-ENDIF ()
 target_link_libraries(taosd dnode)
diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt
index 2a9d0c8535..b7cb20896b 100644
--- a/source/os/CMakeLists.txt
+++ b/source/os/CMakeLists.txt
@@ -52,3 +52,7 @@ else()
         os PUBLIC dl m rt
     )
 endif()
+
+IF (JEMALLOC_ENABLED)
+    target_link_libraries(os PUBLIC -ljemalloc)
+ENDIF ()
\ No newline at end of file
diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt
index 45a6f8c16f..552b77e6e9 100644
--- a/tools/shell/CMakeLists.txt
+++ b/tools/shell/CMakeLists.txt
@@ -28,7 +28,4 @@ target_include_directories(
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
-IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
-    add_dependencies(taosd jemalloc)
-ENDIF ()
 SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

From 9f1c89e3ab3055118eb87efaa46dbdfe052f1bda Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 11 Oct 2022 19:56:24 +0800
Subject: [PATCH 44/44] docs: fix a few taosbenchmark doc typos (#17302)

---
 docs/en/14-reference/05-taosbenchmark.md | 4 ++--
 docs/zh/14-reference/05-taosbenchmark.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md
index 8f63dddfb7..bde5e33034 100644
--- a/docs/en/14-reference/05-taosbenchmark.md
+++ b/docs/en/14-reference/05-taosbenchmark.md
@@ -23,7 +23,7 @@ There are two ways to install taosBenchmark:
 
 TaosBenchmark needs to be executed on the terminal of the operating system, it supports two configuration methods: [Command-line arguments](#command-line-arguments-in-detail) and [JSON configuration file](#configuration-file-parameters-in-detail). These two methods are mutually exclusive. Users can use `-f <json file>` to specify a configuration file. When running taosBenchmark with command-line arguments to control its behavior, users should use other parameters for configuration, but not the `-f` parameter. In addition, taosBenchmark offers a special way of running without parameters.
 
-taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscriptionm configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test.
+taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscription configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test.
 
 **Make sure that the TDengine cluster is running correctly before running taosBenchmark. **
 
@@ -340,7 +340,7 @@ The configuration parameters for specifying super table tag columns and data col
 
 - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values.
 
-- **sma**: Insert the column into the BSMA. Enter `yes` or `no`. The default is `no`.
+- **sma**: Insert the column into the SMA. Enter `yes` or `no`. The default is `no`.
 
 #### insertion behavior configuration parameters
 
diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md
index 0d6aad6240..6a6d9e3878 100644
--- a/docs/zh/14-reference/05-taosbenchmark.md
+++ b/docs/zh/14-reference/05-taosbenchmark.md
@@ -340,7 +340,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 
 - **values** : nchar/binary 列/标签的值域，将从值中随机选择。
 
-- **sma**: 将该列加入bsma中，值为 "yes" 或者 "no"，默认为 "no"。
+- **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。
 
 #### 插入行为配置参数