From b017d2a4c8480cd2df03a6e78309e4074a979419 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 15 Apr 2022 15:04:17 +0800
Subject: [PATCH 01/16] refactor: adjust logs and code style

---
 source/common/src/tglobal.c | 2 +-
 source/util/src/tconfig.c   | 6 +++---
 tools/shell/src/shellMain.c | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 9226e9aa37..49f4afc12b 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -238,7 +238,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
 
   if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
     if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
-      uError("failed to load from config file:%s since %s", cfgFile, terrstr());
+      uInfo("cfg file:%s not read since %s", cfgFile, terrstr());
       return 0;
     }
   }
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index 74d7c15e78..06a6b8317d 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -591,12 +591,12 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
 }
 
 int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
-  uInfo("load from env variables not implemented yet");
+  uDebug("load from env variables not implemented yet");
   return 0;
 }
 
 int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
-  uInfo("load from env file not implemented yet");
+  uDebug("load from env file not implemented yet");
   return 0;
 }
 
@@ -655,6 +655,6 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
 }
 
 int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
-  uInfo("load from apoll url not implemented yet");
+  uDebug("load from apoll url not implemented yet");
   return 0;
 }
diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c
index 70563c79e6..8ca8142eca 100644
--- a/tools/shell/src/shellMain.c
+++ b/tools/shell/src/shellMain.c
@@ -45,7 +45,7 @@ static tsem_t cancelSem;
 
 static struct argp_option options[] = {
   {"host",       'h', "HOST",       0,                   "TDengine server FQDN to connect. The default host is localhost."},
-  {"password",   'p', 0,   0,                   "The password to use when connecting to the server."},
+  {"password",   'p', 0,            0,                   "The password to use when connecting to the server."},
   {"port",       'P', "PORT",       0,                   "The TCP/IP port number to use for the connection."},
   {"user",       'u', "USER",       0,                   "The user name to use when connecting to the server."},
   {"auth",       'A', "Auth",       0,                   "The auth string to use when connecting to the server."},

From 03a9679f975d085d7ae21fc9996697b786b67af5 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Sat, 16 Apr 2022 19:40:50 +0800
Subject: [PATCH 02/16] fix(query): translate var type from value node to
 SVariant should include var header

---
 source/libs/nodes/src/nodesUtilFuncs.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 758f9d5d6f..d4457a056e 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -1226,7 +1226,7 @@ void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) {
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
-      pVal->pz = pNode->datum.p + VARSTR_HEADER_SIZE;
+      pVal->pz = pNode->datum.p;
       break;
     case TSDB_DATA_TYPE_JSON:
     case TSDB_DATA_TYPE_DECIMAL:

From 884bf3775f13308182cd59598a099fa430e4fbc3 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 10:21:25 +0800
Subject: [PATCH 03/16] fix: definite lost in tcache.c found by valgrind

---
 source/util/src/tcache.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c
index c4e85a425e..a69f11f285 100644
--- a/source/util/src/tcache.c
+++ b/source/util/src/tcache.c
@@ -910,6 +910,8 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1)
 
 void taosStopCacheRefreshWorker(void) {
   stopRefreshWorker = true;
+  taosThreadJoin(cacheRefreshWorker, NULL);
+  taosArrayDestroy(pCacheArrayList);
 }
 
 size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) {

From 0d262b874745a409800856e976b719b40fe8fdce Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 10:22:08 +0800
Subject: [PATCH 04/16] fix: definite lost in tlog.c found by valgrind

---
 include/os/osMemory.h          |  12 +--
 source/client/src/clientMain.c |   2 +-
 source/util/src/tlog.c         | 163 +++++++++++++++++----------------
 3 files changed, 92 insertions(+), 85 deletions(-)

diff --git a/include/os/osMemory.h b/include/os/osMemory.h
index 4efe69e204..92d9319d5c 100644
--- a/include/os/osMemory.h
+++ b/include/os/osMemory.h
@@ -36,12 +36,12 @@ void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(void *ptr);
 int32_t taosMemorySize(void *ptr);
 
-#define taosMemoryFreeClear(ptr) \
-  do {                           \
-    if (ptr) {                   \
-      taosMemoryFree((void*)ptr);       \
-      (ptr) = NULL;              \
-    }                            \
+#define taosMemoryFreeClear(ptr)   \
+  do {                             \
+    if (ptr) {                     \
+      taosMemoryFree((void *)ptr); \
+      (ptr) = NULL;                \
+    }                              \
   } while (0)
 
 #ifdef __cplusplus
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 73ac059276..2e6cd4ce17 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -69,9 +69,9 @@ void taos_cleanup(void) {
   rpcCleanup();
   catalogDestroy();
   schedulerDestroy();
-  taosCloseLog();
 
   tscInfo("all local resources released");
+  taosCloseLog();
 }
 
 setConfRet taos_set_config(const char *config) {
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index b37b1b2f69..3dce260b10 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -38,27 +38,26 @@
 #define LOG_BUF_MUTEX(x)  ((x)->buffMutex)
 
 typedef struct {
-  char           *buffer;
-  int32_t         buffStart;
-  int32_t         buffEnd;
-  int32_t         buffSize;
-  int32_t         minBuffSize;
-  TdFilePtr       pFile;
-  int32_t         stop;
-  TdThread       asyncThread;
+  char         *buffer;
+  int32_t       buffStart;
+  int32_t       buffEnd;
+  int32_t       buffSize;
+  int32_t       minBuffSize;
+  TdFilePtr     pFile;
+  int32_t       stop;
+  TdThread      asyncThread;
   TdThreadMutex buffMutex;
-  tsem_t          buffNotEmpty;
 } SLogBuff;
 
 typedef struct {
-  int32_t         fileNum;
-  int32_t         maxLines;
-  int32_t         lines;
-  int32_t         flag;
-  int32_t         openInProgress;
-  pid_t           pid;
-  char            logName[LOG_FILE_NAME_LEN];
-  SLogBuff       *logHandle;
+  int32_t       fileNum;
+  int32_t       maxLines;
+  int32_t       lines;
+  int32_t       flag;
+  int32_t       openInProgress;
+  pid_t         pid;
+  char          logName[LOG_FILE_NAME_LEN];
+  SLogBuff     *logHandle;
   TdThreadMutex logMutex;
 } SLogObj;
 
@@ -100,7 +99,7 @@ int64_t dbgBigWN = 0;
 int64_t dbgWSize = 0;
 
 static void     *taosAsyncOutputLog(void *param);
-static int32_t   taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t msgLen);
+static int32_t   taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen);
 static SLogBuff *taosLogBuffNew(int32_t bufSize);
 static void      taosCloseLogByFd(TdFilePtr pFile);
 static int32_t   taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
@@ -136,16 +135,24 @@ static void taosStopLog() {
   }
 }
 
+static void taosLogBuffDestroy() {
+  taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex);
+  taosCloseFile(&tsLogObj.logHandle->pFile);
+  taosMemoryFreeClear(tsLogObj.logHandle->buffer);
+  memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer));
+  taosThreadMutexDestroy(&tsLogObj.logMutex);
+  taosMemoryFreeClear(tsLogObj.logHandle);
+  memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle));
+  tsLogObj.logHandle = NULL;
+}
+
 void taosCloseLog() {
   taosStopLog();
   if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) {
     taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL);
   }
   tsLogInited = 0;
-  // In case that other threads still use log resources causing invalid write in valgrind
-  // we comment two lines below.
-  // taosLogBuffDestroy(tsLogObj.logHandle);
-  // taosCloseLog();
+  taosLogBuffDestroy(tsLogObj.logHandle);
 }
 
 static bool taosLockLogFile(TdFilePtr pFile) {
@@ -231,7 +238,7 @@ static int32_t taosOpenNewLogFile() {
     tsLogObj.openInProgress = 1;
 
     uInfo("open new log file ......");
-    TdThread      thread;
+    TdThread     thread;
     TdThreadAttr attr;
     taosThreadAttrInit(&attr);
     taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED);
@@ -506,45 +513,45 @@ static void taosCloseLogByFd(TdFilePtr pFile) {
 }
 
 static SLogBuff *taosLogBuffNew(int32_t bufSize) {
-  SLogBuff *tLogBuff = NULL;
+  SLogBuff *pLogBuf = NULL;
 
-  tLogBuff = taosMemoryCalloc(1, sizeof(SLogBuff));
-  if (tLogBuff == NULL) return NULL;
+  pLogBuf = taosMemoryCalloc(1, sizeof(SLogBuff));
+  if (pLogBuf == NULL) return NULL;
 
-  LOG_BUF_BUFFER(tLogBuff) = taosMemoryMalloc(bufSize);
-  if (LOG_BUF_BUFFER(tLogBuff) == NULL) goto _err;
+  LOG_BUF_BUFFER(pLogBuf) = taosMemoryMalloc(bufSize);
+  if (LOG_BUF_BUFFER(pLogBuf) == NULL) goto _err;
 
-  LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0;
-  LOG_BUF_SIZE(tLogBuff) = bufSize;
-  tLogBuff->minBuffSize = bufSize / 10;
-  tLogBuff->stop = 0;
+  LOG_BUF_START(pLogBuf) = LOG_BUF_END(pLogBuf) = 0;
+  LOG_BUF_SIZE(pLogBuf) = bufSize;
+  pLogBuf->minBuffSize = bufSize / 10;
+  pLogBuf->stop = 0;
 
-  if (taosThreadMutexInit(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err;
-  // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0);
+  if (taosThreadMutexInit(&LOG_BUF_MUTEX(pLogBuf), NULL) < 0) goto _err;
+  // tsem_init(&(pLogBuf->buffNotEmpty), 0, 0);
 
-  return tLogBuff;
+  return pLogBuf;
 
 _err:
-  taosMemoryFreeClear(LOG_BUF_BUFFER(tLogBuff));
-  taosMemoryFreeClear(tLogBuff);
+  taosMemoryFreeClear(LOG_BUF_BUFFER(pLogBuf));
+  taosMemoryFreeClear(pLogBuf);
   return NULL;
 }
 
-static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, const char *msg, int32_t msgLen) {
+static void taosCopyLogBuffer(SLogBuff *pLogBuf, int32_t start, int32_t end, const char *msg, int32_t msgLen) {
   if (start > end) {
-    memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen);
+    memcpy(LOG_BUF_BUFFER(pLogBuf) + end, msg, msgLen);
   } else {
-    if (LOG_BUF_SIZE(tLogBuff) - end < msgLen) {
-      memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, LOG_BUF_SIZE(tLogBuff) - end);
-      memcpy(LOG_BUF_BUFFER(tLogBuff), msg + LOG_BUF_SIZE(tLogBuff) - end, msgLen - LOG_BUF_SIZE(tLogBuff) + end);
+    if (LOG_BUF_SIZE(pLogBuf) - end < msgLen) {
+      memcpy(LOG_BUF_BUFFER(pLogBuf) + end, msg, LOG_BUF_SIZE(pLogBuf) - end);
+      memcpy(LOG_BUF_BUFFER(pLogBuf), msg + LOG_BUF_SIZE(pLogBuf) - end, msgLen - LOG_BUF_SIZE(pLogBuf) + end);
     } else {
-      memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen);
+      memcpy(LOG_BUF_BUFFER(pLogBuf) + end, msg, msgLen);
     }
   }
-  LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff);
+  LOG_BUF_END(pLogBuf) = (LOG_BUF_END(pLogBuf) + msgLen) % LOG_BUF_SIZE(pLogBuf);
 }
 
-static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t msgLen) {
+static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen) {
   int32_t        start = 0;
   int32_t        end = 0;
   int32_t        remainSize = 0;
@@ -552,13 +559,13 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms
   char           tmpBuf[40] = {0};
   int32_t        tmpBufLen = 0;
 
-  if (tLogBuff == NULL || tLogBuff->stop) return -1;
+  if (pLogBuf == NULL || pLogBuf->stop) return -1;
 
-  taosThreadMutexLock(&LOG_BUF_MUTEX(tLogBuff));
-  start = LOG_BUF_START(tLogBuff);
-  end = LOG_BUF_END(tLogBuff);
+  taosThreadMutexLock(&LOG_BUF_MUTEX(pLogBuf));
+  start = LOG_BUF_START(pLogBuf);
+  end = LOG_BUF_END(pLogBuf);
 
-  remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1);
+  remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(pLogBuf) - end - 1);
 
   if (lostLine > 0) {
     sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine);
@@ -568,47 +575,47 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t ms
   if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) {
     lostLine++;
     tsAsyncLogLostLines++;
-    taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff));
+    taosThreadMutexUnlock(&LOG_BUF_MUTEX(pLogBuf));
     return -1;
   }
 
   if (lostLine > 0) {
-    taosCopyLogBuffer(tLogBuff, start, end, tmpBuf, tmpBufLen);
+    taosCopyLogBuffer(pLogBuf, start, end, tmpBuf, tmpBufLen);
     lostLine = 0;
   }
 
-  taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen);
+  taosCopyLogBuffer(pLogBuf, LOG_BUF_START(pLogBuf), LOG_BUF_END(pLogBuf), msg, msgLen);
 
   // int32_t w = atomic_sub_fetch_32(&waitLock, 1);
   /*
-  if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) {
-    tsem_post(&(tLogBuff->buffNotEmpty));
+  if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(pLogBuf) * 4 /5))) {
+    tsem_post(&(pLogBuf->buffNotEmpty));
     dbgPostN++;
   } else {
     dbgNoPostN++;
   }
   */
 
-  taosThreadMutexUnlock(&LOG_BUF_MUTEX(tLogBuff));
+  taosThreadMutexUnlock(&LOG_BUF_MUTEX(pLogBuf));
 
   return 0;
 }
 
-static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t end) {
+static int32_t taosGetLogRemainSize(SLogBuff *pLogBuf, int32_t start, int32_t end) {
   int32_t rSize = end - start;
 
-  return rSize >= 0 ? rSize : LOG_BUF_SIZE(tLogBuff) + rSize;
+  return rSize >= 0 ? rSize : LOG_BUF_SIZE(pLogBuf) + rSize;
 }
 
-static void taosWriteLog(SLogBuff *tLogBuff) {
+static void taosWriteLog(SLogBuff *pLogBuf) {
   static int32_t lastDuration = 0;
   int32_t        remainChecked = 0;
   int32_t        start, end, pollSize;
 
   do {
     if (remainChecked == 0) {
-      start = LOG_BUF_START(tLogBuff);
-      end = LOG_BUF_END(tLogBuff);
+      start = LOG_BUF_START(pLogBuf);
+      end = LOG_BUF_END(pLogBuf);
 
       if (start == end) {
         dbgEmptyW++;
@@ -616,8 +623,8 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
         return;
       }
 
-      pollSize = taosGetLogRemainSize(tLogBuff, start, end);
-      if (pollSize < tLogBuff->minBuffSize) {
+      pollSize = taosGetLogRemainSize(pLogBuf, start, end);
+      if (pollSize < pLogBuf->minBuffSize) {
         lastDuration += tsWriteInterval;
         if (lastDuration < LOG_MAX_WAIT_MSEC) {
           break;
@@ -628,38 +635,38 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
     }
 
     if (start < end) {
-      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
+      taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize);
     } else {
-      int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start;
-      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
+      int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start;
+      taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize);
 
-      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end);
+      taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end);
     }
 
     dbgWN++;
     dbgWSize += pollSize;
 
-    if (pollSize < tLogBuff->minBuffSize) {
+    if (pollSize < pLogBuf->minBuffSize) {
       dbgSmallWN++;
       if (tsWriteInterval < LOG_MAX_INTERVAL) {
         tsWriteInterval += LOG_INTERVAL_STEP;
       }
-    } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) {
+    } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 3) {
       dbgBigWN++;
       tsWriteInterval = LOG_MIN_INTERVAL;
-    } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) {
+    } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 4) {
       if (tsWriteInterval > LOG_MIN_INTERVAL) {
         tsWriteInterval -= LOG_INTERVAL_STEP;
       }
     }
 
-    LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + pollSize) % LOG_BUF_SIZE(tLogBuff);
+    LOG_BUF_START(pLogBuf) = (LOG_BUF_START(pLogBuf) + pollSize) % LOG_BUF_SIZE(pLogBuf);
 
-    start = LOG_BUF_START(tLogBuff);
-    end = LOG_BUF_END(tLogBuff);
+    start = LOG_BUF_START(pLogBuf);
+    end = LOG_BUF_END(pLogBuf);
 
-    pollSize = taosGetLogRemainSize(tLogBuff, start, end);
-    if (pollSize < tLogBuff->minBuffSize) {
+    pollSize = taosGetLogRemainSize(pLogBuf, start, end);
+    if (pollSize < pLogBuf->minBuffSize) {
       break;
     }
 
@@ -670,16 +677,16 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
 }
 
 static void *taosAsyncOutputLog(void *param) {
-  SLogBuff *tLogBuff = (SLogBuff *)param;
+  SLogBuff *pLogBuf = (SLogBuff *)param;
   setThreadName("log");
 
   while (1) {
     taosMsleep(tsWriteInterval);
 
     // Polling the buffer
-    taosWriteLog(tLogBuff);
+    taosWriteLog(pLogBuf);
 
-    if (tLogBuff->stop) break;
+    if (pLogBuf->stop) break;
   }
 
   return NULL;

From b4ceaa0f0d40fafcbd76c866b52a27d0f9c2cd0c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 10:22:31 +0800
Subject: [PATCH 05/16] fix: definite lost in tfs found by valgrind

---
 source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index bf33e85b95..a212d7edf4 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -262,6 +262,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
   vmCloseVnodes(pMgmt);
   vmStopWorker(pMgmt);
   vnodeCleanup();
+  tfsClose(pMgmt->pTfs);
   // walCleanUp();
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;

From 9ba91dbafa1431573a18d775bc85054f6aa8255f Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 10:44:00 +0800
Subject: [PATCH 06/16] test: invalid synatx in tsim

---
 tests/script/tsim/parser/fourArithmetic-basic.sim  | 1 -
 tests/script/tsim/query/charScalarFunction.sim     | 1 -
 tests/script/tsim/query/scalarFunction.sim         | 1 -
 tests/script/tsim/query/session.sim                | 1 -
 tests/script/tsim/tmq/basic1.sim                   | 1 -
 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim | 1 -
 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim | 1 -
 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim | 1 -
 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim | 1 -
 tests/script/tsim/tmq/mainConsumerInMultiTopic.sim | 1 -
 tests/script/tsim/tmq/mainConsumerInOneTopic.sim   | 1 -
 tests/script/tsim/tmq/multiTopic.sim               | 1 -
 tests/script/tsim/tmq/oneTopic.sim                 | 1 -
 tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim    | 1 -
 14 files changed, 14 deletions(-)

diff --git a/tests/script/tsim/parser/fourArithmetic-basic.sim b/tests/script/tsim/parser/fourArithmetic-basic.sim
index eee294dc80..dde451279d 100644
--- a/tests/script/tsim/parser/fourArithmetic-basic.sim
+++ b/tests/script/tsim/parser/fourArithmetic-basic.sim
@@ -1,7 +1,6 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/query/charScalarFunction.sim b/tests/script/tsim/query/charScalarFunction.sim
index 18ee93f610..f1575d7293 100644
--- a/tests/script/tsim/query/charScalarFunction.sim
+++ b/tests/script/tsim/query/charScalarFunction.sim
@@ -2,7 +2,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim
index be75e1a21c..80d2dafa84 100644
--- a/tests/script/tsim/query/scalarFunction.sim
+++ b/tests/script/tsim/query/scalarFunction.sim
@@ -2,7 +2,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim
index 28f021e4ed..6f4010c3f3 100644
--- a/tests/script/tsim/query/session.sim
+++ b/tests/script/tsim/query/session.sim
@@ -2,7 +2,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim
index 84927c83a0..f7bd273624 100644
--- a/tests/script/tsim/tmq/basic1.sim
+++ b/tests/script/tsim/tmq/basic1.sim
@@ -11,7 +11,6 @@
 system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim
index 4fd4e60648..cf53cc7bcd 100644
--- a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim
+++ b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim
index 3f199836f4..fe37ffffcb 100644
--- a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim
+++ b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim
index 818a87a5a1..76f3c93e27 100644
--- a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim
+++ b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim
index 1789a6bef3..86e0b32980 100644
--- a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim
+++ b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim
index e0c463df65..d846632da4 100644
--- a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim
+++ b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim
index 985237e34b..f6fccee5e5 100644
--- a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim
+++ b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/multiTopic.sim b/tests/script/tsim/tmq/multiTopic.sim
index 4ad2df38f8..3795e82e86 100644
--- a/tests/script/tsim/tmq/multiTopic.sim
+++ b/tests/script/tsim/tmq/multiTopic.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/oneTopic.sim b/tests/script/tsim/tmq/oneTopic.sim
index dbe6dbcb27..d4353f60c9 100644
--- a/tests/script/tsim/tmq/oneTopic.sim
+++ b/tests/script/tsim/tmq/oneTopic.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0
diff --git a/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim b/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim
index 2bad16fbaa..01bf91fd9f 100644
--- a/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim
+++ b/tests/script/tsim/tmq/overlapTopic2Con1Cgrp.sim
@@ -14,7 +14,6 @@
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/cfg.sh -n dnode1
 system sh/exec.sh -n dnode1 -s start
 
 $loop_cnt = 0

From f56c354c00a33909817c5c786b652da8f68e872a Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 10:58:40 +0800
Subject: [PATCH 07/16] fix: startup is slow in valgrind mode

---
 tests/tsim/src/simExe.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index 5a18084fff..a4db0eff13 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -18,7 +18,7 @@
 
 void simLogSql(char *sql, bool useSharp) {
   static TdFilePtr pFile = NULL;
-  char         filename[256];
+  char             filename[256];
   sprintf(filename, "%s/sim.sql", simScriptDir);
   if (pFile == NULL) {
     // fp = fopen(filename, "w");
@@ -305,7 +305,8 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
   return true;
 }
 
-void simReplaceStr(char *buf, char *src, char *dst) {
+bool simReplaceStr(char *buf, char *src, char *dst) {
+  bool  replaced = false;
   char *begin = strstr(buf, src);
   if (begin != NULL) {
     int32_t srcLen = (int32_t)strlen(src);
@@ -320,13 +321,16 @@ void simReplaceStr(char *buf, char *src, char *dst) {
     }
 
     memcpy(begin, dst, dstLen);
+    replaced = true;
   }
 
   simInfo("system cmd is %s", buf);
+  return replaced;
 }
 
 bool simExecuteSystemCmd(SScript *script, char *option) {
   char buf[4096] = {0};
+  bool replaced = false;
 
 #ifndef WINDOWS
   sprintf(buf, "cd %s; ", simScriptDir);
@@ -341,7 +345,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) {
   }
 
   if (useValgrind) {
-    simReplaceStr(buf, "exec.sh", "exec.sh -v");
+    replaced = simReplaceStr(buf, "exec.sh", "exec.sh -v");
   }
 
   simLogSql(buf, true);
@@ -359,6 +363,11 @@ bool simExecuteSystemCmd(SScript *script, char *option) {
 
   sprintf(script->system_exit_code, "%d", code);
   script->linePos++;
+  if (replaced && strstr(buf, "start") != NULL) {
+    simInfo("====> startup is slow in valgrind mode, so sleep 5 seconds after exec.sh -s start");
+    taosMsleep(5000);
+  }
+
   return true;
 }
 
@@ -774,7 +783,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) {
 
 bool simExecuteRestfulCmd(SScript *script, char *rest) {
   TdFilePtr pFile = NULL;
-  char  filename[256];
+  char      filename[256];
   sprintf(filename, "%s/tmp.sql", simScriptDir);
   // fp = fopen(filename, "w");
   pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);

From 9de53b0019f6e50e494892bbd126eea057175162 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 18 Apr 2022 11:25:39 +0800
Subject: [PATCH 08/16] fix valgrind memleak

---
 source/libs/qworker/src/qworker.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index a9a360e03e..f97d7ffae0 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -1470,6 +1470,9 @@ void qWorkerDestroy(void **qWorkerMgmt) {
 
   //TODO FREE ALL
 
+  taosHashCleanup(mgmt->ctxHash);
+  taosHashCleanup(mgmt->schHash);
+
   taosMemoryFreeClear(*qWorkerMgmt);
 }
 

From 6105dfc934df8c0fbd0fccf684a74043a010d6a9 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Mon, 18 Apr 2022 11:39:08 +0800
Subject: [PATCH 09/16] refactor(query): refactor concat/concat_ws and param
 check

---
 source/libs/function/src/builtins.c | 42 ++++++++++++++++++-----------
 source/libs/scalar/src/sclfunc.c    | 40 +++++++--------------------
 2 files changed, 36 insertions(+), 46 deletions(-)

diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index bba5145812..9766a658fd 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -231,33 +231,45 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, int32_t minParaNum, int32_t maxParaNum, int32_t primaryParaNo) {
+static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, int32_t minParaNum, int32_t maxParaNum, bool hasSep) {
   int32_t paraNum = LIST_LENGTH(pFunc->pParameterList);
   if (paraNum < minParaNum || paraNum > maxParaNum) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  uint8_t resultType = TSDB_DATA_TYPE_NCHAR;
+  uint8_t resultType = TSDB_DATA_TYPE_BINARY;
   int32_t resultBytes = 0;
   int32_t sepBytes = 0;
-  for (int32_t i = 0; i < LIST_LENGTH(pFunc->pParameterList); ++i) {
+
+  /* For concat/concat_ws function, if params have NCHAR type, promote the final result to NCHAR */
+  for (int32_t i = 0; i < paraNum; ++i) {
     SNode* pPara = nodesListGetNode(pFunc->pParameterList, i);
     uint8_t paraType = ((SExprNode*)pPara)->resType.type;
-    int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes;
     if (!IS_VAR_DATA_TYPE(paraType)) {
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
-    if (i < primaryParaNo) {
-      sepBytes = paraBytes;
-      continue;
+    if (TSDB_DATA_TYPE_NCHAR == paraType) {
+      resultType = paraType;
     }
-    if (TSDB_DATA_TYPE_BINARY == paraType) {
-      resultType = TSDB_DATA_TYPE_BINARY;
-    }
-    resultBytes += paraBytes;
   }
-  if (sepBytes > 0) {
-    resultBytes += sepBytes * (paraNum - 2);
+
+  for (int32_t i = 0; i < paraNum; ++i) {
+    SNode* pPara = nodesListGetNode(pFunc->pParameterList, i);
+    uint8_t paraType = ((SExprNode*)pPara)->resType.type;
+    int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes;
+    int32_t factor = 1;
+    if (TSDB_DATA_TYPE_NCHAR == resultType && TSDB_DATA_TYPE_VARCHAR == paraType) {
+      factor *= TSDB_NCHAR_SIZE;
+    }
+    resultBytes += paraBytes * factor;
+
+    if (i == 0) {
+      sepBytes = paraBytes * factor;
+    }
+  }
+
+  if (hasSep) {
+    resultBytes += sepBytes * (paraNum - 3);
   }
 
   pFunc->node.resType = (SDataType) { .bytes = resultBytes, .type = resultType };
@@ -265,11 +277,11 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
 }
 
 static int32_t translateConcat(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  return translateConcatImpl(pFunc, pErrBuf, len, 2, 8, 0);
+  return translateConcatImpl(pFunc, pErrBuf, len, 2, 8, false);
 }
 
 static int32_t translateConcatWs(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  return translateConcatImpl(pFunc, pErrBuf, len, 3, 9, 1);
+  return translateConcatImpl(pFunc, pErrBuf, len, 3, 9, true);
 }
 
 static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 97cc80b946..84d47adc29 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -313,8 +313,8 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t concatCopyHelper(const char *input, char *output, bool hasNcharCol, int32_t type, int16_t *dataLen) {
-  if (hasNcharCol && type == TSDB_DATA_TYPE_VARCHAR) {
+static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) {
+  if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
     TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
     bool ret = taosMbsToUcs4(varDataVal(input), varDataLen(input), newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, NULL);
     if (!ret) {
@@ -345,10 +345,6 @@ static int32_t getNumOfNullEntries(SColumnInfoData *pColumnInfoData, int32_t num
 }
 
 int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings
-    return TSDB_CODE_FAILED;
-  }
-
   SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
   SColumnInfoData *pOutputData = pOutput->columnData;
   char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
@@ -356,15 +352,8 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 
   int32_t inputLen = 0;
   int32_t numOfRows = 0;
-  bool hasNcharCol = false;
+  bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false;
   for (int32_t i = 0; i < inputNum; ++i) {
-    int32_t type = GET_PARAM_TYPE(&pInput[i]);
-    if (!IS_VAR_DATA_TYPE(type)) {
-      return TSDB_CODE_FAILED;
-    }
-    if (type == TSDB_DATA_TYPE_NCHAR) {
-      hasNcharCol = true;
-    }
     if (pInput[i].numOfRows > numOfRows) {
       numOfRows = pInput[i].numOfRows;
     }
@@ -373,7 +362,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
     pInputData[i] = pInput[i].columnData;
     input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0];
     int32_t factor = 1;
-    if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) {
+    if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) {
       factor = TSDB_NCHAR_SIZE;
     }
 
@@ -405,7 +394,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 
     int16_t dataLen = 0;
     for (int32_t i = 0; i < inputNum; ++i) {
-      int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen);
+      int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen);
       if (ret != TSDB_CODE_SUCCESS) {
         return ret;
       }
@@ -428,10 +417,6 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 
 
 int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator
-    return TSDB_CODE_FAILED;
-  }
-
   SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
   SColumnInfoData *pOutputData = pOutput->columnData;
   char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
@@ -439,15 +424,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
 
   int32_t inputLen = 0;
   int32_t numOfRows = 0;
-  bool hasNcharCol = false;
+  bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false;
   for (int32_t i = 1; i < inputNum; ++i) {
-    int32_t type = GET_PARAM_TYPE(&pInput[i]);
-    if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i]))) {
-      return TSDB_CODE_FAILED;
-    }
-    if (type == TSDB_DATA_TYPE_NCHAR) {
-      hasNcharCol = true;
-    }
     if (pInput[i].numOfRows > numOfRows) {
       numOfRows = pInput[i].numOfRows;
     }
@@ -456,7 +434,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
     pInputData[i] = pInput[i].columnData;
     input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0];
     int32_t factor = 1;
-    if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) {
+    if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) {
       factor = TSDB_NCHAR_SIZE;
     }
 
@@ -487,7 +465,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
         continue;
       }
 
-      int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen);
+      int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen);
       if (ret != TSDB_CODE_SUCCESS) {
         return ret;
       }
@@ -499,7 +477,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
       if (i < inputNum - 1) {
         //insert the separator
         char *sep = pInputData[0]->pData;
-        int32_t ret = concatCopyHelper(sep, output, hasNcharCol, GET_PARAM_TYPE(&pInput[0]), &dataLen);
+        int32_t ret = concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen);
         if (ret != TSDB_CODE_SUCCESS) {
           return ret;
         }

From 649a9ac230fa725c9d340a0b3f34994cd413440c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 11:42:39 +0800
Subject: [PATCH 10/16] fix: memory overwrite while process nchar in sim script

---
 tests/tsim/inc/simInt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h
index c8b13736c7..24119b1433 100644
--- a/tests/tsim/inc/simInt.h
+++ b/tests/tsim/inc/simInt.h
@@ -33,7 +33,7 @@
 #define MAX_BACKGROUND_SCRIPT_NUM 10
 #define MAX_FILE_NAME_LEN         256
 #define MAX_ERROR_LEN             1024
-#define MAX_QUERY_VALUE_LEN       40
+#define MAX_QUERY_VALUE_LEN       10240
 #define MAX_QUERY_COL_NUM         20
 #define MAX_QUERY_ROW_NUM         20
 #define MAX_SYSTEM_RESULT_LEN     2048

From 715787286cfadaa2f51a38901ee778d5215b9154 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 18 Apr 2022 14:24:23 +0800
Subject: [PATCH 11/16] ehn(query): set the parameters for system database to
 be NULL by default when responsing "show databases" command.

---
 source/dnode/mnode/impl/src/mndDb.c | 207 +++++++++++++++-------------
 1 file changed, 112 insertions(+), 95 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 13fbcdf9b3..9889743a82 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -1354,7 +1354,7 @@ char *mndGetDbStr(char *src) {
   return pos;
 }
 
-static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables) {
+static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb) {
   int32_t cols = 0;
 
   char* buf = taosMemoryMalloc(pShow->bytes[cols]);
@@ -1366,100 +1366,117 @@ static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, in
     ASSERT(0);
   }
 
-  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, buf, false);
-  taosMemoryFree(buf);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.quorum, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false);
-
-  char tmp[128] = {0};
-  int32_t len = 0;
-  if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
-    len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0);
-  } else {
-    len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2);
-  }
-
-  varDataSetLen(tmp, len);
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)tmp, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.fsyncPeriod, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false);
-
-  char *prec = NULL;
-  switch (pDb->cfg.precision) {
-    case TSDB_TIME_PRECISION_MILLI:
-      prec = TSDB_TIME_PRECISION_MILLI_STR;
-      break;
-    case TSDB_TIME_PRECISION_MICRO:
-      prec = TSDB_TIME_PRECISION_MICRO_STR;
-      break;
-    case TSDB_TIME_PRECISION_NANO:
-      prec = TSDB_TIME_PRECISION_NANO_STR;
-      break;
-    default:
-      prec = "none";
-      break;
-  }
-
-  char t[10] = {0};
-  STR_WITH_SIZE_TO_VARSTR(t, prec, 2);
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)t, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false);
-
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false);
-
   char *status = "ready";
-  char b[24] = {0};
+  char  b[24] = {0};
   STR_WITH_SIZE_TO_VARSTR(b, status, strlen(status));
 
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataAppend(pColInfo, rows, (const char *)b, false);
+  if (sysDb) {
+    for(int32_t i = 0; i < pShow->numOfColumns; ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+      if (i == 0) {
+        colDataAppend(pColInfo, rows, buf, false);
+      } else if (i == 3) {
+        colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
+      } else if (i == 20) {
+        colDataAppend(pColInfo, rows, b, false);
+      } else {
+        colDataAppendNULL(pColInfo, rows);
+      }
+    }
+  } else {
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, buf, false);
+    taosMemoryFree(buf);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.quorum, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false);
+
+    char    tmp[128] = {0};
+    int32_t len = 0;
+    if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
+      len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2,
+                    pDb->cfg.daysToKeep0);
+    } else {
+      len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1,
+                    pDb->cfg.daysToKeep2);
+    }
+
+    varDataSetLen(tmp, len);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)tmp, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.fsyncPeriod, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false);
+
+    char *prec = NULL;
+    switch (pDb->cfg.precision) {
+      case TSDB_TIME_PRECISION_MILLI:
+        prec = TSDB_TIME_PRECISION_MILLI_STR;
+        break;
+      case TSDB_TIME_PRECISION_MICRO:
+        prec = TSDB_TIME_PRECISION_MICRO_STR;
+        break;
+      case TSDB_TIME_PRECISION_NANO:
+        prec = TSDB_TIME_PRECISION_NANO_STR;
+        break;
+      default:
+        prec = "none";
+        break;
+    }
+
+    char t[10] = {0};
+    STR_WITH_SIZE_TO_VARSTR(t, prec, 2);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)t, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    colDataAppend(pColInfo, rows, (const char *)b, false);
+  }
 
   //  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
   //  *(int8_t *)pWrite = pDb->cfg.update;
@@ -1508,13 +1525,13 @@ static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo
   if (!pShow->sysDbRsp) {
     SDbObj infoschemaDb = {0};
     setInformationSchemaDbCfg(&infoschemaDb);
-    dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14);
+    dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true);
 
     numOfRows += 1;
 
     SDbObj perfschemaDb = {0};
     setPerfSchemaDbCfg(&perfschemaDb);
-    dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3);
+    dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true);
 
     numOfRows += 1;
     pShow->sysDbRsp = true;
@@ -1529,7 +1546,7 @@ static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo
     int32_t numOfTables = 0;
     sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
 
-    dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables);
+    dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false);
     numOfRows++;
     sdbRelease(pSdb, pDb);
   }

From 595d2ce352b2412013d05f812141c52ddd5c2762 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 14:37:45 +0800
Subject: [PATCH 12/16] enh: support variables like  $data[0][1] and
 $data(db)[0] in tsim scripts

---
 tests/script/tsim/db/alter_option.sim |  2 +-
 tests/script/tsim/db/basic6.sim       |  1 -
 tests/script/tsim/user/basic1.sim     | 38 ++++++-------
 tests/tsim/inc/simInt.h               |  4 +-
 tests/tsim/src/simExe.c               | 77 ++++++++++++++++++++++++---
 5 files changed, 92 insertions(+), 30 deletions(-)

diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim
index e9795bd8d2..525de9c5c4 100644
--- a/tests/script/tsim/db/alter_option.sim
+++ b/tests/script/tsim/db/alter_option.sim
@@ -72,7 +72,7 @@ print rows: $rows
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
 print ====> dataX_db
-print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+print $data(db)[0] $data(db)[1] $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
 
 if $rows != 3 then
   return -1
diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim
index 4d0a1e9ee9..f682dcc816 100644
--- a/tests/script/tsim/db/basic6.sim
+++ b/tests/script/tsim/db/basic6.sim
@@ -17,7 +17,6 @@ print =============== step1
 # quorum  presicion 
 sql create database $db vgroups 8 replica 1 days 2880 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
 sql show databases
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1 $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
 
 if $rows != 3 then
diff --git a/tests/script/tsim/user/basic1.sim b/tests/script/tsim/user/basic1.sim
index 7af5ba8d00..06a52c6604 100644
--- a/tests/script/tsim/user/basic1.sim
+++ b/tests/script/tsim/user/basic1.sim
@@ -9,9 +9,9 @@ if $rows != 1 then
   return -1
 endi
 
-print $data00 $data01 $data02
-print $data10 $data11 $data22
-print $data20 $data11 $data22
+print $data[0][0] $data[0][1] $data[0][2]
+print $data[1][0] $data[1][1] $data[1][2]
+print $data[2][0] $data[1][2] $data[2][2]
 
 sql_error show accounts;
 sql_error create account a pass "a"
@@ -25,10 +25,10 @@ if $rows != 2 then
   return -1
 endi
 
-print $data00 $data01 $data02
-print $data10 $data11 $data12
-print $data20 $data11 $data22
-print $data30 $data31 $data32
+print $data[0][0] $data[0][1] $data[0][2]
+print $data[1][0] $data[1][1] $data[1][2]
+print $data[2][0] $data[1][2] $data[2][2]
+print $data[3][0] $data[3][1] $data[3][2]
 
 print =============== create user2
 sql create user user2 PASS 'user2'
@@ -37,10 +37,10 @@ if $rows != 3 then
   return -1
 endi
 
-print $data00 $data01 $data02
-print $data10 $data11 $data12
-print $data20 $data11 $data22
-print $data30 $data31 $data32
+print $data[0][0] $data[0][1] $data[0][2]
+print $data[1][0] $data[1][1] $data[1][2]
+print $data[2][0] $data[1][2] $data[2][2]
+print $data[3][0] $data[3][1] $data[3][2]
 print $data40 $data41 $data42
 
 print =============== drop user1
@@ -50,10 +50,10 @@ if $rows != 2 then
   return -1
 endi
 
-print $data00 $data01 $data02
-print $data10 $data11 $data22
-print $data20 $data11 $data22
-print $data30 $data31 $data32
+print $data[0][0] $data[0][1] $data[0][2]
+print $data[1][0] $data[1][1] $data[1][2]
+print $data[2][0] $data[1][2] $data[2][2]
+print $data[3][0] $data[3][1] $data[3][2]
 
 print =============== restart taosd
 system sh/exec.sh -n dnode1 -s stop
@@ -66,9 +66,9 @@ if $rows != 2 then
   return -1
 endi
 
-print $data00 $data01 $data02
-print $data10 $data11 $data22
-print $data20 $data11 $data22
-print $data30 $data31 $data32
+print $data[0][0] $data[0][1] $data[0][2]
+print $data[1][0] $data[1][1] $data[1][2]
+print $data[2][0] $data[1][2] $data[2][2]
+print $data[3][0] $data[3][1] $data[3][2]
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h
index c8b13736c7..5ea372dee1 100644
--- a/tests/tsim/inc/simInt.h
+++ b/tests/tsim/inc/simInt.h
@@ -34,8 +34,8 @@
 #define MAX_FILE_NAME_LEN         256
 #define MAX_ERROR_LEN             1024
 #define MAX_QUERY_VALUE_LEN       40
-#define MAX_QUERY_COL_NUM         20
-#define MAX_QUERY_ROW_NUM         20
+#define MAX_QUERY_COL_NUM         100
+#define MAX_QUERY_ROW_NUM         100
 #define MAX_SYSTEM_RESULT_LEN     2048
 #define MAX_VAR_LEN               100
 #define MAX_VAR_NAME_LEN          32
diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index 5a18084fff..af612603a3 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -18,7 +18,7 @@
 
 void simLogSql(char *sql, bool useSharp) {
   static TdFilePtr pFile = NULL;
-  char         filename[256];
+  char             filename[256];
   sprintf(filename, "%s/sim.sql", simScriptDir);
   if (pFile == NULL) {
     // fp = fopen(filename, "w");
@@ -49,6 +49,41 @@ char *simParseHostName(char *varName) {
   return hostName;
 }
 
+static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) {
+  if (beginLen <= 5) {
+    *num = 0;
+  } else {
+    *num = atoi(begin + 5);
+  }
+}
+
+static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) {
+  const char *number = strstr(begin, "][");
+  if (number == NULL) {
+    *num = 0;
+  } else {
+    *num = atoi(number + 2);
+  }
+}
+
+static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, int32_t keyLen) {
+  key[0] = 0;
+  for (int32_t i = 5; i < beginLen && i - 5 < keyLen; ++i) {
+    if (begin[i] != 0 && begin[i] != ']' && begin[i] != ')') {
+      key[i - 5] = begin[i];
+    }
+  }
+}
+
+static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) {
+  const char *number = strstr(begin, ")[");
+  if (number == NULL) {
+    *num = 0;
+  } else {
+    *num = atoi(number + 2);
+  }
+}
+
 char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
   if (strncmp(varName, "hostname", 8) == 0) {
     return simParseHostName(varName);
@@ -66,13 +101,40 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
 
   if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content;
 
-  // variable like data2_192.168.0.1
   if (strncmp(varName, "data", 4) == 0) {
     if (varLen < 6) {
       return "null";
     }
 
-    if (varName[5] == '_') {
+    int32_t row = 0;
+    int32_t col = 0;
+    char    keyVal[1024] = {0};
+    int32_t keyLen = 1024;
+
+    if (varName[4] == '[') {
+      // $data[0][1]
+      simFindFirstNum(varName, varLen, &row);
+      simFindSecondNum(varName, varLen, &col);
+      if (row < 0 || row >= MAX_QUERY_ROW_NUM) {
+        return "null";
+      }
+      if (col < 0 || col >= MAX_QUERY_COL_NUM) {
+        return "null";
+      }
+      simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]);
+      return script->data[row][col];
+    } else if (varName[4] == '(') {
+      // $data(db)[0]
+      simFindFirstKeyVal(varName, varLen, keyVal, keyLen);
+      simFindSecondKeyNum(varName, varLen, &col);
+      for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) {
+        if (strncmp(keyVal, script->data[i][0], keyLen) == 0) {
+          simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]);
+          return script->data[i][col];
+        }
+      }
+    } else if (varName[5] == '_') {
+      // data2_db
       int32_t col = varName[4] - '0';
       if (col < 0 || col >= MAX_QUERY_COL_NUM) {
         return "null";
@@ -90,6 +152,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
       }
       return "null";
     } else if (varName[6] == '_') {
+      // data21_db
       int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0');
       if (col < 0 || col >= MAX_QUERY_COL_NUM) {
         return "null";
@@ -107,6 +170,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
       }
       return "null";
     } else {
+      // $data00
       int32_t row = varName[4] - '0';
       int32_t col = varName[5] - '0';
       if (row < 0 || row >= MAX_QUERY_ROW_NUM) {
@@ -119,6 +183,8 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
       simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]);
       return script->data[row][col];
     }
+
+    return "null";
   }
 
   for (int32_t i = 0; i < script->varLen; ++i) {
@@ -127,9 +193,6 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
       continue;
     }
     if (strncmp(varName, var->varName, varLen) == 0) {
-      // if (strlen(var->varValue) != 0)
-      //  simDebug("script:%s, var:%s, value:%s", script->fileName,
-      //  var->varName, var->varValue);
       return var->varValue;
     }
   }
@@ -774,7 +837,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) {
 
 bool simExecuteRestfulCmd(SScript *script, char *rest) {
   TdFilePtr pFile = NULL;
-  char  filename[256];
+  char      filename[256];
   sprintf(filename, "%s/tmp.sql", simScriptDir);
   // fp = fopen(filename, "w");
   pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);

From 5569c846245454d196c843656bcf91e79c7fffc7 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Mon, 18 Apr 2022 15:06:13 +0800
Subject: [PATCH 13/16] bsma integrate

---
 include/common/trow.h                    |  21 +++--
 include/common/ttypes.h                  |   4 +-
 source/common/src/trow.c                 |  55 ++++++++++--
 source/common/src/ttypes.c               | 102 +++++++++++------------
 source/dnode/vnode/src/tsdb/tsdbCommit.c |  31 +++++--
 5 files changed, 140 insertions(+), 73 deletions(-)

diff --git a/include/common/trow.h b/include/common/trow.h
index 9a09bd3c8f..dbbdef84a0 100644
--- a/include/common/trow.h
+++ b/include/common/trow.h
@@ -48,9 +48,12 @@ extern "C" {
 #define TD_VTYPE_NONE 0x02U  // none or unknown/undefined
 #define TD_VTYPE_MAX  0x03U  //
 
-#define TD_VTYPE_NORM_BYTE 0x0U
-#define TD_VTYPE_NULL_BYTE 0x55U
-#define TD_VTYPE_NONE_BYTE 0xAAU
+#define TD_VTYPE_NORM_BYTE_I 0x0U
+#define TD_VTYPE_NULL_BYTE_I 0xFFU
+
+#define TD_VTYPE_NORM_BYTE_II 0x0U
+#define TD_VTYPE_NULL_BYTE_II 0x55U
+#define TD_VTYPE_NONE_BYTE_II 0xAAU
 
 #define TD_ROWS_ALL_NORM  0x00U
 #define TD_ROWS_NULL_NORM 0x01U
@@ -224,8 +227,10 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
 int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode);
 static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
 static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
-static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode);
-static FORCE_INLINE bool    tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx);
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
+                                               int8_t bitmapMode);
+static FORCE_INLINE bool    tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode);
+bool                        tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode);
 int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
                              int8_t bitmapMode);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
@@ -233,7 +238,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa
 static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
                                                   col_id_t colId);
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
+int32_t                     tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
 
 /**
  * @brief
@@ -327,9 +332,9 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) {
+static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) {
   TDRowValT valType = 0;
-  tdGetBitmapValTypeII(pBitmap, idx, &valType);
+  tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode);
   if (tdValTypeIsNorm(valType)) {
     return true;
   }
diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 1a09368bd3..7f2e59ea85 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -248,8 +248,8 @@ typedef struct tDataTypeDescriptor {
                       int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
   int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
                         int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
-  void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
-                     int16_t *maxindex, int16_t *numofnull);
+  void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min,
+                     int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypes[15];
diff --git a/source/common/src/trow.c b/source/common/src/trow.c
index 7392b0bf8b..a1a2d236f9 100644
--- a/source/common/src/trow.c
+++ b/source/common/src/trow.c
@@ -16,10 +16,19 @@
 #define _DEFAULT_SOURCE
 #include "trow.h"
 
-const uint8_t tdVTypeByte[3] = {
-    TD_VTYPE_NORM_BYTE,  // TD_VTYPE_NORM
-    TD_VTYPE_NONE_BYTE,  // TD_VTYPE_NONE
-    TD_VTYPE_NULL_BYTE,  // TD_VTYPE_NULL
+const uint8_t tdVTypeByte[2][3] = {{
+                                       // 2 bits
+                                       TD_VTYPE_NORM_BYTE_II,
+                                       TD_VTYPE_NONE_BYTE_II,
+                                       TD_VTYPE_NULL_BYTE_II,
+                                   },
+                                   {
+                                       // 1 bit
+                                       TD_VTYPE_NORM_BYTE_I,
+                                       TD_VTYPE_NULL_BYTE_I,
+                                       TD_VTYPE_NULL_BYTE_I, // padding
+                                   }
+
 };
 
 // declaration
@@ -266,21 +275,53 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit
 //   }
 // }
 
+/**
+ * @brief Set bitmap area by byte preferentially and then by bit.
+ *
+ * @param pBitmap
+ * @param nEle
+ * @param valType
+ * @param bitmapMode 0 for 2 bits, 1 for 1 bit
+ * @return int32_t
+ */
 int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) {
   TASSERT(valType < TD_VTYPE_MAX);
-  int16_t nBytes = nEle / TD_VTYPE_PARTS;
+  int32_t nBytes = (bitmapMode == 0 ? nEle / TD_VTYPE_PARTS : nEle / TD_VTYPE_PARTS_I);
+  uint8_t vTypeByte = tdVTypeByte[bitmapMode][valType];
   for (int i = 0; i < nBytes; ++i) {
-    *(uint8_t *)pBitmap = tdVTypeByte[valType];
+    *(uint8_t *)pBitmap = vTypeByte;
     pBitmap = POINTER_SHIFT(pBitmap, 1);
   }
-  int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS;
 
+  int32_t nLeft = nEle - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
   for (int j = 0; j < nLeft; ++j) {
     tdSetBitmapValType(pBitmap, j, valType, bitmapMode);
   }
   return TSDB_CODE_SUCCESS;
 }
 
+bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
+  int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
+  uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
+  for (int i = 0; i < nBytes; ++i) {
+    if (*((uint8_t *)pBitmap) != vTypeByte) {
+      return false;
+    }
+    pBitmap = POINTER_SHIFT(pBitmap, 1);
+  }
+
+  int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
+
+  for (int j = 0; j < nLeft; ++j) {
+    uint8_t vType;
+    tdGetBitmapValType(pBitmap, j, &vType, bitmapMode);
+    if (vType != TD_VTYPE_NORM) {
+      return false;
+    }
+  }
+  return true;
+}
+
 static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c
index d4371e11f7..cb01a17273 100644
--- a/source/common/src/ttypes.c
+++ b/source/common/src/ttypes.c
@@ -50,8 +50,8 @@ const int32_t TYPE_BYTES[15] = {
     }                                                                          \
   } while (0)
 
-static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_bool(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                            int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
   *max = INT64_MIN;
@@ -62,7 +62,7 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (data[i] == TSDB_DATA_BOOL_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -71,8 +71,8 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO
   }
 }
 
-static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_i8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                          int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
   *max = INT64_MIN;
@@ -83,7 +83,7 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -92,8 +92,8 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR
   }
 }
 
-static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_u8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                          int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint8_t *data = (uint8_t *)pData;
   uint64_t _min = UINT64_MAX;
   uint64_t _max = 0;
@@ -106,7 +106,7 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -119,8 +119,8 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR
   *sum = _sum;
 }
 
-static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_i16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int16_t *data = (int16_t *)pData;
   *min = INT64_MAX;
   *max = INT64_MIN;
@@ -131,7 +131,7 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -140,8 +140,8 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf
   }
 }
 
-static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_u16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint16_t *data = (uint16_t *)pData;
   uint64_t  _min = UINT64_MAX;
   uint64_t  _max = 0;
@@ -154,7 +154,7 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -167,8 +167,8 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf
   *sum = _sum;
 }
 
-static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_i32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int32_t *data = (int32_t *)pData;
   *min = INT64_MAX;
   *max = INT64_MIN;
@@ -179,7 +179,7 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -188,8 +188,8 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf
   }
 }
 
-static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_u32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint32_t *data = (uint32_t *)pData;
   uint64_t  _min = UINT64_MAX;
   uint64_t  _max = 0;
@@ -202,7 +202,7 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -215,8 +215,8 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf
   *sum = _sum;
 }
 
-static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_i64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int64_t *data = (int64_t *)pData;
   *min = INT64_MAX;
   *max = INT64_MIN;
@@ -227,7 +227,7 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -236,8 +236,8 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf
   }
 }
 
-static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_u64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint64_t *data = (uint64_t *)pData;
   uint64_t  _min = UINT64_MAX;
   uint64_t  _max = 0;
@@ -250,7 +250,7 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -263,8 +263,8 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf
   *sum = _sum;
 }
 
-static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_f(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                         int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   float *data = (float *)pData;
   float  fmin = FLT_MAX;
   float  fmax = -FLT_MAX;
@@ -276,7 +276,7 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -300,8 +300,8 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo
   SET_DOUBLE_VAL(min, fmin);
 }
 
-static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                         int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_d(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                         int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   double *data = (double *)pData;
   double  dmin = DBL_MAX;
   double  dmax = -DBL_MAX;
@@ -313,7 +313,7 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -337,14 +337,14 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo
   SET_DOUBLE_PTR(min, &dmin);
 }
 
-static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_bin(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                           int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
     }
 
@@ -358,14 +358,14 @@ static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOf
   *maxIndex = 0;
 }
 
-static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
-                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
+static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
+                            int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
     // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
-    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
+    if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
       (*numOfNull) += 1;
     }
 
@@ -405,19 +405,19 @@ tDataTypeDescriptor tDataTypes[15] = {
 };
 
 char tTokenTypeSwitcher[13] = {
-    TSDB_DATA_TYPE_NULL,    // no type
-    TSDB_DATA_TYPE_BINARY,  // TK_ID
-    TSDB_DATA_TYPE_BOOL,    // TK_BOOL
-    TSDB_DATA_TYPE_BIGINT,  // TK_TINYINT
-    TSDB_DATA_TYPE_BIGINT,  // TK_SMALLINT
-    TSDB_DATA_TYPE_BIGINT,  // TK_INTEGER
-    TSDB_DATA_TYPE_BIGINT,  // TK_BIGINT
-    TSDB_DATA_TYPE_DOUBLE,  // TK_FLOAT
-    TSDB_DATA_TYPE_DOUBLE,  // TK_DOUBLE
-    TSDB_DATA_TYPE_BINARY,  // TK_STRING
-    TSDB_DATA_TYPE_BIGINT,  // TK_TIMESTAMP
+    TSDB_DATA_TYPE_NULL,     // no type
+    TSDB_DATA_TYPE_BINARY,   // TK_ID
+    TSDB_DATA_TYPE_BOOL,     // TK_BOOL
+    TSDB_DATA_TYPE_BIGINT,   // TK_TINYINT
+    TSDB_DATA_TYPE_BIGINT,   // TK_SMALLINT
+    TSDB_DATA_TYPE_BIGINT,   // TK_INTEGER
+    TSDB_DATA_TYPE_BIGINT,   // TK_BIGINT
+    TSDB_DATA_TYPE_DOUBLE,   // TK_FLOAT
+    TSDB_DATA_TYPE_DOUBLE,   // TK_DOUBLE
+    TSDB_DATA_TYPE_BINARY,   // TK_STRING
+    TSDB_DATA_TYPE_BIGINT,   // TK_TIMESTAMP
     TSDB_DATA_TYPE_VARCHAR,  // TK_BINARY
-    TSDB_DATA_TYPE_NCHAR,   // TK_NCHAR
+    TSDB_DATA_TYPE_NCHAR,    // TK_NCHAR
 };
 
 float  floatMin = -FLT_MAX, floatMax = FLT_MAX;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index cb4937d758..09616a8969 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -1201,6 +1201,23 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
   }
 }
 
+/**
+ * @brief Write SDataCols to data file.
+ *
+ * @param pRepo
+ * @param pTable
+ * @param pDFile
+ * @param pDFileAggr
+ * @param pDataCols The pDataCols would be generated from mem/imem directly with 2 bits bitmap or from tsdbRead
+ * interface with 1 bit bitmap.
+ * @param pBlock
+ * @param isLast
+ * @param isSuper
+ * @param ppBuf
+ * @param ppCBuf
+ * @param ppExBuf
+ * @return int
+ */
 int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
                        SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
   STsdbCfg     *pCfg = REPO_CFG(pRepo);
@@ -1244,14 +1261,15 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     pBlockCol->type = pDataCol->type;
     pAggrBlkCol->colId = pDataCol->colId;
 
-    if (tDataTypes[pDataCol->type].statisFunc) {
+    if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) {
 #if 0
       (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max),
                                                &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
                                                &(pBlockCol->numOfNull));
 #endif
-      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
-                                               &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
+      (*tDataTypes[pDataCol->type].statisFunc)(pDataCols->bitmapMode, pDataCol->pBitmap, pDataCol->pData, rowsToWrite,
+                                               &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum),
+                                               &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
                                                &(pAggrBlkCol->numOfNull));
 
       if (pAggrBlkCol->numOfNull == 0) {
@@ -1259,13 +1277,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
       } else {
         TD_SET_COL_ROWS_MISC(pBlockCol);
       }
+    } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) {
+      // check if all rows normal
+      TD_SET_COL_ROWS_NORM(pBlockCol);
     } else {
       TD_SET_COL_ROWS_MISC(pBlockCol);
     }
 
     ++nColsNotAllNull;
 
-    if (pColumn->sma) {
+    if (isSuper && pColumn->sma) {
       ++nColsOfBlockSma;
     }
   }
@@ -1277,7 +1298,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   uint32_t toffset = 0;
   int32_t  tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest);
   int32_t  lsize = tsize;
-  uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest);
+  uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsOfBlockSma, SBlockVerLatest);
   int32_t  keyLen = 0;
   int32_t  nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
   int32_t  sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps;

From 5ca1f7922b701966cb981be8523041c5ea73b4ed Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 18 Apr 2022 16:05:06 +0800
Subject: [PATCH 14/16] style: format file

---
 source/dnode/mgmt/mgmt_mnode/src/mmFile.c |  2 +-
 source/dnode/mnode/impl/src/mndBnode.c    | 10 +++++-----
 source/dnode/mnode/impl/src/mndMnode.c    | 10 +++++-----
 source/dnode/mnode/impl/src/mndQnode.c    |  8 ++++----
 source/dnode/mnode/impl/src/mndSnode.c    |  8 ++++----
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
index baf16f591d..75c48e79eb 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
@@ -121,7 +121,7 @@ int32_t mmWriteFile(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq, bool deploye
   int32_t len = 0;
   int32_t maxLen = 4096;
   char   *content = taosMemoryCalloc(1, maxLen + 1);
-  
+
   len += snprintf(content + len, maxLen - len, "{\n");
   len += snprintf(content + len, maxLen - len, "  \"mnodes\": [{\n");
 
diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c
index f7c4a6c225..168bd1db76 100644
--- a/source/dnode/mnode/impl/src/mndBnode.c
+++ b/source/dnode/mnode/impl/src/mndBnode.c
@@ -33,7 +33,7 @@ static int32_t  mndProcessCreateBnodeReq(SNodeMsg *pReq);
 static int32_t  mndProcessCreateBnodeRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessDropBnodeReq(SNodeMsg *pReq);
 static int32_t  mndProcessDropBnodeRsp(SNodeMsg *pRsp);
-static int32_t  mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextBnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitBnode(SMnode *pMnode) {
@@ -437,7 +437,7 @@ static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp) {
   return 0;
 }
 
-static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -449,8 +449,8 @@ static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pObj->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->bytes[cols]);
@@ -459,7 +459,7 @@ static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     colDataAppend(pColInfo, numOfRows, buf, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pObj->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index 6be7097e6a..0d734d8afe 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -615,7 +615,7 @@ static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp) {
   return 0;
 }
 
-static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -628,8 +628,8 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pObj->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->bytes[cols]);
@@ -638,11 +638,11 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     colDataAppend(pColInfo, numOfRows, b1, false);
 
     const char *roles = mndGetRoleStr(pObj->role);
-    char* b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE);
+    char       *b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE);
     STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->bytes[cols]);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) b2, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pObj->roleTime, false);
diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c
index 3b622795cb..96dba9baf6 100644
--- a/source/dnode/mnode/impl/src/mndQnode.c
+++ b/source/dnode/mnode/impl/src/mndQnode.c
@@ -34,7 +34,7 @@ static int32_t  mndProcessCreateQnodeRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessDropQnodeReq(SNodeMsg *pReq);
 static int32_t  mndProcessDropQnodeRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessQnodeListReq(SNodeMsg *pReq);
-static int32_t  mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextQnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitQnode(SMnode *pMnode) {
@@ -497,7 +497,7 @@ static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp) {
   return 0;
 }
 
-static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -510,8 +510,8 @@ static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*)&pObj->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]);
diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c
index 686cf1400d..b7d0ed8f5a 100644
--- a/source/dnode/mnode/impl/src/mndSnode.c
+++ b/source/dnode/mnode/impl/src/mndSnode.c
@@ -33,7 +33,7 @@ static int32_t  mndProcessCreateSnodeReq(SNodeMsg *pReq);
 static int32_t  mndProcessCreateSnodeRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessDropSnodeReq(SNodeMsg *pReq);
 static int32_t  mndProcessDropSnodeRsp(SNodeMsg *pRsp);
-static int32_t  mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextSnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitSnode(SMnode *pMnode) {
@@ -447,7 +447,7 @@ static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp) {
   return 0;
 }
 
-static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -459,8 +459,8 @@ static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*)&pObj->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]);

From 865bc40dd84d6b16ab2d981119d25419d0a9a293 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Mon, 18 Apr 2022 16:09:03 +0800
Subject: [PATCH 15/16] [test: add python test cases]

---
 tests/system-test/fulltest.sh |  17 +++
 tests/system-test/test.py     | 198 ++++++++++++++++++++++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100644 tests/system-test/fulltest.sh
 create mode 100644 tests/system-test/test.py

diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
new file mode 100644
index 0000000000..ce15beae6e
--- /dev/null
+++ b/tests/system-test/fulltest.sh
@@ -0,0 +1,17 @@
+python3 ./test.py -f 2-query/function_elapsed.py
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/system-test/test.py b/tests/system-test/test.py
new file mode 100644
index 0000000000..31afd027ec
--- /dev/null
+++ b/tests/system-test/test.py
@@ -0,0 +1,198 @@
+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/
+
+# -*- coding: utf-8 -*-
+import sys
+import getopt
+import subprocess
+import time
+from distutils.log import warn as printf
+from fabric2 import Connection
+sys.path.append("../pytest")
+from util.log import *
+from util.dnodes import *
+from util.cases import *
+
+import taos
+
+
+if __name__ == "__main__":
+    
+    fileName = "all"
+    deployPath = ""
+    masterIp = ""
+    testCluster = False
+    valgrind = 0
+    logSql = True
+    stop = 0
+    restart = False
+    windows = 0
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
+    for key, value in opts:
+        if key in ['-h', '--help']:
+            tdLog.printNoPrefix(
+                'A collection of test cases written using Python')
+            tdLog.printNoPrefix('-f Name of test case file written by Python')
+            tdLog.printNoPrefix('-p Deploy Path for Simulator')
+            tdLog.printNoPrefix('-m Master Ip for Simulator')
+            tdLog.printNoPrefix('-l <True:False> logSql Flag')
+            tdLog.printNoPrefix('-s stop All dnodes')
+            tdLog.printNoPrefix('-c Test Cluster Flag')
+            tdLog.printNoPrefix('-g valgrind Test Flag')
+            tdLog.printNoPrefix('-r taosd restart test')
+            tdLog.printNoPrefix('-w taos on windows')
+            sys.exit(0)
+
+        if key in ['-r', '--restart']: 
+            restart = True
+
+        if key in ['-f', '--file']:
+            fileName = value
+
+        if key in ['-p', '--path']:
+            deployPath = value
+
+        if key in ['-m', '--master']:
+            masterIp = value
+
+        if key in ['-l', '--logSql']:
+            if (value.upper() == "TRUE"):
+                logSql = True
+            elif (value.upper() == "FALSE"):
+                logSql = False
+            else:
+                tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
+                sys.exit(0)
+
+        if key in ['-c', '--cluster']:
+            testCluster = True
+
+        if key in ['-g', '--valgrind']:
+            valgrind = 1
+
+        if key in ['-s', '--stop']:
+            stop = 1
+
+        if key in ['-w', '--windows']:
+            windows = 1
+
+    if (stop != 0):
+        if (valgrind == 0):
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
+
+        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+        processID = subprocess.check_output(psCmd, shell=True)
+
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        for port in range(6030, 6041):
+            usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port
+            processID = subprocess.check_output(usePortPID, shell=True)
+
+            if processID:
+                killCmd = "kill -TERM %s" % processID
+                os.system(killCmd)
+            fuserCmd = "fuser -k -n tcp %d" % port
+            os.system(fuserCmd)
+        if valgrind:
+            time.sleep(2)
+
+        tdLog.info('stop All dnodes')
+    
+    if masterIp == "":
+        host = '127.0.0.1'
+    else:
+        host = masterIp
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    if windows:
+        tdCases.logSql(logSql)
+        tdLog.info("Procedures for testing self-deployment")
+        td_clinet = TDSimClient("C:\\TDengine")
+        td_clinet.deploy()
+        remote_conn = Connection("root@%s"%host)
+        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+            remote_conn.run("python3 ./test.py")
+        conn = taos.connect(
+            host="%s"%(host),
+            config=td_clinet.cfgDir)
+        tdCases.runOneWindows(conn, fileName)
+    else:
+        tdDnodes.init(deployPath)
+        tdDnodes.setTestCluster(testCluster)
+        tdDnodes.setValgrind(valgrind)
+        tdDnodes.stopAll()
+        is_test_framework = 0
+        key_word = 'tdCases.addLinux'
+        try:
+            if key_word in open(fileName).read():
+                is_test_framework = 1
+        except:
+            pass
+        if is_test_framework:
+            moduleName = fileName.replace(".py", "").replace("/", ".")
+            uModule = importlib.import_module(moduleName)
+            try:
+                ucase = uModule.TDTestCase()
+                tdDnodes.deploy(1,ucase.updatecfgDict)
+            except :
+                tdDnodes.deploy(1,{})
+        else:
+            pass
+            tdDnodes.deploy(1,{})
+        tdDnodes.start(1)
+
+
+
+        tdCases.logSql(logSql)
+
+        if testCluster:
+            tdLog.info("Procedures for testing cluster")
+            if fileName == "all":
+                tdCases.runAllCluster()
+            else:
+                tdCases.runOneCluster(fileName)
+        else:
+            tdLog.info("Procedures for testing self-deployment")
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            if fileName == "all":
+                tdCases.runAllLinux(conn)
+            else:
+                tdCases.runOneLinux(conn, fileName)
+        if restart:
+            if fileName == "all":
+                tdLog.info("not need to query ")
+            else:    
+                sp = fileName.rsplit(".", 1)
+                if len(sp) == 2 and sp[1] == "py":
+                    tdDnodes.stopAll()
+                    tdDnodes.start(1)
+                    time.sleep(1)            
+                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    tdLog.info("query test after taosd restart")
+                    tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+                else:
+                    tdLog.info("not need to query")
+    conn.close()

From 485adffc1f7be1ac3685aa002c6fd1f8d683ad14 Mon Sep 17 00:00:00 2001
From: cpwu <cpwu@taosdata.com>
Date: Mon, 18 Apr 2022 16:34:05 +0800
Subject: [PATCH 16/16] add between python case

---
 tests/system-test/2-query/between.py | 193 +++++++++++++++++++++++++++
 1 file changed, 193 insertions(+)
 create mode 100644 tests/system-test/2-query/between.py

diff --git a/tests/system-test/2-query/between.py b/tests/system-test/2-query/between.py
new file mode 100644
index 0000000000..8a5260bbaa
--- /dev/null
+++ b/tests/system-test/2-query/between.py
@@ -0,0 +1,193 @@
+import taos
+import sys
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def run(self):  # sourcery skip: extract-duplicate-method
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        tdSql.execute(
+            '''create table if not exists supt
+            (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
+            tags(location binary(64), type int, isused bool , family nchar(64))'''
+        )
+        tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')")
+        tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')")
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        for i in range(10):
+            tdSql.execute(
+                f"insert into t1 values (now()+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})"
+            )
+            tdSql.execute(
+                f"insert into t2 values (now()-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})"
+            )
+        tdSql.execute(
+            f"insert into t1 values (now()+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)"
+        )
+        tdSql.execute(
+            f"insert into t2 values (now()-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)"
+        )
+        tdSql.execute(
+            f"insert into t2 values (now()-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)"
+        )
+
+        tdLog.printNoPrefix("==========step3:query timestamp type")
+
+        tdSql.query("select * from t1 where ts between now()-1m and now()+10m")
+        tdSql.checkRows(10)
+        tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'")
+        tdSql.checkRows(11)
+        tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'")
+        tdSql.checkRows(0)
+        tdSql.query("select * from t1 where ts between -2793600 and 31507199")
+        tdSql.checkRows(0)
+        tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000")
+        tdSql.checkRows(11)
+
+        tdLog.printNoPrefix("==========step4:query int type")
+
+        tdSql.query("select * from t1 where c1 between 32767 and 32776")
+        tdSql.checkRows(10)
+        tdSql.query("select * from t1 where c1 between 32766.9 and 32776.1")
+        tdSql.checkRows(10)
+        tdSql.query("select * from t1 where c1 between 32776 and 32767")
+        tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c1 between 'a' and 'e'")
+        # tdSql.query("select * from t1 where c1 between 0x64 and 0x69")
+        # tdSql.checkRows(6)
+        tdSql.error("select * from t1 where c1 not between 100 and 106")
+        tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}")
+        tdSql.checkRows(1)
+        tdSql.error(f"select * from t2 where c1 between null and {1-2**31}")
+        # tdSql.checkRows(3)
+        tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}")
+        tdSql.checkRows(1)
+
+        tdLog.printNoPrefix("==========step5:query float type")
+
+        tdSql.query("select * from t1 where c2 between 20.0 and 21.0")
+        tdSql.checkRows(10)
+        tdSql.query(f"select * from t1 where c2 between {-3.4*10**38-1} and {3.4*10**38+1}")
+        tdSql.checkRows(11)
+        tdSql.query("select * from t1 where c2 between 21.0 and 20.0")
+        tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c2 between 'DC3' and 'SYN'")
+        tdSql.error("select * from t1 where c2 not between 0.1 and 0.2")
+        # tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}")
+        # tdSql.checkRows(1)
+        tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}")
+        tdSql.checkRows(2)
+        tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}")
+        # tdSql.checkRows(3)
+
+        tdLog.printNoPrefix("==========step6:query bigint type")
+
+        tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}")
+        tdSql.checkRows(10)
+        tdSql.error(f"select * from t1 where c3 between {-2**63} and {2**63}")
+        # tdSql.checkRows(11)
+        tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}")
+        tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c3 between 'a' and 'z'")
+        tdSql.error("select * from t1 where c3 not between 1 and 2")
+        tdSql.query(f"select * from t1 where c3 between {2**63-2} and {2**63-1}")
+        tdSql.checkRows(1)
+        tdSql.error(f"select * from t2 where c3 between {-2**63} and {1-2**63}")
+        # tdSql.checkRows(3)
+        tdSql.error(f"select * from t2 where c3 between null and {1-2**63}")
+        # tdSql.checkRows(2)
+
+        tdLog.printNoPrefix("==========step7:query double type")
+
+        tdSql.query(f"select * from t1 where c4 between {3.4*10**38} and {3.4*10**38+10}")
+        tdSql.checkRows(10)
+        tdSql.query(f"select * from t1 where c4 between {1.7*10**308+1} and {1.7*10**308+2}")
+        # 因为精度原因，在超出bigint边界后，数值不能进行准确的判断
+        # tdSql.checkRows(0)
+        tdSql.query(f"select * from t1 where c4 between {3.4*10**38+10} and {3.4*10**38}")
+        # tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c4 between 'a' and 'z'")
+        tdSql.error("select * from t1 where c4 not between 1 and 2")
+        tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}")
+        tdSql.checkRows(1)
+        tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}")
+        # tdSql.checkRows(3)
+        tdSql.error(f"select * from t2 where c4 between null and {-1.7*10**308}")
+        # tdSql.checkRows(3)
+
+        tdLog.printNoPrefix("==========step8:query smallint type")
+
+        tdSql.query("select * from t1 where c5 between 127 and 136")
+        tdSql.checkRows(10)
+        tdSql.query("select * from t1 where c5 between 126.9 and 135.9")
+        tdSql.checkRows(9)
+        tdSql.query("select * from t1 where c5 between 136 and 127")
+        tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c5 between '~' and 'ˆ'")
+        tdSql.error("select * from t1 where c5 not between 1 and 2")
+        tdSql.query("select * from t1 where c5 between 32767 and 32768")
+        tdSql.checkRows(1)
+        tdSql.query("select * from t2 where c5 between -32768 and -32767")
+        tdSql.checkRows(1)
+        tdSql.error("select * from t2 where c5 between null and -32767")
+        # tdSql.checkRows(1)
+
+        tdLog.printNoPrefix("==========step9:query tinyint type")
+
+        tdSql.query("select * from t1 where c6 between 0 and 9")
+        tdSql.checkRows(10)
+        tdSql.query("select * from t1 where c6 between -1.1 and 8.9")
+        tdSql.checkRows(9)
+        tdSql.query("select * from t1 where c6 between 9 and 0")
+        tdSql.checkRows(0)
+        tdSql.error("select * from t1 where c6 between 'NUL' and 'HT'")
+        tdSql.error("select * from t1 where c6 not between 1 and 2")
+        tdSql.query("select * from t1 where c6 between 127 and 128")
+        tdSql.checkRows(1)
+        tdSql.query("select * from t2 where c6 between -128 and -127")
+        tdSql.checkRows(1)
+        tdSql.error("select * from t2 where c6 between null and -127")
+        # tdSql.checkRows(3)
+
+        tdLog.printNoPrefix("==========step10:invalid query type")
+
+        tdSql.query("select * from supt where location between 'beijing' and 'shanghai'")
+        tdSql.checkRows(23)
+        # 非0值均解析为1，因此"between 负值 and o"解析为"between 1 and 0"
+        tdSql.query("select * from supt where isused between 0 and 1")
+        tdSql.checkRows(23)
+        tdSql.query("select * from supt where isused between -1 and 0")
+        tdSql.checkRows(0)
+        tdSql.error("select * from supt where isused between false and true")
+        tdSql.query("select * from supt where family between '拖拉机' and '自行车'")
+        tdSql.checkRows(23)
+
+        tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type")
+
+        tdSql.error("select * from t1 where c6 between 0x7f and 0x80")      # check filter HEX
+        tdSql.error("select * from t1 where c6 between 0b1 and 0b11111")    # check filter BIN
+        tdSql.error("select * from t1 where c6 between 0b1 and 0x80")
+        tdSql.error("select * from t1 where c6=0b1")
+        tdSql.error("select * from t1 where c6=0x1")
+        # 八进制数据会按照十进制数据进行判定
+        tdSql.query("select * from t1 where c6 between 01 and 0200")        # check filter OCT
+        tdSql.checkRows(10)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())