From 1df7f8d274a2c44cb7a503ea86a87fededa256b1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 6 Jun 2022 23:06:25 +0800
Subject: [PATCH 01/63] test:update some test scripts.

---
 tests/script/tsim/query/complex_having.sim | 12 ++++++------
 tests/script/tsim/query/crash_sql.sim      |  6 ++----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/script/tsim/query/complex_having.sim b/tests/script/tsim/query/complex_having.sim
index ef58228f34..6a4aa6ea28 100644
--- a/tests/script/tsim/query/complex_having.sim
+++ b/tests/script/tsim/query/complex_having.sim
@@ -105,7 +105,7 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 7 then
+if $rows != 8 then
     return -1
 endi
 
@@ -214,9 +214,9 @@ print =================== count all rows
 sql select count(c1) from stb1
 print ====> sql : select count(c1) from stb1
 print ====> rows: $data00
-if $data00 != 20 then
-  print expect 20, actual: $data00
-    return -1
+if $data00 != 17 then
+  print expect 17, actual: $data00
+  return -1
 endi
 
 #=================================================
@@ -246,7 +246,7 @@ print =================== count all rows
 sql select count(c1) from stb1
 print ====> sql : select count(c1) from stb1
 print ====> rows: $data00
-if $data00 != 20 then
+if $data00 != 17 then
   return -1
 endi
 
@@ -279,7 +279,7 @@ endi
 sql  select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> sql : select sum(c1) ,count(c1) from ct4 group by c1 having count(c7)  < 1  or  sum(c1) > 2 ;
 print ====> rows: $rows
-if $rows != 7 then
+if $rows != 8 then
     return -1
 endi
 
diff --git a/tests/script/tsim/query/crash_sql.sim b/tests/script/tsim/query/crash_sql.sim
index 44671fbb0d..b2b9239232 100644
--- a/tests/script/tsim/query/crash_sql.sim
+++ b/tests/script/tsim/query/crash_sql.sim
@@ -25,7 +25,7 @@ sql connect
 print =============== create database
 sql create database db
 sql show databases
-if $rows != 2 then
+if $rows != 3 then
   return -1
 endi
 
@@ -96,8 +96,6 @@ sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL,
 
 print ================ start query ======================
 print ================ SQL used to cause taosd or taos shell  crash
-sql select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
-
-
+sql_error select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
 
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

From aba3ea28b5f1c1eb1bee319681d221cf96c76fae Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 6 Jun 2022 23:35:25 +0800
Subject: [PATCH 02/63] refactor: do some internal refactor.

---
 source/client/inc/clientInt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 0630704d93..4ba74af5e6 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -45,7 +45,7 @@ extern "C" {
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 #define HEARTBEAT_INTERVAL         1500  // ms
-#define SYNC_ON_TOP_OF_ASYNC       0
+#define SYNC_ON_TOP_OF_ASYNC       1
 
 enum {
   RES_TYPE__QUERY = 1,

From 3e6bb0558131d91e414708a94bd1c57bf6eafdb9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 6 Jun 2022 23:39:54 +0800
Subject: [PATCH 03/63] refactor: do some internal refactor.

---
 source/client/src/clientMain.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index d192b65bf0..bcc9447fb3 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -500,7 +500,11 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
       return 0;
     }
 
-    doFetchRows(pRequest, false, true);
+#if SYNC_ON_TOP_OF_ASYNC
+    doAsyncFetchRow(pRequest, false, true);
+#else
+    doFetchRows(pRequest, true, true);
+#endif
 
     // TODO refactor
     SReqResultInfo *pResultInfo = &pRequest->body.resInfo;

From 9740ccf9fa46e7b8262b81a58e3c75cc6a7973a2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 7 Jun 2022 09:08:44 +0800
Subject: [PATCH 04/63] test:update test case.

---
 source/client/test/clientTests.cpp | 31 +-----------------------------
 1 file changed, 1 insertion(+), 30 deletions(-)

diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index f16baba0e7..853db80b09 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -772,41 +772,12 @@ create table m1(ts timestamp, k int) tags(a int);
 create table tm0 using m1 tags(1);
 create table tm1 using m1 tags(2);
 insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22);
-
  */
 TEST(testCase, async_api_test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  taos_query(pConn, "use test");
-
-  TAOS_RES* pRes = taos_query(pConn, "select * from t1");
-
-  taos_query(pConn, "alter table t1 add column b int");
-  pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed, reason:%s\n", taos_errstr(pRes));
-  }
-
-//  int32_t n = 0;
-//  TAOS_ROW    pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t     numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t* length = taos_fetch_lengths(pRes);
-//    for(int32_t i = 0; i < numOfFields; ++i) {
-//      printf("(%d):%d " , i, length[i]);
-//    }
-//    printf("\n");
-//
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
-//    memset(str, 0, sizeof(str));
-//  }
-
-  taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
+  taos_query_a(pConn, "alter database test keep 2400", queryCallback, pConn);
   getchar();
   taos_close(pConn);
 }

From 6c6259c51b8b398f6868cc90e3aee049cf668698 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 7 Jun 2022 09:27:27 +0800
Subject: [PATCH 05/63] fix: a problem of parser async

---
 source/libs/parser/src/parAstParser.c       | 6 ++++++
 source/libs/parser/test/parInitialATest.cpp | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c
index c79820a950..e6a905103a 100644
--- a/source/libs/parser/src/parAstParser.c
+++ b/source/libs/parser/src/parAstParser.c
@@ -179,6 +179,10 @@ static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* p
   return cxt.errCode;
 }
 
+static int32_t collectMetaKeyFromAlterDatabase(SCollectMetaKeyCxt* pCxt, SAlterDatabaseStmt* pStmt) {
+  return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
+}
+
 static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) {
   int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
   if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) {
@@ -376,6 +380,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromSetOperator(pCxt, (SSetOperator*)pStmt);
     case QUERY_NODE_SELECT_STMT:
       return collectMetaKeyFromSelect(pCxt, (SSelectStmt*)pStmt);
+    case QUERY_NODE_ALTER_DATABASE_STMT:
+      return collectMetaKeyFromAlterDatabase(pCxt, (SAlterDatabaseStmt*)pStmt);
     case QUERY_NODE_CREATE_TABLE_STMT:
       return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
     case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp
index 784586dfb2..595aa049fd 100644
--- a/source/libs/parser/test/parInitialATest.cpp
+++ b/source/libs/parser/test/parInitialATest.cpp
@@ -39,6 +39,8 @@ TEST_F(ParserInitialATest, alterDatabase) {
   useDb("root", "test");
 
   run("ALTER DATABASE wxy_db CACHELAST 1 FSYNC 200 WAL 1");
+
+  run("ALTER DATABASE wxy_db KEEP 2400");
 }
 
 // todo ALTER local

From fb07e8b73eb67bfc401379f6cbb1c6437e26943b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 7 Jun 2022 11:15:33 +0800
Subject: [PATCH 06/63] refactor:do some internal refactor and add some logs.

---
 source/client/src/clientImpl.c     |  3 ++-
 source/client/src/clientMain.c     | 11 +++++++----
 source/libs/catalog/src/ctgAsync.c |  4 ++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index c71c7d3487..7601e082e4 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -580,7 +580,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
-    tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+    tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+        pRequest->retry, pRequest->requestId);
     pRequest->prevCode = code;
     doAsyncQuery(pRequest, true);
     return;
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index bcc9447fb3..3ca93d178c 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -629,8 +629,10 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) {
     taosMemoryFree(pWrapper);
     launchAsyncQuery(pRequest, pQuery);
   } else {
+    tscDebug("error happens, code:%d", code);
     if (NEED_CLIENT_HANDLE_ERROR(code)) {
-      tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+      tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+               pRequest->retry, pRequest->requestId);
       pRequest->prevCode = code;
       doAsyncQuery(pRequest, true);
       return;
@@ -703,13 +705,14 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
 void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   SParseContext* pCxt = NULL;
   STscObj *pTscObj = pRequest->pTscObj;
+  int32_t code = 0;
 
   if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
-    pRequest->code = pRequest->prevCode;
+    code = pRequest->prevCode;
     goto _error;
   }
 
-  int32_t code = createParseContext(pRequest, &pCxt);
+  code = createParseContext(pRequest, &pCxt);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -746,7 +749,7 @@ void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   }
 
   _error:
-  tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+  tscError("0x%"PRIx64" error happens, code:%d - %s, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId);
   terrno = code;
   pRequest->code = code;
   pRequest->body.queryFp(pRequest->body.param, pRequest, code);
diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c
index f43e559244..63df81e73c 100644
--- a/source/libs/catalog/src/ctgAsync.c
+++ b/source/libs/catalog/src/ctgAsync.c
@@ -559,7 +559,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
   SCtgJob* pJob = pTask->pJob;
   int32_t code = 0;
 
-  qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
+  qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
 
   if (rspCode) {
     int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
@@ -572,7 +572,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
 
   int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
   if (taskDone < taosArrayGetSize(pJob->pTasks)) {
-    qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
+    qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
     return TSDB_CODE_SUCCESS;
   }
 

From 8a00b3ab86bdf9a5fd0ff85749778208cc72e973 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 7 Jun 2022 11:43:25 +0800
Subject: [PATCH 07/63] test: update test cases.

---
 source/client/test/clientTests.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index 853db80b09..0950fc3923 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -772,12 +772,13 @@ create table m1(ts timestamp, k int) tags(a int);
 create table tm0 using m1 tags(1);
 create table tm1 using m1 tags(2);
 insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22);
+
  */
 TEST(testCase, async_api_test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  taos_query_a(pConn, "alter database test keep 2400", queryCallback, pConn);
+  taos_query_a(pConn, "insert into tm0 values(now()+0s, 1)", queryCallback, pConn);
   getchar();
   taos_close(pConn);
 }

From ac0c75ab2dd76d2dddf5e5dc35473b45a6affe50 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 7 Jun 2022 13:28:33 +0800
Subject: [PATCH 08/63] fix: a problem of parser async

---
 source/libs/parser/src/parAstParser.c |  3 ++-
 source/libs/parser/src/parInsert.c    | 11 +++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c
index e6a905103a..f05987c307 100644
--- a/source/libs/parser/src/parAstParser.c
+++ b/source/libs/parser/src/parAstParser.c
@@ -110,7 +110,8 @@ static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFu
   if (fmIsBuiltinFunc(pFunc->functionName)) {
     return TSDB_CODE_SUCCESS;
   }
-  return reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
+  pCxt->errCode = reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
+  return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
 }
 
 static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTableNode* pRealTable,
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 8f17d500ab..2cfc06b575 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -1006,7 +1006,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
     }
 
     SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]];
-    char*    tmpTokenBuf = taosMemoryCalloc(1, sToken.n);  //todo this can be optimize with parse column
+    char*    tmpTokenBuf = taosMemoryCalloc(1, sToken.n);  // todo this can be optimize with parse column
     code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg);
     if (code != TSDB_CODE_SUCCESS) {
       taosMemoryFree(tmpTokenBuf);
@@ -1018,7 +1018,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
         taosMemoryFree(tmpTokenBuf);
         goto end;
       }
-      if(isNullStr(&sToken)) {
+      if (isNullStr(&sToken)) {
         code = tTagNew(pTagVals, 1, true, &pTag);
       } else {
         code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
@@ -1530,10 +1530,13 @@ typedef struct SInsertParseSyntaxCxt {
 } SInsertParseSyntaxCxt;
 
 static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) {
-  SToken sToken;
+  SToken  sToken;
+  int32_t expectRightParenthesis = 1;
   while (1) {
     NEXT_TOKEN(pCxt->pSql, sToken);
-    if (TK_NK_RP == sToken.type) {
+    if (TK_NK_LP == sToken.type) {
+      ++expectRightParenthesis;
+    } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) {
       break;
     }
     if (0 == sToken.n) {

From cd90a9fa302837bf9d980d3af8beb63634a818bb Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 7 Jun 2022 14:26:18 +0800
Subject: [PATCH 09/63] add error code for each task

---
 include/libs/catalog/catalog.h       | 23 ++++++-----
 source/libs/catalog/inc/catalogInt.h | 12 +++---
 source/libs/catalog/src/ctgAsync.c   | 59 ++++++++++++++++------------
 source/libs/catalog/src/ctgUtil.c    | 10 ++++-
 4 files changed, 61 insertions(+), 43 deletions(-)

diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h
index ee237741c3..567719e3f0 100644
--- a/include/libs/catalog/catalog.h
+++ b/include/libs/catalog/catalog.h
@@ -71,16 +71,21 @@ typedef struct SCatalogReq {
   bool    forceUpdate;    
 } SCatalogReq;
 
+typedef struct SMetaRes {
+  int32_t code;
+  void*   pRes;
+} SMetaRes;
+
 typedef struct SMetaData {
-  SArray    *pDbVgroup;   // SArray<SArray<SVgroupInfo>*>
-  SArray    *pDbCfg;      // SArray<SDbCfgInfo>
-  SArray    *pDbInfo;     // SArray<SDbInfo>
-  SArray    *pTableMeta;  // SArray<STableMeta*>
-  SArray    *pTableHash;  // SArray<SVgroupInfo>
-  SArray    *pUdfList;    // SArray<SFuncInfo>
-  SArray    *pIndex;      // SArray<SIndexInfo>
-  SArray    *pUser;       // SArray<bool>
-  SArray    *pQnodeList;  // SArray<SQueryNodeAddr>
+  SArray    *pDbVgroup;   // pRes = SArray<SVgroupInfo>*
+  SArray    *pDbCfg;      // pRes = SDbCfgInfo*
+  SArray    *pDbInfo;     // pRes = SDbInfo*
+  SArray    *pTableMeta;  // pRes = STableMeta*
+  SArray    *pTableHash;  // pRes = SVgroupInfo*
+  SArray    *pUdfList;    // pRes = SFuncInfo*
+  SArray    *pIndex;      // pRes = SIndexInfo*
+  SArray    *pUser;       // pRes = bool*
+  SArray    *pQnodeList;  // pRes = SQueryNodeAddr*
 } SMetaData;
 
 typedef struct SCatalogCfg {
diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h
index 1eec92f47e..0ea45cdd7e 100644
--- a/source/libs/catalog/inc/catalogInt.h
+++ b/source/libs/catalog/inc/catalogInt.h
@@ -173,7 +173,6 @@ typedef struct SCtgJob {
   SArray*          pTasks;
   int32_t          taskDone;
   SMetaData        jobRes;
-  int32_t          rspCode;
 
   uint64_t         queryId;
   SCatalog*        pCtg; 
@@ -201,11 +200,12 @@ typedef struct SCtgMsgCtx {
 
 typedef struct SCtgTask {
   CTG_TASK_TYPE type;
-  int32_t  taskId;
-  SCtgJob *pJob;
-  void* taskCtx;
-  SCtgMsgCtx msgCtx;
-  void* res;
+  int32_t       taskId;
+  SCtgJob*      pJob;
+  void*         taskCtx;
+  SCtgMsgCtx    msgCtx;
+  int32_t       code;
+  void*         res;
 } SCtgTask;
 
 typedef int32_t (*ctgLanchTaskFp)(SCtgTask*);
diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c
index 63df81e73c..bc2c8dad7a 100644
--- a/source/libs/catalog/src/ctgAsync.c
+++ b/source/libs/catalog/src/ctgAsync.c
@@ -437,13 +437,14 @@ _return:
 int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableMeta) {
-    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, POINTER_BYTES);
+    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableMeta) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableMeta, &pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableMeta, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -451,14 +452,14 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
 int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbVgroup) {
-    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES);
+    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbVgroup) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res);
-  pTask->res = NULL;
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbVgroup, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -466,13 +467,14 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
 int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableHash) {
-    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo));
+    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableHash) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableHash, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableHash, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -480,21 +482,29 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
 int32_t ctgDumpIndexRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pIndex) {
-    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo));
+    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pIndex) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pIndex, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pIndex, &res);
 
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
+  if (NULL == pJob->jobRes.pQnodeList) {
+    pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
+    if (NULL == pJob->jobRes.pQnodeList) {
+      CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
 
-  TSWAP(pJob->jobRes.pQnodeList, pTask->res); 
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pQnodeList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -502,13 +512,14 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
 int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbCfg) {
-    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo));
+    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbCfg) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbCfg, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbCfg, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -516,13 +527,14 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
 int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbInfo) {
-    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SDbInfo));
+    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbInfo) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbInfo, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbInfo, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -530,13 +542,14 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
 int32_t ctgDumpUdfRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUdfList) {
-    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo));
+    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUdfList) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUdfList, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUdfList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -544,13 +557,14 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
 int32_t ctgDumpUserRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUser) {
-    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool));
+    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUser) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUser, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUser, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -561,14 +575,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
 
   qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
 
-  if (rspCode) {
-    int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
-    if (0 == lastCode) {
-      CTG_ERR_JRET(rspCode);
-    }
-
-    return TSDB_CODE_SUCCESS;  
-  }
+  pTask->code = rspCode;
 
   int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
   if (taskDone < taosArrayGetSize(pJob->pTasks)) {
diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c
index 962edf9af1..558d1eb8d6 100644
--- a/source/libs/catalog/src/ctgUtil.c
+++ b/source/libs/catalog/src/ctgUtil.c
@@ -61,10 +61,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pUdfList);
   pData->pUdfList = NULL;
 
+/*
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
     SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
     taosArrayDestroy(pInfo->pRetensions);
   }
+*/  
   taosArrayDestroy(pData->pDbCfg);
   pData->pDbCfg = NULL;
 
@@ -320,8 +322,12 @@ void ctgFreeTask(SCtgTask* pTask) {
       break;
     }
     case CTG_TASK_GET_DB_CFG: {
-      taosMemoryFreeClear(pTask->taskCtx);      
-      taosMemoryFreeClear(pTask->res);
+      taosMemoryFreeClear(pTask->taskCtx);
+      if (pTask->res) {
+        SDbCfgInfo* pInfo = (SDbCfgInfo*)pTask->res;
+        taosArrayDestroy(pInfo->pRetensions);
+        taosMemoryFreeClear(pTask->res);
+      }
       break;
     }
     case CTG_TASK_GET_DB_INFO: {

From d98a4b467db365108a66c2a6da9a20bb23d7e4b1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 7 Jun 2022 15:52:46 +0800
Subject: [PATCH 10/63] fix(query): set the schemaless flag.

---
 source/client/inc/clientInt.h  | 2 +-
 source/client/src/clientMain.c | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 4ba74af5e6..3a6f3badc0 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -144,7 +144,7 @@ typedef struct STscObj {
   int32_t       numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo* pAppInfo;
   SHashObj*     pRequests;
-  int8_t        schemalessType;
+  int8_t        schemalessType;  // todo remove it, this attribute should be move to request
 } STscObj;
 
 typedef struct SResultColumn {
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 3ca93d178c..1a1925e244 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -697,6 +697,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
                        .pTransporter = pTscObj->pAppInfo->pTransporter,
                        .pStmtCb = NULL,
                        .pUser = pTscObj->user,
+                       .schemalessType = pTscObj->schemalessType,
                        .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                        .async = true,};
   return TSDB_CODE_SUCCESS;

From 4ab552234bb732ef932d2b7900c0c518347ff2c2 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 7 Jun 2022 17:14:20 +0800
Subject: [PATCH 11/63] fix: a problem of parser async

---
 source/libs/parser/src/parAstParser.c         |   2 +-
 source/libs/parser/src/parUtil.c              | 198 ++++++++----------
 .../libs/parser/test/mockCatalogService.cpp   |  75 +++----
 3 files changed, 118 insertions(+), 157 deletions(-)

diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c
index f05987c307..6555ec3a7d 100644
--- a/source/libs/parser/src/parAstParser.c
+++ b/source/libs/parser/src/parAstParser.c
@@ -108,7 +108,7 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt);
 
 static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFunctionNode* pFunc) {
   if (fmIsBuiltinFunc(pFunc->functionName)) {
-    return TSDB_CODE_SUCCESS;
+    return DEAL_RES_CONTINUE;
   }
   pCxt->errCode = reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
   return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index f30cc1111f..192a4baf80 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -482,20 +482,6 @@ static int32_t buildDbReq(SHashObj* pDbsHash, SArray** pDbs) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildTableMetaReq(SHashObj* pTableMetaHash, SArray** pTableMeta) {
-  return buildTableReq(pTableMetaHash, pTableMeta);
-}
-
-static int32_t buildDbVgroupReq(SHashObj* pDbVgroupHash, SArray** pDbVgroup) {
-  return buildDbReq(pDbVgroupHash, pDbVgroup);
-}
-
-static int32_t buildTableVgroupReq(SHashObj* pTableVgroupHash, SArray** pTableVgroup) {
-  return buildTableReq(pTableVgroupHash, pTableVgroup);
-}
-
-static int32_t buildDbCfgReq(SHashObj* pDbCfgHash, SArray** pDbCfg) { return buildDbReq(pDbCfgHash, pDbCfg); }
-
 static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) {
   if (NULL != pUserAuthHash) {
     *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo));
@@ -535,15 +521,18 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) {
 }
 
 int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
-  int32_t code = buildTableMetaReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
+  int32_t code = buildTableReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildDbVgroupReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup);
+    code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildTableVgroupReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash);
+    code = buildTableReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildDbCfgReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg);
+    code = buildDbReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildDbReq(pMetaCache->pDbInfo, &pCatalogReq->pDbInfo);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = buildUserAuthReq(pMetaCache->pUserAuth, &pCatalogReq->pUser);
@@ -554,51 +543,39 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
   return code;
 }
 
-static int32_t putTableMetaToCache(const SArray* pTableMetaReq, const SArray* pTableMetaData, SHashObj* pTableMeta) {
-  int32_t ntables = taosArrayGetSize(pTableMetaReq);
+static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj* pHash) {
+  SMetaRes* pRes = taosArrayGet(pData, index);
+  return taosHashPut(pHash, pKey, len, &pRes, POINTER_BYTES);
+}
+
+static int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput) {
+  SMetaRes** pRes = taosHashGet(pHash, pKey, len);
+  if (NULL == pRes || NULL == *pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  if (TSDB_CODE_SUCCESS == (*pRes)->code) {
+    *pOutput = (*pRes)->pRes;
+  }
+  return (*pRes)->code;
+}
+
+static int32_t putTableDataToCache(const SArray* pTableReq, const SArray* pTableData, SHashObj* pTable) {
+  int32_t ntables = taosArrayGetSize(pTableReq);
   for (int32_t i = 0; i < ntables; ++i) {
     char fullName[TSDB_TABLE_FNAME_LEN];
-    tNameExtractFullName(taosArrayGet(pTableMetaReq, i), fullName);
-    if (TSDB_CODE_SUCCESS !=
-        taosHashPut(pTableMeta, fullName, strlen(fullName), taosArrayGet(pTableMetaData, i), POINTER_BYTES)) {
+    tNameExtractFullName(taosArrayGet(pTableReq, i), fullName);
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(fullName, strlen(fullName), pTableData, i, pTable)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t putDbVgroupToCache(const SArray* pDbVgroupReq, const SArray* pDbVgroupData, SHashObj* pDbVgroup) {
-  int32_t nvgs = taosArrayGetSize(pDbVgroupReq);
+static int32_t putDbDataToCache(const SArray* pDbReq, const SArray* pDbData, SHashObj* pDb) {
+  int32_t nvgs = taosArrayGetSize(pDbReq);
   for (int32_t i = 0; i < nvgs; ++i) {
-    char* pDbFName = taosArrayGet(pDbVgroupReq, i);
-    if (TSDB_CODE_SUCCESS !=
-        taosHashPut(pDbVgroup, pDbFName, strlen(pDbFName), taosArrayGet(pDbVgroupData, i), POINTER_BYTES)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t putTableVgroupToCache(const SArray* pTableVgroupReq, const SArray* pTableVgroupData,
-                                     SHashObj* pTableVgroup) {
-  int32_t ntables = taosArrayGetSize(pTableVgroupReq);
-  for (int32_t i = 0; i < ntables; ++i) {
-    char fullName[TSDB_TABLE_FNAME_LEN];
-    tNameExtractFullName(taosArrayGet(pTableVgroupReq, i), fullName);
-    SVgroupInfo* pInfo = taosArrayGet(pTableVgroupData, i);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pTableVgroup, fullName, strlen(fullName), &pInfo, POINTER_BYTES)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t putDbCfgToCache(const SArray* pDbCfgReq, const SArray* pDbCfgData, SHashObj* pDbCfg) {
-  int32_t nvgs = taosArrayGetSize(pDbCfgReq);
-  for (int32_t i = 0; i < nvgs; ++i) {
-    char*       pDbFName = taosArrayGet(pDbCfgReq, i);
-    SDbCfgInfo* pInfo = taosArrayGet(pDbCfgData, i);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pDbCfg, pDbFName, strlen(pDbFName), &pInfo, POINTER_BYTES)) {
+    char* pDbFName = taosArrayGet(pDbReq, i);
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(pDbFName, strlen(pDbFName), pDbData, i, pDb)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -611,7 +588,7 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse
     SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i);
     char           key[USER_AUTH_KEY_MAX_LEN] = {0};
     int32_t        len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pUserAuth, key, len, taosArrayGet(pUserAuthData, i), sizeof(bool))) {
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -621,9 +598,8 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse
 static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHashObj* pUdf) {
   int32_t num = taosArrayGetSize(pUdfReq);
   for (int32_t i = 0; i < num; ++i) {
-    char*      pFunc = taosArrayGet(pUdfReq, i);
-    SFuncInfo* pInfo = taosArrayGet(pUdfData, i);
-    if (TSDB_CODE_SUCCESS != taosHashPut(pUdf, pFunc, strlen(pFunc), &pInfo, POINTER_BYTES)) {
+    char* pFunc = taosArrayGet(pUdfReq, i);
+    if (TSDB_CODE_SUCCESS != putMetaDataToHash(pFunc, strlen(pFunc), pUdfData, i, pUdf)) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -631,15 +607,18 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas
 }
 
 int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
-  int32_t code = putTableMetaToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, pMetaCache->pTableMeta);
+  int32_t code = putTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, pMetaCache->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
-    code = putDbVgroupToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, pMetaCache->pDbVgroup);
+    code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, pMetaCache->pDbVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = putTableVgroupToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, pMetaCache->pTableVgroup);
+    code = putTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, pMetaCache->pTableVgroup);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = putDbCfgToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, pMetaCache->pDbCfg);
+    code = putDbDataToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, pMetaCache->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = putDbDataToCache(pCatalogReq->pDbInfo, pMetaData->pDbInfo, pMetaCache->pDbInfo);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = putUserAuthToCache(pCatalogReq->pUser, pMetaData->pUser, pMetaCache->pUserAuth);
@@ -679,16 +658,15 @@ int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCac
 int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  STableMeta** pRes = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName));
-  if (NULL == pRes || NULL == *pRes) {
-    parserError("getTableMetaFromCache error: %s", fullName);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  STableMeta* pTableMeta = NULL;
+  int32_t     code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableMeta, (void**)&pTableMeta);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pMeta = tableMetaDup(pTableMeta);
+    if (NULL == *pMeta) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
   }
-  *pMeta = tableMetaDup(*pRes);
-  if (NULL == *pMeta) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** pDbs) {
@@ -708,19 +686,16 @@ int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache*
 }
 
 int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) {
-  SArray** pRes = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName));
-  if (NULL == pRes) {
-    parserError("getDbVgInfoFromCache error: %s", pDbFName);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
-  }
-  // *pRes is null, which is a legal value, indicating that the user DB has not been created
-  if (NULL != *pRes) {
-    *pVgInfo = taosArrayDup(*pRes);
+  SArray* pVgList = NULL;
+  int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbVgroup, (void**)&pVgList);
+  // pVgList is null, which is a legal value, indicating that the user DB has not been created
+  if (TSDB_CODE_SUCCESS == code && NULL != pVgList) {
+    *pVgInfo = taosArrayDup(pVgList);
     if (NULL == *pVgInfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+      code = TSDB_CODE_OUT_OF_MEMORY;
     }
   }
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) {
@@ -736,30 +711,28 @@ int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaC
 int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  SVgroupInfo** pRes = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName));
-  if (NULL == pRes || NULL == *pRes) {
-    parserError("getTableVgroupFromCache error: %s", fullName);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  SVgroupInfo* pVg = NULL;
+  int32_t      code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableVgroup, (void**)&pVg);
+  if (TSDB_CODE_SUCCESS == code) {
+    memcpy(pVgroup, pVg, sizeof(SVgroupInfo));
   }
-  memcpy(pVgroup, *pRes, sizeof(SVgroupInfo));
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
-  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbCfg);
+  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbInfo);
 }
 
 int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                                 int32_t* pTableNum) {
-  SDbInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
-  if (NULL == pRes || NULL == *pRes) {
-    parserError("getDbVgVersionFromCache error: %s", pDbFName);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  SDbInfo* pDbInfo = NULL;
+  int32_t  code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pVersion = pDbInfo->vgVer;
+    *pDbId = pDbInfo->dbId;
+    *pTableNum = pDbInfo->tbNum;
   }
-  *pVersion = (*pRes)->vgVer;
-  *pDbId = (*pRes)->dbId;
-  *pTableNum = (*pRes)->tbNum;
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
@@ -767,13 +740,12 @@ int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pM
 }
 
 int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) {
-  SDbCfgInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
-  if (NULL == pRes || NULL == *pRes) {
-    parserError("getDbCfgFromCache error: %s", pDbFName);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  SDbCfgInfo* pDbCfg = NULL;
+  int32_t     code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbCfg, (void**)&pDbCfg);
+  if (TSDB_CODE_SUCCESS == code) {
+    memcpy(pInfo, pDbCfg, sizeof(SDbCfgInfo));
   }
-  memcpy(pInfo, *pRes, sizeof(SDbCfgInfo));
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseMetaCache* pMetaCache) {
@@ -806,13 +778,12 @@ int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, con
                              bool* pPass) {
   char    key[USER_AUTH_KEY_MAX_LEN] = {0};
   int32_t len = userAuthToStringExt(pUser, pDbFName, type, key);
-  bool*   pRes = taosHashGet(pMetaCache->pUserAuth, key, len);
-  if (NULL == pRes) {
-    parserError("getUserAuthFromCache error: %s, %s, %d", pUser, pDbFName, type);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  bool*   pRes = NULL;
+  int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pRes);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pPass = *pRes;
   }
-  *pPass = *pRes;
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) {
@@ -826,11 +797,10 @@ int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) {
 }
 
 int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo) {
-  SFuncInfo** pRes = taosHashGet(pMetaCache->pUdf, pFunc, strlen(pFunc));
-  if (NULL == pRes || NULL == *pRes) {
-    parserError("getUdfInfoFromCache error: %s", pFunc);
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  SFuncInfo* pFuncInfo = NULL;
+  int32_t    code = getMetaDataFromHash(pFunc, strlen(pFunc), pMetaCache->pUdf, (void**)&pFuncInfo);
+  if (TSDB_CODE_SUCCESS == code) {
+    memcpy(pInfo, pFuncInfo, sizeof(SFuncInfo));
   }
-  memcpy(pInfo, *pRes, sizeof(SFuncInfo));
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp
index 4834d2d377..0c37c875c0 100644
--- a/source/libs/parser/test/mockCatalogService.cpp
+++ b/source/libs/parser/test/mockCatalogService.cpp
@@ -367,49 +367,40 @@ class MockCatalogServiceImpl {
   }
 
   int32_t getAllTableMeta(SArray* pTableMetaReq, SArray** pTableMetaData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pTableMetaReq) {
       int32_t ntables = taosArrayGetSize(pTableMetaReq);
-      *pTableMetaData = taosArrayInit(ntables, POINTER_BYTES);
+      *pTableMetaData = taosArrayInit(ntables, sizeof(SMetaRes));
       for (int32_t i = 0; i < ntables; ++i) {
-        STableMeta* pMeta = NULL;
-        code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), &pMeta);
-        if (TSDB_CODE_SUCCESS == code) {
-          taosArrayPush(*pTableMetaData, &pMeta);
-        } else {
-          break;
-        }
+        SMetaRes res = {0};
+        res.code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), (STableMeta**)&res.pRes);
+        taosArrayPush(*pTableMetaData, &res);
       }
     }
-    return code;
+    return TSDB_CODE_SUCCESS;
   }
 
   int32_t getAllTableVgroup(SArray* pTableVgroupReq, SArray** pTableVgroupData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pTableVgroupReq) {
       int32_t ntables = taosArrayGetSize(pTableVgroupReq);
-      *pTableVgroupData = taosArrayInit(ntables, sizeof(SVgroupInfo));
+      *pTableVgroupData = taosArrayInit(ntables, sizeof(SMetaRes));
       for (int32_t i = 0; i < ntables; ++i) {
-        SVgroupInfo vgInfo = {0};
-        code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), &vgInfo);
-        if (TSDB_CODE_SUCCESS == code) {
-          taosArrayPush(*pTableVgroupData, &vgInfo);
-        } else {
-          break;
-        }
+        SMetaRes res = {0};
+        res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo));
+        res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), (SVgroupInfo*)res.pRes);
+        taosArrayPush(*pTableVgroupData, &res);
       }
     }
-    return code;
+    return TSDB_CODE_SUCCESS;
   }
 
   int32_t getAllDbVgroup(SArray* pDbVgroupReq, SArray** pDbVgroupData) const {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbVgroupReq) {
       int32_t ndbs = taosArrayGetSize(pDbVgroupReq);
-      *pDbVgroupData = taosArrayInit(ndbs, POINTER_BYTES);
+      *pDbVgroupData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        int64_t zeroVg = 0;
-        taosArrayPush(*pDbVgroupData, &zeroVg);
+        SMetaRes res = {0};
+        taosArrayPush(*pDbVgroupData, &res);
       }
     }
     return code;
@@ -419,10 +410,11 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbCfgReq) {
       int32_t ndbs = taosArrayGetSize(pDbCfgReq);
-      *pDbCfgData = taosArrayInit(ndbs, sizeof(SDbCfgInfo));
+      *pDbCfgData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        SDbCfgInfo dbCfg = {0};
-        taosArrayPush(*pDbCfgData, &dbCfg);
+        SMetaRes res = {0};
+        res.pRes = taosMemoryCalloc(1, sizeof(SDbCfgInfo));
+        taosArrayPush(*pDbCfgData, &res);
       }
     }
     return code;
@@ -432,10 +424,11 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pDbInfoReq) {
       int32_t ndbs = taosArrayGetSize(pDbInfoReq);
-      *pDbInfoData = taosArrayInit(ndbs, sizeof(SDbCfgInfo));
+      *pDbInfoData = taosArrayInit(ndbs, sizeof(SMetaRes));
       for (int32_t i = 0; i < ndbs; ++i) {
-        SDbInfo dbInfo = {0};
-        taosArrayPush(*pDbInfoData, &dbInfo);
+        SMetaRes res = {0};
+        res.pRes = taosMemoryCalloc(1, sizeof(SDbInfo));
+        taosArrayPush(*pDbInfoData, &res);
       }
     }
     return code;
@@ -445,31 +438,29 @@ class MockCatalogServiceImpl {
     int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pUserAuthReq) {
       int32_t num = taosArrayGetSize(pUserAuthReq);
-      *pUserAuthData = taosArrayInit(num, sizeof(bool));
+      *pUserAuthData = taosArrayInit(num, sizeof(SMetaRes));
       for (int32_t i = 0; i < num; ++i) {
-        bool pass = true;
-        taosArrayPush(*pUserAuthData, &pass);
+        SMetaRes res = {0};
+        res.pRes = taosMemoryCalloc(1, sizeof(bool));
+        *(bool*)(res.pRes) = true;
+        taosArrayPush(*pUserAuthData, &res);
       }
     }
     return code;
   }
 
   int32_t getAllUdf(SArray* pUdfReq, SArray** pUdfData) const {
-    int32_t code = TSDB_CODE_SUCCESS;
     if (NULL != pUdfReq) {
       int32_t num = taosArrayGetSize(pUdfReq);
-      *pUdfData = taosArrayInit(num, sizeof(SFuncInfo));
+      *pUdfData = taosArrayInit(num, sizeof(SMetaRes));
       for (int32_t i = 0; i < num; ++i) {
-        SFuncInfo info = {0};
-        code = catalogGetUdfInfo((char*)taosArrayGet(pUdfReq, i), &info);
-        if (TSDB_CODE_SUCCESS == code) {
-          taosArrayPush(*pUdfData, &info);
-        } else {
-          break;
-        }
+        SMetaRes res = {0};
+        res.pRes = taosMemoryCalloc(1, sizeof(SFuncInfo));
+        res.code = catalogGetUdfInfo((char*)taosArrayGet(pUdfReq, i), (SFuncInfo*)res.pRes);
+        taosArrayPush(*pUdfData, &res);
       }
     }
-    return code;
+    return TSDB_CODE_SUCCESS;
   }
 
   uint64_t                      id_;

From aecc8ce9bcdf97bce2f0d9eed9e4ea9b1a8b351f Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 08:48:29 +0800
Subject: [PATCH 12/63] fix double free issue

---
 source/libs/catalog/src/ctgUtil.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c
index 558d1eb8d6..b5160fe8de 100644
--- a/source/libs/catalog/src/ctgUtil.c
+++ b/source/libs/catalog/src/ctgUtil.c
@@ -47,11 +47,13 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
 void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pTableMeta);
   pData->pTableMeta = NULL;
-  
+
+/*  
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
     SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
     taosArrayDestroy(*pArray);
   }
+*/
   taosArrayDestroy(pData->pDbVgroup);
   pData->pDbVgroup = NULL;
   

From f5da576a898629a8181bd2a3e953879a0afa430a Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 08:57:20 +0800
Subject: [PATCH 13/63] fix in compare issue

---
 source/libs/parser/src/parInsert.c     |  4 ++++
 source/libs/parser/src/parTranslater.c | 23 ++++++++++++++++++-----
 source/libs/qworker/src/qwDbg.c        | 14 +++++++++-----
 source/libs/scalar/src/filter.c        |  8 ++++++++
 4 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 8f17d500ab..434667e179 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -2040,6 +2040,10 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA
     return TSDB_CODE_QRY_APP_ERROR;
   }
 
+  if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) {
+    return TSDB_CODE_TSC_STMT_API_ERROR;
+  }
+
   SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
   if (tags->numOfBound <= 0) {
     *fieldNum = 0;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1ba47d0d15..d34dd81d4f 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -817,6 +817,10 @@ static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNo
   return DEAL_RES_CONTINUE;
 }
 
+static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
+  return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale);
+}
+
 static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
   SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
@@ -824,7 +828,20 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
   if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) {
-    ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType;
+    SNodeListNode* pRight = (SNodeListNode*)pOp->pRight;
+    bool first = true;
+    SDataType targetDt = {0};
+    SNode* pNode = NULL;
+    FOREACH(pNode, pRight->pNodeList) {
+      SDataType dt = ((SExprNode*)pNode)->resType;
+      if (first) {
+        targetDt = dt;
+        first = false;
+      } else if (!dataTypeEqual(&dt, &targetDt)) {
+        return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName);
+      }
+    }
+    pRight->dataType = targetDt;
   }
   if (nodesIsRegularOp(pOp)) {
     if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
@@ -2015,10 +2032,6 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
   return (SNode*)pCol;
 }
 
-static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
-  return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale);
-}
-
 static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) {
   SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
   if (NULL == pFunc) {
diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c
index 3914541157..368c3bb517 100644
--- a/source/libs/qworker/src/qwDbg.c
+++ b/source/libs/qworker/src/qwDbg.c
@@ -9,7 +9,7 @@
 #include "tmsg.h"
 #include "tname.h"
 
-SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = true};
+SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = false};
 
 int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) {
   if (!gQWDebug.statusEnable) {
@@ -97,7 +97,11 @@ _return:
   QW_RET(code);
 }
 
-void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {}
+void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) {
+  QW_LOCK(QW_READ, &sch->tasksLock);
+  QW_DLOG("the %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taosHashGetSize(sch->tasksHash));
+  QW_UNLOCK(QW_READ, &sch->tasksLock);
+}
 
 void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
   if (!gQWDebug.dumpEnable) {
@@ -106,7 +110,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
 
   QW_LOCK(QW_READ, &mgmt->schLock);
 
-  /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/
+  QW_DUMP("total remain scheduler num %d", taosHashGetSize(mgmt->schHash));
 
   void         *key = NULL;
   size_t        keyLen = 0;
@@ -116,14 +120,14 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
   void *pIter = taosHashIterate(mgmt->schHash, NULL);
   while (pIter) {
     sch = (SQWSchStatus *)pIter;
-    qwDbgDumpSchInfo(sch, i);
+    qwDbgDumpSchInfo(mgmt, sch, i);
     ++i;
     pIter = taosHashIterate(mgmt->schHash, pIter);
   }
 
   QW_UNLOCK(QW_READ, &mgmt->schLock);
 
-  /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/
+  QW_DUMP("total remain ctx num %d", taosHashGetSize(mgmt->ctxHash));
 }
 
 
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index 12e9ec5aea..6704bfdde3 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -3669,6 +3669,14 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
           stat->scalarMode = true;
           return DEAL_RES_CONTINUE;
         }
+      } else {
+        SColumnNode *refNode = (SColumnNode *)node->pLeft;
+        SNodeListNode *listNode = (SNodeListNode *)node->pRight;
+        int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type);
+        if (0 != type && type != refNode->node.resType.type) {
+          stat->scalarMode = true;
+          return DEAL_RES_CONTINUE;
+        }
       }
     }
 

From a552e1fcca580e645dd1bb10e978c2d4173f3d0b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 8 Jun 2022 09:30:59 +0800
Subject: [PATCH 14/63] test(query): fix a typo.

---
 source/libs/scheduler/src/schJob.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c
index 7a0272a4d0..bf14032669 100644
--- a/source/libs/scheduler/src/schJob.c
+++ b/source/libs/scheduler/src/schJob.c
@@ -535,7 +535,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
         SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
 
-      SCH_TASK_DLOG("set %dth condidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
+      SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
 
       ++addNum;
     }

From 8dcd05301eb9fc84093d9a829443c4b5269ec274 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 09:47:50 +0800
Subject: [PATCH 15/63] add assert

---
 source/libs/parser/src/parUtil.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 192a4baf80..e5902dff1d 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -714,6 +714,8 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName,
   SVgroupInfo* pVg = NULL;
   int32_t      code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableVgroup, (void**)&pVg);
   if (TSDB_CODE_SUCCESS == code) {
+    ASSERT(0 != pVg->vgId);
+    ASSERT(0 != pVg->epSet.eps[pVg->epSet.inUse].fqdn[0]);
     memcpy(pVgroup, pVg, sizeof(SVgroupInfo));
   }
   return code;

From 9cfdf922974c034241ae97a6f772fca77ffeda64 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 10:24:24 +0800
Subject: [PATCH 16/63] fix rc issue

---
 source/libs/scheduler/src/schJob.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c
index bf14032669..a0903d94bf 100644
--- a/source/libs/scheduler/src/schJob.c
+++ b/source/libs/scheduler/src/schJob.c
@@ -887,6 +887,7 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod
     if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) {
       schNotifyUserQueryRes(pJob);
     } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) {
+      atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
       schNotifyUserFetchRes(pJob);
     }
   }
@@ -915,6 +916,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) {
   } else if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) {
     schNotifyUserQueryRes(pJob);
   } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) {
+    atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
     schNotifyUserFetchRes(pJob);
   }
 
@@ -935,6 +937,8 @@ void schProcessOnDataFetched(SSchJob *job) {
   if (job->attr.syncSchedule) {
     tsem_post(&job->rspSem);
   } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&job->userCb, SCH_FETCH_CB, 0)) {
+    atomic_val_compare_exchange_8(&job->userFetch, 1, 0);
+    
     schNotifyUserFetchRes(job);
   }
 }
@@ -1673,9 +1677,9 @@ int32_t schAsyncFetchRows(SSchJob *pJob) {
   }
 
   if (pJob->attr.explainMode == EXPLAIN_MODE_STATIC) {
-    SCH_ERR_JRET(schNotifyUserFetchRes(pJob));
-    
     atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
+
+    SCH_ERR_JRET(schNotifyUserFetchRes(pJob));    
   } else {
     pJob->userCb = SCH_FETCH_CB;
     

From cd56e3b11f78edd85904770f2ac1b5b668c5d6f9 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 15:30:57 +0800
Subject: [PATCH 17/63] collect db table list

---
 include/util/taoserror.h               |  6 +--
 source/client/src/clientImpl.c         |  1 +
 source/libs/parser/src/parTranslater.c | 67 ++++++++++++--------------
 source/libs/qcom/src/querymsg.c        | 10 ++--
 4 files changed, 41 insertions(+), 43 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 842c8a44e1..1729122765 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -339,9 +339,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE       TAOS_DEF_ERROR_CODE(0, 0x060D)
 #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE    TAOS_DEF_ERROR_CODE(0, 0x060E)
 #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP       TAOS_DEF_ERROR_CODE(0, 0x060F)
-#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0600)
-#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0601)
-#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0602)
+#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0610)
+#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0611)
+#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0612)
 #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS       TAOS_DEF_ERROR_CODE(0, 0x0613)
 #define TSDB_CODE_TDB_TABLE_RECONFIGURE         TAOS_DEF_ERROR_CODE(0, 0x0614)
 #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0615)
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 7601e082e4..0cc4b2f5d8 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -593,6 +593,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     pRequest->code = code;
   }
 
+  tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
     removeMeta(pTscObj, pRequest->tableList);
   }
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1ba47d0d15..2a2f664120 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -103,15 +103,14 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) {
 
 static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
-  if (pParCxt->async) {
-    code = getTableMetaFromCache(pCxt->pMetaCache, pName, pMeta);
-  } else {
-    code = collectUseDatabase(pName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
-      code = collectUseTable(pName, pCxt->pTables);
-    }
-    if (TSDB_CODE_SUCCESS == code) {
+  int32_t        code = collectUseDatabase(pName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectUseTable(pName, pCxt->pTables);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getTableMetaFromCache(pCxt->pMetaCache, pName, pMeta);
+    } else {
       code = catalogGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pMeta);
     }
   }
@@ -150,12 +149,11 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr
   SParseContext* pParCxt = pCxt->pParseCxt;
   char           fullDbName[TSDB_DB_FNAME_LEN];
   tNameGetFullDbName(pName, fullDbName);
-  int32_t code = TSDB_CODE_SUCCESS;
-  if (pParCxt->async) {
-    code = getDbVgInfoFromCache(pCxt->pMetaCache, fullDbName, pVgInfo);
-  } else {
-    code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
+  int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getDbVgInfoFromCache(pCxt->pMetaCache, fullDbName, pVgInfo);
+    } else {
       code = catalogGetDBVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, fullDbName, pVgInfo);
     }
   }
@@ -175,15 +173,14 @@ static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray*
 
 static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pName, SVgroupInfo* pInfo) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
-  if (pParCxt->async) {
-    code = getTableVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
-  } else {
-    code = collectUseDatabase(pName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
-      code = collectUseTable(pName, pCxt->pTables);
-    }
-    if (TSDB_CODE_SUCCESS == code) {
+  int32_t        code = collectUseDatabase(pName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectUseTable(pName, pCxt->pTables);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getTableVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
+    } else {
       code = catalogGetTableHashVgroup(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pInfo);
     }
   }
@@ -203,12 +200,11 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName,
 static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                               int32_t* pTableNum) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t        code = TSDB_CODE_SUCCESS;
-  if (pParCxt->async) {
-    code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum);
-  } else {
-    code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
+  int32_t        code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum);
+    } else {
       code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum);
     }
   }
@@ -224,12 +220,11 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName));
   char dbFname[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(&name, dbFname);
-  int32_t code = TSDB_CODE_SUCCESS;
-  if (pParCxt->async) {
-    code = getDbCfgFromCache(pCxt->pMetaCache, dbFname, pInfo);
-  } else {
-    code = collectUseDatabaseImpl(dbFname, pCxt->pDbs);
-    if (TSDB_CODE_SUCCESS == code) {
+  int32_t code = collectUseDatabaseImpl(dbFname, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    if (pParCxt->async) {
+      code = getDbCfgFromCache(pCxt->pMetaCache, dbFname, pInfo);
+    } else {
       code = catalogGetDBCfg(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, dbFname, pInfo);
     }
   }
diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c
index 949a58c2fa..db58c9bf56 100644
--- a/source/libs/qcom/src/querymsg.c
+++ b/source/libs/qcom/src/querymsg.c
@@ -294,10 +294,6 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
   pTableMeta->sversion = msg->sversion;
   pTableMeta->tversion = msg->tversion;
 
-  if (isStb) {
-    qDebug("stable %s meta returned, suid:%" PRIx64, msg->stbName, pTableMeta->suid);
-  }
-
   pTableMeta->tableInfo.numOfTags = msg->numOfTags;
   pTableMeta->tableInfo.precision = msg->precision;
   pTableMeta->tableInfo.numOfColumns = msg->numOfColumns;
@@ -308,6 +304,12 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
     pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
   }
 
+  qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId %d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 
+    " tagNum %d colNum %d precision %d rowSize %d", 
+    msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, pTableMeta->suid,
+    pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags, pTableMeta->tableInfo.numOfColumns,
+    pTableMeta->tableInfo.precision, pTableMeta->tableInfo.rowSize);
+
   *pMeta = pTableMeta;
   return TSDB_CODE_SUCCESS;
 }

From e8a3a0395268f254f1b7257e686a4ddc03a18c7b Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 15:34:22 +0800
Subject: [PATCH 18/63] add retry error code

---
 include/libs/qcom/query.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index a717a6b333..94e68ad48a 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -210,8 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_RM_TBLMETA_ERROR(_code)                                                   \
   ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST ||       \
    (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
-   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG) || \
-   (_code == TSDB_CODE_PAR_INVALID_DROP_COL))
+   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
+   (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)

From 9bee5e2cee0702753c1945779ef451d8d0735aec Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 8 Jun 2022 16:08:26 +0800
Subject: [PATCH 19/63] refactor: do some internal refactor.

---
 include/common/tdatablock.h               |  4 +-
 source/client/src/clientImpl.c            | 11 ++++++
 source/common/src/tdatablock.c            | 45 +++++++++++++++++------
 source/dnode/mnode/impl/src/mndShow.c     |  2 +-
 source/libs/executor/src/dataDispatcher.c |  8 ++--
 source/libs/executor/src/executorimpl.c   | 32 +---------------
 source/libs/executor/src/tsort.c          |  4 +-
 7 files changed, 55 insertions(+), 51 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index ab76cb404a..af46535c94 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -206,7 +206,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
 size_t blockDataGetSize(const SSDataBlock* pBlock);
 size_t blockDataGetRowSize(SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
 
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
@@ -238,7 +238,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
                             const char* stbFullName, int32_t vgId);
 
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
-  return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
+  return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
 }
 
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 62026dd577..9da0c77653 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1340,6 +1340,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
   uint64_t groupId = *(uint64_t*)p;
   p += sizeof(uint64_t);
 
+  // check fields
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    int16_t type = *(int16_t*) p;
+    p += sizeof(int16_t);
+
+    int32_t bytes = *(int32_t*) p;
+    p += sizeof(int32_t);
+
+    ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
+  }
+
   int32_t* colLength = (int32_t*)p;
   p += sizeof(int32_t) * numOfCols;
 
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 99ff4a4a42..8c51f1d50c 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -682,9 +682,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
  * @param pBlock
  * @return
  */
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
-  // | total rows/total length | block group id | each column length |
-  return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols) {
+  // | total rows/total length | block group id | column schema | each column length |
+  return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t);
 }
 
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@@ -1244,7 +1244,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 }
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock);
+  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   int32_t rowSize = pBlock->info.rowSize;
 
@@ -1883,34 +1883,44 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
 
 void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                          int8_t needCompress) {
+  // todo extract method
   int32_t* actualLen = (int32_t*)data;
   data += sizeof(int32_t);
 
   uint64_t* groupId = (uint64_t*)data;
   data += sizeof(uint64_t);
 
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    *((int16_t*) data) = pColInfoData->info.type;
+    data += sizeof(int16_t);
+
+    *((int32_t*) data) = pColInfoData->info.bytes;
+    data += sizeof(int32_t);
+  }
+
   int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
 
-  *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
+  *dataLen = blockDataGetSerialMetaSize(numOfCols);;
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
     SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col);
 
     // copy the null bitmap
+    size_t metaSize = 0;
     if (IS_VAR_DATA_TYPE(pColRes->info.type)) {
-      size_t metaSize = numOfRows * sizeof(int32_t);
+      metaSize = numOfRows * sizeof(int32_t);
       memcpy(data, pColRes->varmeta.offset, metaSize);
-      data += metaSize;
-      (*dataLen) += metaSize;
     } else {
-      int32_t len = BitmapLen(numOfRows);
-      memcpy(data, pColRes->nullbitmap, len);
-      data += len;
-      (*dataLen) += len;
+      metaSize = BitmapLen(numOfRows);
+      memcpy(data, pColRes->nullbitmap, metaSize);
     }
 
+    data += metaSize;
+    (*dataLen) += metaSize;
+
     if (needCompress) {
       colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress);
       data += colSizes[col];
@@ -1939,6 +1949,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
   pBlock->info.groupId = *(uint64_t*)pStart;
   pStart += sizeof(uint64_t);
 
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    pColInfoData->info.type = *(int16_t*)pStart;
+    pStart += sizeof(int16_t);
+
+    pColInfoData->info.bytes = *(int32_t*)pStart;
+    pStart += sizeof(int32_t);
+  }
+
+  blockDataEnsureCapacity(pBlock, numOfRows);
+
   int32_t* colLen = (int32_t*)pStart;
   pStart += sizeof(int32_t) * numOfCols;
 
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index 6b70825ed4..6e569a04cc 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -266,7 +266,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
-         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock);
+         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {
diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c
index eaa366b7e7..b7c7102143 100644
--- a/source/libs/executor/src/dataDispatcher.c
+++ b/source/libs/executor/src/dataDispatcher.c
@@ -68,10 +68,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
 }
 
 // data format:
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
-// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
-// |                |  (4 bytes)   |(8 bytes) | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
+// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
+// |                |  (4 bytes)   |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
 // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
 // recorded in the first segment, next to the struct header
 static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 464fb5c957..3a7f1929b9 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -2559,37 +2559,7 @@ int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadIn
       }
     }
 
-    blockDataEnsureCapacity(pBlock, numOfRows);
-
-    int32_t  dataLen = *(int32_t*)pStart;
-    uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t));
-    pStart += sizeof(int32_t) + sizeof(uint64_t);
-
-    int32_t* colLen = (int32_t*)(pStart);
-    pStart += sizeof(int32_t) * numOfCols;
-
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      colLen[i] = htonl(colLen[i]);
-      ASSERT(colLen[i] >= 0);
-
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-      if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-        pColInfoData->varmeta.length = colLen[i];
-        pColInfoData->varmeta.allocLen = colLen[i];
-
-        memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
-        pStart += sizeof(int32_t) * numOfRows;
-
-        pColInfoData->pData = taosMemoryMalloc(colLen[i]);
-      } else {
-        memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
-        pStart += BitmapLen(numOfRows);
-      }
-
-      memcpy(pColInfoData->pData, pStart, colLen[i]);
-      pStart += colLen[i];
-    }
-
+    blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
     // data from mnode
     relocateColumnData(pRes, pColList, pBlock->pDataBlock);
     taosArrayDestroy(pBlock->pDataBlock);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 7581836d59..310aed8a36 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -144,7 +144,9 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
   (*sourceId) += 1;
 
   int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
-  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize;  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+
+  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
   ASSERT(numOfRows > 0);
   return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
 }

From d49f1dbc152997d73d2201c5a3a33002cf8b9e35 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 17:38:51 +0800
Subject: [PATCH 20/63] enh(query): add histogram function distributed
 splitting

TD-16321
---
 include/libs/function/functionMgt.h |  2 ++
 source/libs/function/src/builtins.c | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h
index e6512b5bac..77376a05d9 100644
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@@ -127,6 +127,8 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_APERCENTILE_MERGE,
   FUNCTION_TYPE_SPREAD_PARTIAL,
   FUNCTION_TYPE_SPREAD_MERGE,
+  FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+  FUNCTION_TYPE_HISTOGRAM_MERGE,
 
   // user defined funcion
   FUNCTION_TYPE_UDF = 10000
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 946c37bfd2..49dbd2ead9 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1393,6 +1393,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = histogramFunction,
     .finalizeFunc = histogramFinalize
   },
+  {
+    .name = "_histogram_partial",
+    .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogram,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = histogramFunctionSetup,
+    .processFunc  = histogramFunction,
+    .finalizeFunc = histogramFinalize
+  },
+  {
+    .name = "_histogram_merge",
+    .type = FUNCTION_TYPE_HISTOGRAM_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogram,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = histogramFunctionSetup,
+    .processFunc  = histogramFunction,
+    .finalizeFunc = histogramFinalize
+  },
   {
     .name = "hyperloglog",
     .type = FUNCTION_TYPE_HYPERLOGLOG,

From 081a8a62b2a8453d7ef5adc89dd54cead7bbac1f Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 17:59:32 +0800
Subject: [PATCH 21/63] add histogram partial/merge translate functions

---
 source/libs/function/inc/builtinsimpl.h |  5 ++-
 source/libs/function/src/builtins.c     | 56 ++++++++++++++++++++++++-
 source/libs/function/src/builtinsimpl.c |  4 ++
 3 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index 981b1eec88..c41949bf20 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -78,13 +78,13 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI
 int32_t percentileFunction(SqlFunctionCtx *pCtx);
 int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
-int32_t getApercentileMaxSize();
 bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t apercentileFunction(SqlFunctionCtx *pCtx);
 int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getApercentileMaxSize();
 
 bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@@ -102,13 +102,13 @@ int32_t topFunction(SqlFunctionCtx *pCtx);
 int32_t bottomFunction(SqlFunctionCtx *pCtx);
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
-int32_t getSpreadInfoSize();
 bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t spreadFunction(SqlFunctionCtx* pCtx);
 int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getSpreadInfoSize();
 
 bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -119,6 +119,7 @@ bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getHistogramInfoSize();
 
 bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t hllFunction(SqlFunctionCtx* pCtx);
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 49dbd2ead9..bd13cdcb47 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -503,6 +503,58 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (isPartial) {
+    if (4 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (!IS_NUMERIC_TYPE(colType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    // param1 ~ param3
+    for (int32_t i = 1; i < numOfParams; ++i) {
+      SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
+      if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      SValueNode* pValue = (SValueNode*)pParamNode;
+
+      pValue->notReserved = true;
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (1 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type != TSDB_DATA_TYPE_BINARY) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, true);
+}
+static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -1397,7 +1449,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "_histogram_partial",
     .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL,
     .classification = FUNC_MGT_AGG_FUNC,
-    .translateFunc = translateHistogram,
+    .translateFunc = translateHistogramPartial,
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
@@ -1407,7 +1459,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "_histogram_merge",
     .type = FUNCTION_TYPE_HISTOGRAM_MERGE,
     .classification = FUNC_MGT_AGG_FUNC,
-    .translateFunc = translateHistogram,
+    .translateFunc = translateHistogramMerge,
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 34adf11032..fc72a05edb 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -3118,6 +3118,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return functionFinalize(pCtx, pBlock);
 }
 
+int32_t getHistogramInfoSize() {
+  return (int32_t)sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
+}
+
 bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
   return true;

From 238b2c1639ac0fe018f947d78d185942908dbdc7 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 8 Jun 2022 18:10:38 +0800
Subject: [PATCH 22/63] fix(tmq): status check

---
 source/dnode/mnode/impl/src/mndConsumer.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 1f8bf06993..bd9f5b6ae9 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -92,6 +92,14 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
   ASSERT(pConsumer);
 
+  mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
+        mndConsumerStatusName(pConsumer->status));
+
+  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
+    mndReleaseConsumer(pMnode, pConsumer);
+    return -1;
+  }
+
   SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
   pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
 

From d5a29fc413e5b0c768deb3a14d421ca6336254ca Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 8 Jun 2022 18:11:16 +0800
Subject: [PATCH 23/63] Revert "feat: add group id to multiway-sort-merge
 operator"

This reverts commit b2d066521b141ab4f4f1e2f4915f4e23f3c5653d.
---
 source/libs/executor/inc/tsort.h        |  6 --
 source/libs/executor/src/sortoperator.c | 74 +------------------------
 source/libs/executor/src/tsort.c        |  4 --
 tests/script/tsim/query/scalarNull.sim  |  1 -
 4 files changed, 1 insertion(+), 84 deletions(-)

diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index fd3581e2bf..c8b1b3ee51 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -130,12 +130,6 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
  */
 void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
 
-/**
- *
- * @param pVHandle
- * @return
- */
-uint64_t tsortGetGroupId(STupleHandle* pVHandle);
 /**
  *
  * @param pSortHandle
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 95f9514b07..3034f409f7 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -231,10 +231,6 @@ typedef struct SMultiwaySortMergeOperatorInfo {
 
   SSDataBlock* pInputBlock;
   int64_t startTs;  // sort start time
-
-  bool  hasGroupId;
-  uint64_t groupId;
-  STupleHandle *prefetchedTuple;
 } SMultiwaySortMergeOperatorInfo;
 
 int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
@@ -273,70 +269,6 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
   return TSDB_CODE_SUCCESS;
 }
 
-SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
-  blockDataCleanup(pDataBlock);
-
-  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
-  if (p == NULL) {
-    return NULL;
-  }
-
-  blockDataEnsureCapacity(p, capacity);
-
-  while (1) {
-
-    STupleHandle* pTupleHandle = NULL;
-    if (pInfo->prefetchedTuple == NULL) {
-      pTupleHandle = tsortNextTuple(pHandle);
-    } else {
-      pTupleHandle = pInfo->prefetchedTuple;
-      pInfo->prefetchedTuple = NULL;
-    }
-
-    if (pTupleHandle == NULL) {
-      break;
-    }
-
-    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
-    if (!pInfo->hasGroupId) {
-      pInfo->groupId = tupleGroupId;
-      pInfo->hasGroupId = true;
-      appendOneRowToDataBlock(p, pTupleHandle);
-    } else if (pInfo->groupId == tupleGroupId) {
-      appendOneRowToDataBlock(p, pTupleHandle);
-    } else {
-      pInfo->prefetchedTuple = pTupleHandle;
-      pInfo->groupId = tupleGroupId;
-      break;
-    }
-
-    if (p->info.rows >= capacity) {
-      break;
-    }
-
-  }
-
-  if (p->info.rows > 0) {
-    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
-      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
-
-      SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
-      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
-      colDataAssign(pDst, pSrc, p->info.rows);
-    }
-
-    pDataBlock->info.rows = p->info.rows;
-    pDataBlock->info.capacity = p->info.rows;
-  }
-
-  blockDataDestroy(p);
-  return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
-}
-
-
 SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -351,11 +283,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   }
 
   SSDataBlock* pBlock =
-      getMultiwaySortedBlockData(pInfo->pSortHandle,
-                         pInfo->binfo.pRes,
-                         pOperator->resultInfo.capacity,
-                         pInfo->pColMatchInfo,
-                                 pInfo);
+      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 846da4a32a..3705d0a57b 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -709,10 +709,6 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
   return colDataGetData(pColInfo, pVHandle->rowIndex);
 }
 
-uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
-  return pVHandle->pBlock->info.groupId;
-}
-
 SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
   SSortExecInfo info = {0};
 
diff --git a/tests/script/tsim/query/scalarNull.sim b/tests/script/tsim/query/scalarNull.sim
index 07bd5e57cd..b08ac1d3d9 100644
--- a/tests/script/tsim/query/scalarNull.sim
+++ b/tests/script/tsim/query/scalarNull.sim
@@ -66,7 +66,6 @@ if $rows != 0 then
   return -1
 endi
 sql select * from tb1 where null;
-print $rows
 if $rows != 0 then
   return -1
 endi

From 9d3524ab6b3530d16dc1e9a773b01a58bf722d96 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Wed, 8 Jun 2022 18:19:20 +0800
Subject: [PATCH 24/63] fix: minor optimization for tsrow

---
 source/common/src/tdataformat.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 9aa94b0216..6037b011af 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -208,7 +208,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     if (iColumn == 0) {
       ASSERT(pColVal->cid == pTColumn->colId);
       ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
-      ASSERT(pTColumn->colId == 0);
+      ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
 
       iColVal++;
     } else {
@@ -352,7 +352,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     ntv = 0;
     iColVal = 1;
 
-    if (flags & 0xf0 == 0) {
+    if ((flags & 0xf0) == 0) {
       switch (flags & 0xf) {
         case TSROW_HAS_VAL:
           pf = (*ppRow)->pData;
@@ -417,13 +417,13 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       }
 
     _set_none:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 0, iColumn - 1, flags);
       }
       continue;
 
     _set_null:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 1, iColumn - 1, flags);
       } else {
         SET_IDX(pidx, pTSKVRow->nCols, nkv, flags);
@@ -433,7 +433,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       continue;
 
     _set_value:
-      if (flags & 0xf0 == 0) {
+      if ((flags & 0xf0) == 0) {
         setBitMap(pb, 2, iColumn - 1, flags);
 
         if (IS_VAR_DATA_TYPE(pTColumn->type)) {
@@ -489,7 +489,7 @@ void tTSRowFree(STSRow2 *pRow) {
 }
 
 void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
-  uint8_t   isTuple = (pRow->flags & 0xf0 == 0) ? 1 : 0;
+  uint8_t   isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0;
   STColumn *pTColumn = &pTSchema->columns[iCol];
   uint8_t   flags = pRow->flags & (uint8_t)0xf;
   SValue    value;
@@ -505,7 +505,7 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal
 
   if (flags == TSROW_HAS_NONE) {
     goto _return_none;
-  } else if (flags == TSROW_HAS_NONE) {
+  } else if (flags == TSROW_HAS_NULL) {
     goto _return_null;
   }
 

From a8da3cfd143caabd0d1b56a83a732c92aa846167 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao@163.com>
Date: Wed, 8 Jun 2022 18:15:48 +0800
Subject: [PATCH 25/63] feat(stream): stream state support apercentile

---
 source/libs/executor/src/timewindowoperator.c | 34 +++++++++++--------
 source/libs/function/inc/builtinsimpl.h       |  1 +
 source/libs/function/src/builtins.c           |  5 ++-
 source/libs/function/src/builtinsimpl.c       | 20 +++++++++++
 4 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index d5e7d5c515..b309478556 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1227,7 +1227,10 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int
   SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
   SResultRowPosition* p1 =
       (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
-  ASSERT(p1);
+  if (!p1) {
+    // window has been closed
+    return;
+  }
   doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
 }
 
@@ -2202,12 +2205,12 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
   }
 }
 
-int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
-                       SDiskbasedBuf* pResultBuf) {
+int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo,
+    int32_t numOfCols, SSDataBlock* pResultBlock) {
   pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
   pBasicInfo->pRes = pResultBlock;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pBasicInfo->pCtx[i].pBuf = pResultBuf;
+    pBasicInfo->pCtx[i].pBuf = NULL;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -2237,16 +2240,15 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx
 
   initResultSizeInfo(pOperator, 4096);
 
-  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
   pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
 
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
@@ -3101,6 +3103,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
   int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
   SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
+  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
 
   SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
   SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -3121,17 +3124,18 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
       .winMap = NULL,
   };
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
-  int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
 
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
     goto _error;
diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index 981b1eec88..56379f0e1f 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -85,6 +85,7 @@ int32_t apercentileFunction(SqlFunctionCtx *pCtx);
 int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 
 bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 946c37bfd2..a7d6fd7371 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1241,7 +1241,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getPercentileFuncEnv,
     .initFunc     = percentileFunctionSetup,
     .processFunc  = percentileFunction,
-    .finalizeFunc = percentileFinalize
+    .finalizeFunc = percentileFinalize,
+    .invertFunc   = NULL,
+    .combineFunc  = NULL,
   },
   {
     .name = "apercentile",
@@ -1252,6 +1254,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = apercentileFunctionSetup,
     .processFunc  = apercentileFunction,
     .finalizeFunc = apercentileFinalize,
+    .combineFunc  = apercentileCombine,
     .pPartialFunc = "_apercentile_partial",
     .pMergeFunc   = "_apercentile_merge"
   },
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 34adf11032..0852779791 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2217,6 +2217,26 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+  SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
+  SAPercentileInfo*    pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+  int32_t type = pDestCtx->input.pData[0]->info.type;
+
+  SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+  SAPercentileInfo*    pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+  ASSERT(pDBuf->algo == pSBuf->algo);
+  if (pDBuf->algo == APERCT_ALGO_TDIGEST) {
+    tdigestMerge(pDBuf->pTDigest, pSBuf->pTDigest);
+  } else {
+    SHistogramInfo* pTmp = tHistogramMerge(pDBuf->pHisto, pSBuf->pHisto, MAX_HISTOGRAM_BIN);
+    memcpy(pDBuf->pHisto, pTmp, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
+    pDBuf->pHisto->elems = (SHistBin*) ((char *)pDBuf->pHisto + sizeof(SHistogramInfo));
+    tHistogramDestroy(&pTmp);
+  }
+  pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
+  return TSDB_CODE_SUCCESS;
+}
+
 bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);
   pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t);

From 6522ff199a3dabb24c9d5c339da15127a05633d8 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 8 Jun 2022 18:23:03 +0800
Subject: [PATCH 26/63] Revert "Revert "feat: add group id to
 multiway-sort-merge operator""

This reverts commit d5a29fc413e5b0c768deb3a14d421ca6336254ca.
---
 source/libs/executor/inc/tsort.h        |  6 ++
 source/libs/executor/src/sortoperator.c | 74 ++++++++++++++++++++++++-
 source/libs/executor/src/tsort.c        |  4 ++
 tests/script/tsim/query/scalarNull.sim  |  1 +
 4 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index c8b1b3ee51..fd3581e2bf 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -130,6 +130,12 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
  */
 void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
 
+/**
+ *
+ * @param pVHandle
+ * @return
+ */
+uint64_t tsortGetGroupId(STupleHandle* pVHandle);
 /**
  *
  * @param pSortHandle
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 3034f409f7..95f9514b07 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -231,6 +231,10 @@ typedef struct SMultiwaySortMergeOperatorInfo {
 
   SSDataBlock* pInputBlock;
   int64_t startTs;  // sort start time
+
+  bool  hasGroupId;
+  uint64_t groupId;
+  STupleHandle *prefetchedTuple;
 } SMultiwaySortMergeOperatorInfo;
 
 int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
@@ -269,6 +273,70 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
   return TSDB_CODE_SUCCESS;
 }
 
+SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
+                                SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
+  blockDataCleanup(pDataBlock);
+
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
+
+  while (1) {
+
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->prefetchedTuple = NULL;
+    }
+
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      pInfo->groupId = tupleGroupId;
+      break;
+    }
+
+    if (p->info.rows >= capacity) {
+      break;
+    }
+
+  }
+
+  if (p->info.rows > 0) {
+    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+
+      SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      colDataAssign(pDst, pSrc, p->info.rows);
+    }
+
+    pDataBlock->info.rows = p->info.rows;
+    pDataBlock->info.capacity = p->info.rows;
+  }
+
+  blockDataDestroy(p);
+  return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
+}
+
+
 SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -283,7 +351,11 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   }
 
   SSDataBlock* pBlock =
-      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+      getMultiwaySortedBlockData(pInfo->pSortHandle,
+                         pInfo->binfo.pRes,
+                         pOperator->resultInfo.capacity,
+                         pInfo->pColMatchInfo,
+                                 pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 3705d0a57b..846da4a32a 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -709,6 +709,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
   return colDataGetData(pColInfo, pVHandle->rowIndex);
 }
 
+uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
+  return pVHandle->pBlock->info.groupId;
+}
+
 SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
   SSortExecInfo info = {0};
 
diff --git a/tests/script/tsim/query/scalarNull.sim b/tests/script/tsim/query/scalarNull.sim
index b08ac1d3d9..07bd5e57cd 100644
--- a/tests/script/tsim/query/scalarNull.sim
+++ b/tests/script/tsim/query/scalarNull.sim
@@ -66,6 +66,7 @@ if $rows != 0 then
   return -1
 endi
 sql select * from tb1 where null;
+print $rows
 if $rows != 0 then
   return -1
 endi

From 1f5ad0f1ec433628b4b7dede710e313fa44af435 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 8 Jun 2022 18:29:51 +0800
Subject: [PATCH 27/63] enh: index support between

---
 source/libs/index/inc/indexComm.h        |   9 ++++----
 source/libs/index/src/.indexFilter.c.swo | Bin 0 -> 16384 bytes
 source/libs/index/src/index.c            |   4 ++--
 source/libs/index/src/indexComm.c        |  27 ++++++++++++++---------
 source/libs/index/src/indexFilter.c      |   2 ++
 source/libs/index/src/indexTfile.c       |   9 ++------
 source/libs/index/test/utilUT.cc         |   4 ++--
 tests/script/tsim/stable/tag_filter.sim  |   2 +-
 8 files changed, 30 insertions(+), 27 deletions(-)
 create mode 100644 source/libs/index/src/.indexFilter.c.swo

diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h
index c338300b57..bcf76b4f20 100644
--- a/source/libs/index/inc/indexComm.h
+++ b/source/libs/index/inc/indexComm.h
@@ -39,13 +39,14 @@ TExeCond      tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
 _cache_range_compare indexGetCompare(RangeType ty);
 
-int32_t indexConvertData(void* src, int8_t type, void** dst);
-int32_t indexConvertDataToStr(void* src, int8_t type, void** dst);
+int32_t idxConvertData(void* src, int8_t type, void** dst);
+int32_t idxConvertDataToStr(void* src, int8_t type, void** dst);
 
-int32_t indexGetDataByteLen(int8_t type);
+int32_t idxGetDataByteLen(int8_t type);
 
-char* indexInt2str(int64_t val, char* dst, int radix);
+char* idxInt2str(int64_t val, char* dst, int radix);
 
+int idxUidCompare(const void* a, const void* b);
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/index/src/.indexFilter.c.swo b/source/libs/index/src/.indexFilter.c.swo
new file mode 100644
index 0000000000000000000000000000000000000000..c61d5e7793e4ca891ef2b8cc1e5b63c72f1431a8
GIT binary patch
literal 16384
zcmeHNU5q4E6)r#pTo(|GBpSkHfK7MXGrha)!m`6I)b@1G6x&nX+g;5%L1?SHZci6m
zUDedj{IIMB4AEdBFFcxv3DFl;i80X^<8RccG2sFKP%#=!e31u?8g<d{+*>vMGd;s1
zCPuB^eA`p^=iGb0d(OG{oa*-e`?O_tWbS~1&ozqjrTd?0|54wkyzrEw_?{P1@#}r=
zxV^&<n|Jc=?ROmBt?h?MU}4?!&lHRW?ylM{F9crXcX+|E+d;v0Exu6*{EqyqY&#+M
z=Q_KIOzTnxQU)%Dfhg?G9@(qhcJRP{YJT&~P3*dBn-|kFZId#PGLSNmGLSNmGLSNm
zGLSNmGVnjbKsdNu`7qXUX}lgc{=Q`5`ycVgiOao-`q$$6eq8?2ME!-hp2y`&@c_v<
ze~;@o;`;Lw^?!@&*W>zI;|VAIFN@2|@%JYu>f_?{D`g;MAY~wBAY~wBAY~wBAY~wB
zAY~wBAZ6hHk^$3Fl%rVrRTMAd{eKewU$|OPz5#p{2!R~%yQ>uCH^9@tNnjsvGw|cR
zit-fjao|Z{9rz${FK`ENBd`Z}^&N`xE8u0|`+yF-xJOaiKovL)FyKaDFYuSQE6SgN
zmw|5r&jB9;9t3hg7I+tM1#mg=%9V=pQ{Xdz1{?xz0ImbB1-^ZSqI?>79B2ZEfjz)0
zmn+J*fX@Pt180DD1FyYJQC<MP44em^03HK4&;i=OHNe%t@860w0nY+9a0|eI8-Z8f
zq9{KCUIe}Xd;&NF*gy|B2z(6*j4qG`-V0n0>;bL>Uc}yf9XJm>0Xzn*00)3s;GMwL
z0PW#k;7xqaDUA6%jC_|FjfE4fVs)X^YBY<*Qll}CI;x-2;#7U5rRn#n6>S0EwWdLB
z?Jmo7+}YzD&oMR!oZWPc6|0rzvfeUQYNeK;E@s(Tw0q++*|B3xZ&oU@f1$uk%VK@c
z=d9D?oij=hnxWlcwj17duobev?%roQwq=IA=s8i}%{1h<?pd7WIxfi>S^2?jD?LCZ
z^r_b5<h4diuNtkUep0WV)@eTBCVTYJt@>q6Z#HDHoa993vRY}Bws(ug?p!-Lpdx7=
zofdf6Q81A<wbH0Jcb?8z<rH!kc~~8Azh?SoUyu5k5<Qur=ZkF*H;DnkZ)keKk{R@h
z4maIskQq>q+2aHGOP?%}Oih+K_HwiAbDp&5c&>#ljQCa?)%VTK1s;UHx0$r^c@Q}v
zn#8OK@w8m2D^joLlLf4KwneHz<A|we8d_QW!E&Ofn0BFXPJx24hGDMCX(o%)+^&a~
z;_Dd8?9?DDWX+S1SS`iCmYn)jsTt8?I0<wG0a$9(2|>^*>xy4SF)yZ>InZpMMhIbN
z7Vlyt(H09jZ8~SLsD83)EV$K;+>SI0J#w5fo|xna%q0>M+sWjHJ+dPVIlLR@*^n{c
zUJZMr1zr>mBC#1Gmi4B6$~nfJfD5<5Sk4MU%=}cd1ou;`m&)4vh2_dN<bLp5GTQ&d
zJnn5&b+|O$P&K`g*yWg%dTFurIzlSi$r73RgzRP2C@xJY*G0wD!HiOUdG`sIPBqoo
z%S|k|0*BlZ)w@fmR8N=cvf<P@2sy@*s$Yy4YByENV!fndH5W6Xv8KBUOJ+4?BNHLX
zgtj|;_#{&7vhVd94?7bs30s$EC5?V=cq-!lrtnjw2Q(k$9I16N+#p6^Qw`19C_7;T
zXG#>nvZ%Nr`Y!X(RQ*+E`m0pf_B@AeeO#}!!8`Lp1gw!CWe-JAR~wcJYbArW?9ldH
z7Q+AJndw^M6zjq8Sm-g1&JkQF8#ty*Ba+{&wjW|S8evf90YpQ14$98^9G&c7#2gC>
z4u<&nqjn4fT6})XyoU|~3*B0@@P0e2Pl1S@=?hunKOF12+2{AiHeG4y>T;=7De0Kv
zLH0g&=mC;6-p6VWyWn<_^bjpE-P;erEy+(1*_I@a&UWeGXc11!wvtA8(kvPzLf<1v
zDK)}bL3=8fGQNGZi{gY3GXtAjXNzbp&J~uEM~P@RWIjHprm-lP1CQjYv0<yTUC-~E
zA@f}2W;kU0(v>!+RjSup_tr~Brjuos$IgmffeVA4h}#m~M{rHr<|d(+42t8s9yO+&
zWUC&uq!piO>>U2B_iTr=%>L}ykfWm|8qp#V!(rv)>y9g$&10bWg1GfOePCMazkTq@
zp+{G}jr(pF@rmtroCvF(S#z=Ix!%l3QHU#{*>=3nnep<VV>+fkUiL6K3K9{Fmrp4Y
zzhl<iaJ-=?EqeX_lwvAUSPoX3W2WP|t8Cp4do1k1s0Y5cifhsw)4I&{aOeh{v%7j>
zI5=7;tgo-nt-8^i=da>7)E({yJSZHWBRMNx#Dbm|ITq`gYaF}Q;r1Gbn}G%mHksFj
ze9jj2CR4jz?t5&JyKqhpt3_?dVihS5vPfK6GjyCetX!=#mDSX`p%t4Iwa#kIdac?h
zi7{1ug`HOGbyYW3=EWLt4PbnY!_8qVyFYMj%wXNb?ajrF8FN@J)r&Y*4E2Oo(To-5
z`81j_vU2ilmJN?(x1$ifZ7HmW>kp*D$m|e9ISy-c76rT;IeCVb?6hVqRhvdUf4EZN
zMObD=U944RvhcYemzEmzAho?IrbxFz*6_L^=Ezyui(Kdk+_YF82EsG<MN8OYKe%WX
z^K6a#0r~serme3_!v^uNN7*L!u4B5ioIEyexO#C=6tI@QXW8A&cx^@OCoW*U5CK<~
z9o&E5PG+dp23vr7=b7eq=Ga{i^>DSmhpyM8^m84re_Sc#6pH`9hxqk1#Jd##m*e2^
zhluyT3w#H79tZ#rco6sia6fQ8@G9c}p95b6o(Db;d<3`wxD5CK;{IoWXMj%uHqZmA
zfDZf>Ie^~+9|ax&R)BkeyMY|=3i1Gd0Db|y1pEwm9N@qrPy|i@w*Y@a4&ZaZc_0Fg
z0ha=o06#(e{~Yi%um*%c0Nf3{2e^Pdz^{Q{0$%~10v3TXuoqAO;^+eLlzvhMQU+26
zQU+26{;L@nNzEjItqfA=;*P`*MbS>Ugt#G-X~=A8HjaYmDbMyHW|j#z5h)Kl%M_L7
z+98GD)OaTevSps_CqXh?_7EHOJ=|Fl;)r`A#MsE-J|eDSjPh4>CzGukGA$?5adBoW
zNoP$YmmB4ZA@a7Gz97>)6m^X(XqYTK3*nQIKO&|{3FA#{YeO<7lLZN7BLm0;gOFVo
z(b4u$k4P9xOYRdH#4U1ggA+ljYFVn|WMrh8XQI*ERti*RQZJ%mJeMR|!^lO`=|pr_
z-f73=e8hyd5`dx%1f88jHU!_fzD$nfSv)6{$&}!^(KtIULnYlz#dJw$GDi^R*pG0p
zo1xgA@&f-+HQP<cK;RB<Fh<^+?4oB$P;Plrk&zP4h;%QDEp9}8D8Wiut0eO@WVs=@
zOA;5`)QkdtX{W?9<&x@1vaj(>C)}8$Y*L=ZS?BD&fi&$}RnsY_cVfjTHN<9*X(cpG
z3d~XR4VkfH2vmaE<MN62ShA9&J;<XBm7QJ~skIs<M5=WmobpvzS;_Z3KQn_w#~Qt|
zA(<5L^)TZ+!e5IY&D=Ub>_&O&%&k^-X8Xjc6=i=iVj<G25dS1<>t%qn?<C(Zs*uCq
zdI6A4Xo5Yz0gfh&iO3x?PxA^WA79!S_%$yGt3F~$IZIM^VR%%}eK=P1mHh6YT*M%v
z+g7#QS}rYD>nlY0IEjML+1Mp^O=O2G;Zb7^AVZrG+!H&=tD!_8NQICbi`64GP*NOW
zpNFsm5Ce7=c-O>R7pkFzEBR%)&kL%d%9mv)v_+^AHmz$Sd7o!XYGVm*xK*hZPm1xu
zdl6|!42E&e3d-=ue)tZVMWxY^LWSqRly>1`>MKa01>+l^m|UC&8>;1&#gZ0}zd;;1
z3KP4Ob~1hh#MZEr0VdN=)azfc!im!Xsw?#*5xw2>4;F%O_Ba}1pD3LGE0quE_)ol9
Bh%x{G

literal 0
HcmV?d00001

diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index 058f3c1915..a5b158c00c 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -289,7 +289,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
   tm->nColName = nColName;
 
   char*   buf = NULL;
-  int32_t len = indexConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
+  int32_t len = idxConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
   assert(len != -1);
 
   tm->colVal = buf;
@@ -625,7 +625,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) {
 
   char* p = buf;
   char  tbuf[65] = {0};
-  indexInt2str((int64_t)key->suid, tbuf, 0);
+  idxInt2str((int64_t)key->suid, tbuf, 0);
 
   SERIALIZE_STR_VAR_TO_BUF(buf, tbuf, strlen(tbuf));
   SERIALIZE_VAR_TO_BUF(buf, '_', char);
diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index d30147d70f..ca9d429e68 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -48,7 +48,7 @@
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
-char* indexInt2str(int64_t val, char* dst, int radix) {
+char* idxInt2str(int64_t val, char* dst, int radix) {
   char     buffer[65] = {0};
   char*    p;
   int64_t  new_val;
@@ -257,7 +257,12 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
   return buf;
 }
 
-int32_t indexConvertData(void* src, int8_t type, void** dst) {
+int idxUidCompare(const void* a, const void* b) {
+  uint64_t l = *(uint64_t*)a;
+  uint64_t r = *(uint64_t*)b;
+  return l - r;
+}
+int32_t idxConvertData(void* src, int8_t type, void** dst) {
   int tlen = -1;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
@@ -342,44 +347,44 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
   // indexMayFillNumbericData(*dst, tlen);
   return tlen;
 }
-int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
+int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
   int     tlen = tDataTypes[type].bytes;
   int32_t bufSize = 64;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int64_t*)src, *dst, -1);
+      idxInt2str(*(int64_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_UTINYINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint8_t*)src, *dst, 1);
+      idxInt2str(*(uint8_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_TINYINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int8_t*)src, *dst, 1);
+      idxInt2str(*(int8_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int16_t*)src, *dst, -1);
+      idxInt2str(*(int16_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint16_t*)src, *dst, -1);
+      idxInt2str(*(uint16_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_INT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(int32_t*)src, *dst, -1);
+      idxInt2str(*(int32_t*)src, *dst, -1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_UINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint32_t*)src, *dst, 1);
+      idxInt2str(*(uint32_t*)src, *dst, 1);
       tlen = strlen(*dst);
       break;
     case TSDB_DATA_TYPE_BIGINT:
@@ -389,7 +394,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
       break;
     case TSDB_DATA_TYPE_UBIGINT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
-      indexInt2str(*(uint64_t*)src, *dst, 1);
+      idxInt2str(*(uint64_t*)src, *dst, 1);
       tlen = strlen(*dst);
     case TSDB_DATA_TYPE_FLOAT:
       *dst = taosMemoryCalloc(1, bufSize + 1);
diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c
index fd2d6a5c8b..2cf83b52e4 100644
--- a/source/libs/index/src/indexFilter.c
+++ b/source/libs/index/src/indexFilter.c
@@ -547,6 +547,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou
       } else if (node->condType == LOGIC_COND_TYPE_NOT) {
         // taosArrayAddAll(output->result, params[m].result);
       }
+      taosArraySort(output->result, idxUidCompare);
+      taosArrayRemoveDuplicate(output->result, idxUidCompare, NULL);
     }
   } else {
     for (int32_t m = 0; m < node->pParameterList->length; m++) {
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 0145df4676..9600b1172e 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -578,8 +578,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
   // ugly code, refactor later
   for (size_t i = 0; i < sz; i++) {
     TFileValue* v = taosArrayGetP((SArray*)data, i);
-    taosArraySort(v->tableId, tfileUidCompare);
-    taosArrayRemoveDuplicate(v->tableId, tfileUidCompare, NULL);
+    taosArraySort(v->tableId, idxUidCompare);
+    taosArrayRemoveDuplicate(v->tableId, idxUidCompare, NULL);
     int32_t tbsz = taosArrayGetSize(v->tableId);
     fstOffset += TF_TABLE_TATOAL_SIZE(tbsz);
   }
@@ -791,11 +791,6 @@ TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
   return rd;
 }
 
-static int tfileUidCompare(const void* a, const void* b) {
-  uint64_t l = *(uint64_t*)a;
-  uint64_t r = *(uint64_t*)b;
-  return l - r;
-}
 static int tfileStrCompare(const void* a, const void* b) {
   int ret = strcmp((char*)a, (char*)b);
   if (ret == 0) {
diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc
index 4a30160244..6858d31325 100644
--- a/source/libs/index/test/utilUT.cc
+++ b/source/libs/index/test/utilUT.cc
@@ -327,13 +327,13 @@ TEST_F(UtilEnv, testFill) {
   for (int i = 0; i < 1000000; i++) {
     int64_t val = i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, 1);
+    idxInt2str(val, buf, 1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
   for (int i = 0; i < 1000000; i++) {
     int64_t val = 0 - i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, -1);
+    idxInt2str(val, buf, -1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
 }
diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim
index c8edfb1ee3..9a2c8abc57 100644
--- a/tests/script/tsim/stable/tag_filter.sim
+++ b/tests/script/tsim/stable/tag_filter.sim
@@ -25,7 +25,7 @@ sql insert into db.ctb5 values(now, 5, "2")
 
 sql create table db.ctb6 using db.stb tags(6, "102")
 sql insert into db.ctb6 values(now, 6, "2")
-
+// int
 sql select * from db.stb where t1 = 1
 if $rows != 1 then 
   return -1

From 1a078d5d56a6762a7bdc50714f376b48b3651552 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 18:36:46 +0800
Subject: [PATCH 28/63] fix in issue

---
 include/common/tmsg.h                |  1 +
 include/libs/function/function.h     |  1 +
 source/dnode/mnode/impl/src/mndSma.c |  9 +++++
 source/libs/scalar/inc/sclInt.h      |  9 ++++-
 source/libs/scalar/src/scalar.c      | 51 +++++++++++++++++++++++++-
 source/libs/scalar/src/sclvector.c   | 53 +++++++++++++---------------
 6 files changed, 94 insertions(+), 30 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 4a3c4b0c3f..5453c8d0ea 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2500,6 +2500,7 @@ typedef struct {
   int64_t   sliding;
   int64_t   dstTbUid;  
   int32_t   dstVgId;  // for stream  
+  SEpSet    epSet;
   char*     expr;
 } STableIndexInfo;
 
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index e8cb363e08..810991f770 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -197,6 +197,7 @@ typedef struct SAggFunctionInfo {
 struct SScalarParam {
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;
+  int32_t          hashValueType;
   void            *param;  // other parameter, such as meta handle from vnode, to extract table name/tag value
   int32_t          numOfRows;
 };
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index 1d47f8fc7a..cde36eac58 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -894,6 +894,15 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn
     info.sliding = pSma->sliding;
     info.dstTbUid = pSma->dstTbUid;
     info.dstVgId = pSma->dstVgId;
+
+    SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId);
+    if (pVg == NULL) {
+      code = -1;
+      sdbRelease(pSdb, pSma);
+      return code;
+    }
+    info.epSet = mndGetVgroupEpset(pMnode, pVg);
+    
     info.expr = taosMemoryMalloc(pSma->exprLen + 1);
     if (info.expr == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h
index 1c2e4a358a..4dd16098ff 100644
--- a/source/libs/scalar/inc/sclInt.h
+++ b/source/libs/scalar/inc/sclInt.h
@@ -22,11 +22,18 @@ extern "C" {
 #include "thash.h"
 #include "query.h"
 
+typedef struct SOperatorValueType {
+  int32_t opResType;
+  int32_t selfType;
+  int32_t peerType;
+} SOperatorValueType;
+
 typedef struct SScalarCtx {
   int32_t      code;
   SArray      *pBlockList;  /* element is SSDataBlock* */
   SHashObj    *pRes;  /* element is SScalarParam */
   void        *param;      // additional parameter (meta actually) for acquire value such as tbname/tags values
+  SOperatorValueType type;
 } SScalarCtx;
 
 
@@ -53,7 +60,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
 SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
 int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
 
-#define GET_PARAM_TYPE(_c)      ((_c)->columnData->info.type)
+#define GET_PARAM_TYPE(_c)      ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType)
 #define GET_PARAM_BYTES(_c)     ((_c)->columnData->info.bytes)
 #define GET_PARAM_PRECISON(_c)  ((_c)->columnData->info.precision)
 
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index d2436b9948..66e4af5ded 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -208,7 +208,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
         SCL_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
 
-      SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, nodeList->dataType.type));
+      int32_t type = vectorGetConvertType(ctx->type.selfType, ctx->type.peerType);
+      if (type == 0) {
+        type = nodeList->dataType.type;
+      }
+      
+      SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type));
+      param->hashValueType = type;
       if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
         taosHashCleanup(param->pHashFilter);
         sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
@@ -334,6 +340,46 @@ _return:
   SCL_RET(code);
 }
 
+int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
+  if (NULL == pNode) {
+    return -1;
+  }
+  
+  switch (nodeType(pNode)) {
+    case QUERY_NODE_VALUE: {
+      SValueNode *valueNode = (SValueNode *)pNode;
+      return valueNode->node.resType.type;
+    }
+    case QUERY_NODE_NODE_LIST: {
+      SNodeListNode *nodeList = (SNodeListNode *)pNode;
+      return nodeList->dataType.type;
+    }
+    case QUERY_NODE_COLUMN: {
+      SColumnNode *colNode = (SColumnNode *)pNode;
+      return colNode->node.resType.type;
+    }
+    case QUERY_NODE_FUNCTION:
+    case QUERY_NODE_OPERATOR:
+    case QUERY_NODE_LOGIC_CONDITION: {
+      SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &pNode, POINTER_BYTES);
+      if (NULL == res) {
+        sclError("no result for node, type:%d, node:%p", nodeType(pNode), pNode);
+        return -1;
+      }
+      return res->columnData->info.type;
+    }
+  }
+
+  return -1;
+}
+
+
+void sclSetOperatorValueType(SOperatorNode *node, SScalarCtx *ctx) {
+  ctx->type.opResType = node->node.resType.type;
+  ctx->type.selfType = sclGetNodeType(node->pLeft, ctx);
+  ctx->type.peerType = sclGetNodeType(node->pRight, ctx);
+}
+
 int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) {
   int32_t code = 0;
   int32_t paramNum = scalarGetOperatorParamNum(node->opType);
@@ -348,8 +394,11 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
     SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  sclSetOperatorValueType(node, ctx);
+
   SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
   if (paramNum > 1) {
+    TSWAP(ctx->type.selfType, ctx->type.peerType);
     SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum));
   }
 
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 59208de3c4..1252e27554 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -627,6 +627,11 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
   SColumnInfoData* pInputCol  = pIn->columnData;
   SColumnInfoData* pOutputCol = pOut->columnData;
 
+  if (NULL == pInputCol) {
+    sclError("input column is NULL, hashFilter %p", pIn->pHashFilter);
+    return TSDB_CODE_APP_ERROR;
+  }
+  
   int16_t inType   = pInputCol->info.type;
   int16_t outType  = pOutputCol->info.type;
 
@@ -826,11 +831,26 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
   return gConvertTypes[type2][type1];
 }
 
-int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) {
-  if (pLeft->pHashFilter != NULL || pRight->pHashFilter != NULL) {
-    return TSDB_CODE_SUCCESS;
+int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) {
+  int32_t code = 0;
+  SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
+  output->numOfRows = input->numOfRows;
+
+  output->columnData = createColumnInfoData(&t, input->numOfRows);
+  if (output->columnData == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  code = vectorConvertImpl(input, output);
+  if (code) {
+//      taosMemoryFreeClear(paramOut1->data);
+    return code;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) {
   int32_t leftType  = GET_PARAM_TYPE(pLeft);
   int32_t rightType = GET_PARAM_TYPE(pRight);
   if (leftType == rightType) {
@@ -859,34 +879,11 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
   }
 
   if (type != GET_PARAM_TYPE(param1)) {
-    SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
-    paramOut1->numOfRows = param1->numOfRows;
-
-    paramOut1->columnData = createColumnInfoData(&t, param1->numOfRows);
-    if (paramOut1->columnData == NULL) {
-      return terrno;
-    }
-
-    code = vectorConvertImpl(param1, paramOut1);
-    if (code) {
-//      taosMemoryFreeClear(paramOut1->data);
-      return code;
-    }
+    return vectorConvertScalarParam(param1, paramOut1, type);
   }
   
   if (type != GET_PARAM_TYPE(param2)) {
-    SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
-    paramOut2->numOfRows = param2->numOfRows;
-
-    paramOut2->columnData = createColumnInfoData(&t, param2->numOfRows);
-    if (paramOut2->columnData == NULL) {
-      return terrno;
-    }
-
-    code = vectorConvertImpl(param2, paramOut2);
-    if (code) {
-      return code;
-    }
+    return vectorConvertScalarParam(param2, paramOut2, type);
   }
 
   return TSDB_CODE_SUCCESS;

From bc7397a2e860db910a1434d854e8276f5feb6da9 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 18:39:07 +0800
Subject: [PATCH 29/63] add epset in index rsp

---
 source/common/src/tmsg.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 51c40827b5..b9b4556255 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -2427,6 +2427,7 @@ int32_t tSerializeSTableIndexInfo(SEncoder *pEncoder, STableIndexInfo* pInfo) {
   if (tEncodeI64(pEncoder, pInfo->sliding) < 0) return -1;
   if (tEncodeI64(pEncoder, pInfo->dstTbUid) < 0) return -1;
   if (tEncodeI32(pEncoder, pInfo->dstVgId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
   if (tEncodeCStr(pEncoder, pInfo->expr) < 0) return -1;
   return 0;
 }
@@ -2459,6 +2460,7 @@ int32_t tDeserializeSTableIndexInfo(SDecoder *pDecoder, STableIndexInfo *pInfo)
   if (tDecodeI64(pDecoder, &pInfo->sliding) < 0) return -1;
   if (tDecodeI64(pDecoder, &pInfo->dstTbUid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pInfo->dstVgId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pInfo->expr) < 0) return -1;
 
   return 0;

From c07bea7ff5ab4083c78c8359b20b2eb4a33ea7bb Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 18:40:16 +0800
Subject: [PATCH 30/63] add histogram partial/merge function

---
 source/libs/function/src/builtinsimpl.c | 45 +++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 3 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index fc72a05edb..41414efba5 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2187,9 +2187,7 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   SAPercentileInfo*       pInfo = (SAPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo);
 
-  int32_t bytesHist   = (int32_t)(sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
-  int32_t bytesDigest = (int32_t)(sizeof(SAPercentileInfo) + TDIGEST_SIZE(COMPRESSION));
-  int32_t resultBytes = TMAX(bytesHist, bytesDigest);
+  int32_t resultBytes = getApercentileMaxSize();
   char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
 
   if (pInfo->algo == APERCT_ALGO_TDIGEST) {
@@ -3332,6 +3330,29 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData* pCol = pInput->pData[0];
+  ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
+
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  int32_t start = pInput->startRowIndex;
+  char* data = colDataGetData(pCol, start);
+  SHistoFuncInfo* pInputInfo = (SHistoFuncInfo *)varDataVal(data);
+
+  pInfo->normalized = pInputInfo->normalized;
+  pInfo->numOfBins  = pInputInfo->numOfBins;
+  pInfo->totalCount += pInputInfo->totalCount;
+  for (int32_t k = 0; k < pInfo->numOfBins; ++k) {
+    pInfo->bins[k].count += pInputInfo->bins[k].count;
+  }
+
+  SET_VAL(GET_RES_INFO(pCtx), 1, 1);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@@ -3368,6 +3389,24 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  int32_t resultBytes = getHistogramInfoSize();
+  char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
+
+  memcpy(varDataVal(res), pInfo, resultBytes);
+  varDataSetLen(res, resultBytes);
+
+  int32_t          slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  colDataAppend(pCol, pBlock->info.rows, res, false);
+
+  taosMemoryFree(res);
+  return pResInfo->numOfRes;
+}
+
 bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHLLInfo);
   return true;

From 716dd2c5e4ff87a480afe7c00a7d5cdbdff2fb68 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 8 Jun 2022 18:41:05 +0800
Subject: [PATCH 31/63] refactor: do some internal refactor, add the column
 info in STableRetrieveRsp.

---
 include/util/tdef.h                |   4 +-
 source/client/test/clientTests.cpp |  39 +++++-----
 source/common/src/tdatablock.c     |   2 +-
 source/libs/command/src/command.c  | 119 +++++++++++++++--------------
 source/libs/command/src/explain.c  |  61 ++++++++-------
 5 files changed, 114 insertions(+), 111 deletions(-)

diff --git a/include/util/tdef.h b/include/util/tdef.h
index ebfb7b696a..e66cc0b7bb 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -356,8 +356,8 @@ typedef enum ELogicConditionType {
 
 #define TSDB_DEFAULT_EXPLAIN_VERBOSE false
 
-#define TSDB_EXPLAIN_RESULT_ROW_SIZE    1024
-#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE    512
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
 
 #define TSDB_MAX_FIELD_LEN             16384
 #define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index f16baba0e7..f9825c50ff 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -780,31 +780,28 @@ TEST(testCase, async_api_test) {
 
   taos_query(pConn, "use test");
 
-  TAOS_RES* pRes = taos_query(pConn, "select * from t1");
-
-  taos_query(pConn, "alter table t1 add column b int");
-  pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
+  TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu");
   if (taos_errno(pRes) != 0) {
     printf("failed, reason:%s\n", taos_errstr(pRes));
   }
 
-//  int32_t n = 0;
-//  TAOS_ROW    pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t     numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t* length = taos_fetch_lengths(pRes);
-//    for(int32_t i = 0; i < numOfFields; ++i) {
-//      printf("(%d):%d " , i, length[i]);
-//    }
-//    printf("\n");
-//
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
-//    memset(str, 0, sizeof(str));
-//  }
+  int32_t n = 0;
+  TAOS_ROW    pRow = NULL;
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  int32_t     numOfFields = taos_num_fields(pRes);
+
+  char str[512] = {0};
+  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+    int32_t* length = taos_fetch_lengths(pRes);
+    for(int32_t i = 0; i < numOfFields; ++i) {
+      printf("(%d):%d " , i, length[i]);
+    }
+    printf("\n");
+
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    printf("%s\n", str);
+    memset(str, 0, sizeof(str));
+  }
 
   taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
   getchar();
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 8c51f1d50c..aee0e15a7b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1902,7 +1902,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
   int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
 
-  *dataLen = blockDataGetSerialMetaSize(numOfCols);;
+  *dataLen = blockDataGetSerialMetaSize(numOfCols);
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c
index 3034b4b02a..f330b7ce16 100644
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@@ -28,91 +28,94 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
   }
 }
 
-// todo : to convert data according to SSDatablock
-static void buildRspData(const STableMeta* pMeta, char* pData) {
-  int32_t* payloadLen = (int32_t*) pData;
-  uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t));
+static SSDataBlock* buildDescResultDataBlock() {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
+  pBlock->info.hasVarCol = true;
 
-  int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t));
-  pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t);
+  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
 
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
 
-  // Field
-  int32_t* pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  return pBlock;
+}
+
+static void setDescResultIntoDataBlock(SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
+  blockDataEnsureCapacity(pBlock, numOfRows);
+  pBlock->info.rows = numOfRows;
+
+  // field
+  SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
+  char buf[DESCRIBE_RESULT_FIELD_LEN] = {0};
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, pMeta->schema[i].name);
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[0];
-    pOffset += 1;
-    pColSizes[0] += len;
+    STR_TO_VARSTR(buf, pMeta->schema[i].name);
+    colDataAppend(pCol1, i, buf, false);
   }
-  
+
   // Type
-  pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, tDataTypes[pMeta->schema[i].type].name);
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[1];
-    pOffset += 1;
-    pColSizes[1] += len;
+    STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
+    colDataAppend(pCol2, i, buf, false);
   }
 
   // Length
-  pData += BitmapLen(numOfRows);
+  SColumnInfoData* pCol3 = taosArrayGet(pBlock->pDataBlock, 2);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    *(int32_t*)pData = getSchemaBytes(pMeta->schema + i);
-    pData += sizeof(int32_t);
+    int32_t bytes = getSchemaBytes(pMeta->schema + i);
+    colDataAppend(pCol3, i, (const char*)&bytes, false);
   }
-  pColSizes[2] = sizeof(int32_t) * numOfRows;
 
   // Note
-  pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  SColumnInfoData* pCol4 = taosArrayGet(pBlock->pDataBlock, 3);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[3];
-    pOffset += 1;
-    pColSizes[3] += len;
+    STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
+    colDataAppend(pCol4, i, buf, false);
   }
-
-  for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) {
-    pColSizes[i] = htonl(pColSizes[i]);
-  }
-
-
-  *payloadLen = (int32_t)(pData - (char*)payloadLen);
-}
-
-static int32_t calcRspSize(const STableMeta* pMeta) {
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
-  return sizeof(SRetrieveTableRsp) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) +
-      (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) +
-      sizeof(int32_t) + sizeof(uint64_t);
 }
 
 static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
-  SDescribeStmt* pDesc = (SDescribeStmt*)pStmt;
-  *pRsp = taosMemoryCalloc(1, calcRspSize(pDesc->pMeta));
+  SDescribeStmt* pDesc = (SDescribeStmt*) pStmt;
+  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta);
+
+  SSDataBlock* pBlock = buildDescResultDataBlock();
+  setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
+
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
   if (NULL == *pRsp) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+
   (*pRsp)->useconds = 0;
   (*pRsp)->completed = 1;
   (*pRsp)->precision = 0;
   (*pRsp)->compressed = 0;
   (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(TABLE_TOTAL_COL_NUM(pDesc->pMeta));
-  buildRspData(pDesc->pMeta, (*pRsp)->data);
+  (*pRsp)->numOfRows = htonl(numOfRows);
+  (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
+
+  int32_t len = 0;
+  blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
+
+  blockDataDestroy(pBlock);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c
index 99ff4d406c..56b041af40 100644
--- a/source/libs/command/src/explain.c
+++ b/source/libs/command/src/explain.c
@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tdatablock.h"
 #include "commandInt.h"
 #include "plannodes.h"
 #include "query.h"
@@ -916,9 +917,32 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
     QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  int32_t colNum = 1;
-  int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum +
-                    sizeof(int32_t) * rowNum + pCtx->dataSize;
+  SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SColumnInfoData infoData = {0};
+  infoData.info.type  = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+
+  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
+  pInfoData->hasNull = false;
+  colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
+
+  char buf[1024] = {0};
+  for (int32_t i = 0; i < rowNum; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    varDataCopy(buf, row->buf);
+    ASSERT(varDataTLen(row->buf) == row->len);
+    colDataAppend(pInfoData, i, buf, false);
+  }
+
+  pBlock->info.numOfCols = 1;
+  pBlock->info.rows = rowNum;
+  pBlock->info.hasVarCol = true;
+
+  int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+
   SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
   if (NULL == rsp) {
     qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
@@ -928,34 +952,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->completed = 1;
   rsp->numOfRows = htonl(rowNum);
 
-  // payload length
-  *(int32_t *)rsp->data =
-      sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
+  int32_t len = 0;
+  blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
-  // group id
-  *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0;
+  rsp->compLen = htonl(len);
 
-  // column length
-  int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
-
-  // varchar column offset segment
-  int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
-
-  // varchar data real payload
-  char *data = (char *)(offset + rowNum);
-
-  char *start = data;
-  for (int32_t i = 0; i < rowNum; ++i) {
-    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
-    offset[i] = data - start;
-
-    varDataCopy(data, row->buf);
-    ASSERT(varDataTLen(row->buf) == row->len);
-    data += row->len;
-  }
-
-  *colLength = htonl(data - start);
-  rsp->compLen = htonl(rspSize);
+  blockDataDestroy(pBlock);
 
   *pRsp = rsp;
   return TSDB_CODE_SUCCESS;

From 9987c66467dc7e49af3eea667cc7a24d0237b07e Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 8 Jun 2022 18:49:46 +0800
Subject: [PATCH 32/63] enh: index support between

---
 tests/script/tsim/stable/tag_filter.sim | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim
index 9a2c8abc57..04b8f21de9 100644
--- a/tests/script/tsim/stable/tag_filter.sim
+++ b/tests/script/tsim/stable/tag_filter.sim
@@ -25,12 +25,13 @@ sql insert into db.ctb5 values(now, 5, "2")
 
 sql create table db.ctb6 using db.stb tags(6, "102")
 sql insert into db.ctb6 values(now, 6, "2")
-// int
+
 sql select * from db.stb where t1 = 1
 if $rows != 1 then 
   return -1
 endi
 
+
 sql select * from db.stb where t1 < 1 
 if $rows != 0 then 
   return -=1
@@ -56,4 +57,21 @@ if $rows != 5 then
   return -1
 endi
 
+sql select * from db.stb where t1 between 1 and 1 
+if $rows != 1 then 
+  return -1
+endi
+
+sql select * from db.stb where t1 between 1 and 6 
+if $rows != 6 then 
+  return -1
+endi
+
+sql select * from db.stb where t1 between 1 and 7 
+if $rows != 6 then 
+  return -1
+endi
+
+
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 9b70420b3c5efeb52d864ff0912b8af3a343e6e4 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 18:58:52 +0800
Subject: [PATCH 33/63] fix case issue

---
 tests/test/c/tmqSim.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index d0b582f375..e4ad4a466c 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -515,13 +515,13 @@ void* consumeThreadFunc(void* param) {
   if (err) {
     pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
     pInfo->consumeMsgCnt = -1;
-    return NULL;
+    //return NULL;
   }
 
   err = tmq_consumer_close(pInfo->tmq);
   if (err) {
     pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-    exit(-1);
+    //exit(-1);
   }
   pInfo->tmq = NULL;
 

From eed875268c3de339a515632630c606256330959c Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 8 Jun 2022 18:59:26 +0800
Subject: [PATCH 34/63] fix: compare group id first when choose tuple from
 loser tree

---
 source/libs/executor/src/tsort.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 846da4a32a..8fb1d1f302 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -359,6 +359,11 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
   SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
   SSDataBlock* pRightBlock = pRightSource->src.pBlock;
 
+  // first sort by block groupId
+  if (pLeftBlock->info.groupId != pRightBlock->info.groupId) {
+    return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1;
+  }
+
   for(int32_t i = 0; i < pInfo->size; ++i) {
     SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
     SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);

From fc1ad1aeac47656a10fcdf2e3f4ad380eae00ae2 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 18:40:16 +0800
Subject: [PATCH 35/63] add histogram partial/merge function

---
 source/libs/function/inc/builtinsimpl.h | 2 ++
 source/libs/function/src/builtins.c     | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index c41949bf20..3c272d3933 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -118,7 +118,9 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
+int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t getHistogramInfoSize();
 
 bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index bd13cdcb47..b4c2bb980e 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1453,7 +1453,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
-    .finalizeFunc = histogramFinalize
+    .finalizeFunc = histogramPartialFinalize
   },
   {
     .name = "_histogram_merge",
@@ -1462,7 +1462,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateHistogramMerge,
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
-    .processFunc  = histogramFunction,
+    .processFunc  = histogramFunctionMerge,
     .finalizeFunc = histogramFinalize
   },
   {

From 9eb8444373c17304319678776db370295034b0d3 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 19:03:10 +0800
Subject: [PATCH 36/63] fix tmq issue

---
 source/dnode/mnode/impl/src/mndConsumer.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 1f8bf06993..6f2ac354f7 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -92,6 +92,11 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
   ASSERT(pConsumer);
 
+  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
+    mndReleaseConsumer(pMnode, pConsumer);
+    return -1;
+  }
+
   SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
   pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
 

From b1226e5d0655162efb65a0689938d2c63b71eaf1 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 8 Jun 2022 18:48:42 +0800
Subject: [PATCH 37/63] feat(tmq): support err2str

---
 include/client/taos.h              | 42 +++++++++++++++---------------
 source/client/src/tmq.c            | 22 +++++++++-------
 source/dnode/vnode/src/tq/tqExec.c |  3 +++
 tests/test/c/tmqSim.c              |  6 ++---
 4 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/include/client/taos.h b/include/client/taos.h
index 7f317b3a9d..5bb528bd7f 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
 typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
-  void *    buffer;
+  void     *buffer;
   uintptr_t buffer_length;
-  int32_t * length;
-  char *    is_null;
+  int32_t  *length;
+  char     *is_null;
   int       num;
 } TAOS_MULTI_BIND;
 
@@ -130,7 +130,7 @@ DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
 DLL_EXPORT int        taos_init(void);
-DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
@@ -147,17 +147,17 @@ DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name
 DLL_EXPORT int        taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 DLL_EXPORT int        taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 
-DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
-DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char *    taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
@@ -179,11 +179,11 @@ DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
-DLL_EXPORT int *       taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
+DLL_EXPORT int        *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
 DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
-DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT int      *taos_fetch_lengths(TAOS_RES *res);
 DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
 
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
@@ -204,17 +204,17 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
+DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
 
 /* --------------------------TMQ INTERFACE------------------------------- */
 
-enum tmq_resp_err_t {
+enum {
   TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
 };
 
-typedef enum tmq_resp_err_t tmq_resp_err_t;
+typedef int32_t tmq_resp_err_t;
 
 typedef struct tmq_t                   tmq_t;
 typedef struct tmq_topic_vgroup_t      tmq_topic_vgroup_t;
@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
-DLL_EXPORT char **     tmq_list_to_c_array(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
 DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES *     tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
@@ -260,7 +260,7 @@ enum tmq_conf_res_t {
 
 typedef enum tmq_conf_res_t tmq_conf_res_t;
 
-DLL_EXPORT tmq_conf_t *   tmq_conf_new();
+DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index ef37eed284..cccb1fa6d0 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -323,7 +323,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
 
 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
-  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
+  pParam->rspErr = code;
   if (pParam->async) {
     if (pParam->automatic && pParam->tmq->commitCb) {
       pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
@@ -432,12 +432,13 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_
     code = pParam->rspErr;
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
+  } else {
+    code = 0;
   }
 
   // avoid double free if msg is sent
   buf = NULL;
 
-  code = 0;
 END:
   if (buf) taosMemoryFree(buf);
   /*if (pParam) taosMemoryFree(pParam);*/
@@ -445,9 +446,9 @@ END:
 
   if (code != 0 && async) {
     if (automatic) {
-      tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
+      tmq->commitCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
     } else {
-      userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
+      userCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
     }
   }
 
@@ -1474,16 +1475,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
-    if (rsp == TMQ_RESP_ERR__FAIL) {
-      return TMQ_RESP_ERR__FAIL;
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
+      return rsp;
     }
 
     tmq_list_t* lst = tmq_list_new();
     rsp = tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
 
-    if (rsp == TMQ_RESP_ERR__FAIL) {
-      return TMQ_RESP_ERR__FAIL;
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
+      return rsp;
     }
   }
   // TODO: free resources
@@ -1493,8 +1494,11 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
 const char* tmq_err2str(tmq_resp_err_t err) {
   if (err == TMQ_RESP_ERR__SUCCESS) {
     return "success";
+  } else if (err == TMQ_RESP_ERR__FAIL) {
+    return "fail";
+  } else {
+    return tstrerror(err);
   }
-  return "fail";
 }
 
 const char* tmq_get_topic_name(TAOS_RES* res) {
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index 31acc60006..7c75d88a83 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -39,6 +39,9 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
 
 static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
   SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
+  if (pSW == NULL) {
+    return -1;
+  }
   taosArrayPush(pRsp->blockSchema, &pSW);
   return 0;
 }
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index d0b582f375..4d9e2275f4 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -514,14 +514,14 @@ void* consumeThreadFunc(void* param) {
   err = tmq_unsubscribe(pInfo->tmq);
   if (err) {
     pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
-    pInfo->consumeMsgCnt = -1;
-    return NULL;
+    /*pInfo->consumeMsgCnt = -1;*/
+    /*return NULL;*/
   }
 
   err = tmq_consumer_close(pInfo->tmq);
   if (err) {
     pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-    exit(-1);
+    /*exit(-1);*/
   }
   pInfo->tmq = NULL;
 

From 9ccc3fb85354fd6bc81be310caa0d8dec0831572 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 19:12:57 +0800
Subject: [PATCH 38/63] fix: enable histogram splitting

---
 source/libs/function/src/builtins.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index b4c2bb980e..f6414a6e20 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1443,7 +1443,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
-    .finalizeFunc = histogramFinalize
+    .finalizeFunc = histogramFinalize,
+    .pPartialFunc = "_histogram_partial",
+    .pMergeFunc   = "_histogram_merge"
   },
   {
     .name = "_histogram_partial",

From ae9e11bb0f2fa057336c9d21f7323f010ab3093f Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 8 Jun 2022 14:44:42 +0800
Subject: [PATCH 39/63] refactor: add sync sem for vnode

---
 source/dnode/mgmt/mgmt_vnode/inc/vmInt.h    |   3 +-
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |   1 -
 source/dnode/mgmt/mgmt_vnode/src/vmInt.c    |   5 +-
 source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 116 +---------
 source/dnode/vnode/inc/vnode.h              |   3 +
 source/dnode/vnode/src/inc/vnd.h            |   1 -
 source/dnode/vnode/src/inc/vnodeInt.h       |   4 +-
 source/dnode/vnode/src/vnd/vnodeOpen.c      |   3 +
 source/dnode/vnode/src/vnd/vnodeSvr.c       |   7 +-
 source/dnode/vnode/src/vnd/vnodeSync.c      | 235 +++++++++++++++-----
 10 files changed, 195 insertions(+), 183 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
index adc18fdced..0f7daf0e1d 100644
--- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
+++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
@@ -52,10 +52,9 @@ typedef struct {
 
 typedef struct {
   int32_t     vgId;
-  int32_t     refCount;
   int32_t     vgVersion;
+  int32_t     refCount;
   int8_t      dropped;
-  int8_t      accessState;
   char       *path;
   SVnode     *pImpl;
   STaosQueue *pWriteQ;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 4c5a32536f..0ae6c2b336 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -323,7 +323,6 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
 
-  // Requests handled by VNODE
   if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index 23927255bb..9e4e7713f2 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -49,10 +49,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   }
 
   pVnode->vgId = pCfg->vgId;
-  pVnode->refCount = 0;
   pVnode->vgVersion = pCfg->vgVersion;
+  pVnode->refCount = 0;
   pVnode->dropped = 0;
-  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
   pVnode->path = tstrdup(pCfg->path);
   pVnode->pImpl = pImpl;
 
@@ -96,7 +95,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {
-    dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped);
+    dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
     vnodeDestroy(path, pMgmt->pTfs);
   }
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index 427b3d5c94..b6913f93f2 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -53,9 +53,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   }
 
   if (IsReq(pMsg)) {
-    if (code != 0 && terrno != 0) {
+    if (code != 0) {
+      if (terrno != 0) code = terrno;
       dError("msg:%p failed to process since %s", pMsg, terrstr());
-      code = terrno;
     }
     vmSendRsp(pMsg, code);
   }
@@ -97,110 +97,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
-static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  int32_t    code = 0;
-  SRpcMsg   *pMsg = NULL;
-  SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    sync = vnodeGetSyncHandle(pVnode->pImpl);
-  SArray    *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg **));
-
-  for (int32_t m = 0; m < numOfMsgs; m++) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-write queue", pVnode->vgId, pMsg);
-
-    if (taosArrayPush(pArray, &pMsg) == NULL) {
-      dError("vgId:%d, failed to push msg:%p to vnode-write array", pVnode->vgId, pMsg);
-      vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
-    }
-  }
-
-  for (int32_t m = 0; m < taosArrayGetSize(pArray); m++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, m);
-    code = vnodePreprocessReq(pVnode->pImpl, pMsg);
-
-    if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
-      dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg);
-      continue;
-    }
-
-    if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) {
-      code = syncPropose(sync, pMsg, false);
-    }
-
-    if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-      dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg);
-      continue;
-    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
-      SEpSet newEpSet = {0};
-      syncGetEpSet(sync, &newEpSet);
-      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
-      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
-        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
-      }
-
-      dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg,
-             newEpSet.numOfEps, newEpSet.inUse);
-      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
-        dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
-      }
-
-      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
-      tmsgSendRedirectRsp(&rsp, &newEpSet);
-    } else {
-      dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code);
-      vmSendRsp(pMsg, code);
-    }
-  }
-
-  for (int32_t i = 0; i < numOfMsgs; i++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
-    dTrace("vgId:%d, msg:%p is freed", pVnode->vgId, pMsg);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-
-  taosArrayDestroy(pArray);
-}
-
-static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-apply queue", pVnode->vgId, pMsg);
-
-    // init response rpc msg
-    SRpcMsg rsp = {0};
-
-    // get original rpc msg
-    assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
-    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
-    SRpcMsg originalRpcMsg;
-    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
-
-    // apply data into tsdb
-    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
-      rsp.code = terrno;
-      dError("vgId:%d, msg:%p failed to apply since %s", pVnode->vgId, pMsg, terrstr());
-    }
-
-    syncApplyMsgDestroy(pSyncApplyMsg);
-    rpcFreeCont(originalRpcMsg.pCont);
-
-    // if leader, send response
-    if (pMsg->info.handle != NULL) {
-      rsp.info = pMsg->info;
-      tmsgSendRsp(&rsp);
-    }
-
-    dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, rsp.code);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-}
-
 static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SRpcMsg   *pMsg = NULL;
@@ -322,7 +218,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   if (pMsg == NULL) return -1;
 
   SMsgHead *pHead = pRpc->pCont;
-  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pMsg->msgType));
+  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType));
 
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = htonl(pHead->vgId);
@@ -362,9 +258,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
 }
 
 int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
-  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue);
+  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg);
   pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
-  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue);
+  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeApplyMsg);
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
   pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
   pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue);
@@ -381,8 +277,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
 void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
-  tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
   tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
+  tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
   tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
   tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ);
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 8c7a78a5af..01e57d5eaf 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -71,6 +71,9 @@ int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
 int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;
diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h
index 5f4f7e70da..b0599b82ef 100644
--- a/source/dnode/vnode/src/inc/vnd.h
+++ b/source/dnode/vnode/src/inc/vnd.h
@@ -84,7 +84,6 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 void    vnodeSyncStart(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
-int32_t vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 77005767fe..377dcf1b6a 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -232,8 +232,10 @@ struct SVnode {
   SWal*      pWal;
   STQ*       pTq;
   SSink*     pSink;
-  int64_t    sync;
   tsem_t     canCommit;
+  int64_t    sync;
+  int32_t    syncCount;
+  sem_t      syncSem;
   SQHandle*  pQuery;
 };
 
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index a85b830616..dfc258b42b 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -81,7 +81,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->state.applied = info.state.committed;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
+  pVnode->syncCount = 0;
 
+  tsem_init(&pVnode->syncSem, 0, 0);
   tsem_init(&(pVnode->canCommit), 0, 1);
 
   // open buffer pool
@@ -175,6 +177,7 @@ void vnodeClose(SVnode *pVnode) {
     vnodeCloseBufPool(pVnode);
     // destroy handle
     tsem_destroy(&(pVnode->canCommit));
+    tsem_destroy(&pVnode->syncSem);
     taosMemoryFree(pVnode);
   }
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 7a9c9ef393..1524d8fe6f 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -91,9 +91,6 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
       }
 
     } break;
-    case TDMT_VND_ALTER_REPLICA: {
-      code = vnodeSyncAlter(pVnode, pMsg);
-    } break;
     default:
       break;
   }
@@ -107,7 +104,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   int32_t len;
   int32_t ret;
 
-  vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+  vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
          version);
 
   pVnode->state.applied = version;
@@ -173,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       break;
   }
 
-  vTrace("vgId:%d, process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
+  vTrace("vgId:%d, process %s request success, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
 
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
     vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 8f98e9cec5..efff466c18 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -16,41 +16,42 @@
 #define _DEFAULT_SOURCE
 #include "vnd.h"
 
-static int32_t   vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg);
-static int32_t   vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg);
-static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode);
-static void      vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static void      vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static void      vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static int32_t   vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
-
-int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
-  SSyncInfo syncInfo = {
-      .vgId = pVnode->config.vgId,
-      .isStandBy = pVnode->config.standby,
-      .syncCfg = pVnode->config.syncCfg,
-      .pWal = pVnode->pWal,
-      .msgcb = NULL,
-      .FpSendMsg = vnodeSyncSendMsg,
-      .FpEqMsg = vnodeSyncEqMsg,
-  };
-
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP);
-  syncInfo.pFsm = vnodeSyncMakeFsm(pVnode);
-
-  pVnode->sync = syncOpen(&syncInfo);
-  if (pVnode->sync <= 0) {
-    vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
-    return -1;
-  }
-
-  setPingTimerMS(pVnode->sync, 3000);
-  setElectTimerMS(pVnode->sync, 500);
-  setHeartbeatTimerMS(pVnode->sync, 100);
-  return 0;
+static inline bool vnodeIsMsgBlock(tmsg_t type) {
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
 }
 
-int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
+static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
+
+static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
+
+  int32_t count = atomic_add_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+}
+
+static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
+  tsem_wait(&pVnode->syncSem);
+  vTrace("vgId:%d, ===> block finish, count:%d", pVnode->config.vgId, count);
+}
+
+static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
+
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  count = atomic_sub_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+  if (count <= 0) {
+    tsem_post(&pVnode->syncSem);
+  }
+}
+
+static int32_t vnodeProcessSyncReconfigReq(SVnode *pVnode, SRpcMsg *pMsg) {
   SAlterVnodeReq req = {0};
   if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -66,29 +67,107 @@ int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
     vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
   }
 
-  int32_t code = syncReconfig(pVnode->sync, &cfg);
-  if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-    // todo refactor
-    SRpcMsg rsp = {.info = pMsg->info, .code = 0};
-    tmsgSendRsp(&rsp);
-    return TSDB_CODE_ACTION_IN_PROGRESS;
-  }
-
-  return code;
+  return syncReconfig(pVnode->sync, &cfg);
 }
 
-void vnodeSyncStart(SVnode *pVnode) {
-  syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
-  if (pVnode->config.standby) {
-    syncStartStandBy(pVnode->sync);
-  } else {
-    syncStart(pVnode->sync);
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t m = 0; m < numOfMsgs; m++) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle);
+
+    if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+      code = vnodeProcessSyncReconfigReq(pVnode, pMsg);
+    } else {
+      code = vnodePreprocessReq(pVnode, pMsg);
+      if (code != 0) {
+        vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
+      } else {
+        code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
+      }
+    }
+
+    if (code == 0) {
+      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
+    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      SEpSet newEpSet = {0};
+      syncGetEpSet(pVnode->sync, &newEpSet);
+      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
+      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
+        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
+      }
+
+      vTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
+             newEpSet.inUse);
+      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
+        vTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
+      }
+
+      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
+      tmsgSendRedirectRsp(&rsp, &newEpSet);
+    } else {
+      if (terrno != 0) code = terrno;
+      vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
+      SRpcMsg rsp = {.code = code, .info = pMsg->info};
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+
+  vnodeWaitBlockMsg(pVnode);
+}
+
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-apply queue", vgId, pMsg);
+
+    // init response rpc msg
+    SRpcMsg rsp = {0};
+
+    // get original rpc msg
+    assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
+    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
+    SRpcMsg originalRpcMsg;
+    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
+
+    // apply data into tsdb
+    if (vnodeProcessWriteReq(pVnode, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
+      rsp.code = terrno;
+      vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
+    }
+
+    syncApplyMsgDestroy(pSyncApplyMsg);
+    rpcFreeCont(originalRpcMsg.pCont);
+
+    vnodePostBlockMsg(pVnode, originalRpcMsg.msgType);
+
+    // if leader, send response
+    if (pMsg->info.handle != NULL) {
+      rsp.info = pMsg->info;
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x handle:%p", vgId, pMsg, rsp.code, pMsg->info.handle);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
   }
 }
 
-void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }
-
-int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
+static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
   if (code != 0) {
     rpcFreeCont(pMsg->pCont);
@@ -97,7 +176,7 @@ int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   return code;
 }
 
-int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   int32_t code = tmsgSendReq(pEpSet, pMsg);
   if (code != 0) {
     rpcFreeCont(pMsg->pCont);
@@ -106,19 +185,19 @@ int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   return code;
 }
 
-int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   vnodeGetSnapshot(pFsm->data, pSnapshot);
   return 0;
 }
 
-void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
+static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
   SVnode *pVnode = pFsm->data;
   vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
 
   // todo rpc response here
 }
 
-void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
   if (pFsm->FpGetSnapshot != NULL) {
     SSnapshot snapshot = {0};
@@ -164,7 +243,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
   }
 }
 
-void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
@@ -172,14 +251,14 @@ void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMet
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
            pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
+static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
   pFsm->data = pVnode;
   pFsm->FpCommitCb = vnodeSyncCommitMsg;
@@ -188,6 +267,42 @@ SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
   pFsm->FpRestoreFinishCb = NULL;
   pFsm->FpReConfigCb = vnodeSyncReconfig;
-
   return pFsm;
-}
\ No newline at end of file
+}
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
+  SSyncInfo syncInfo = {
+      .vgId = pVnode->config.vgId,
+      .isStandBy = pVnode->config.standby,
+      .syncCfg = pVnode->config.syncCfg,
+      .pWal = pVnode->pWal,
+      .msgcb = NULL,
+      .FpSendMsg = vnodeSyncSendMsg,
+      .FpEqMsg = vnodeSyncEqMsg,
+  };
+
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP);
+  syncInfo.pFsm = vnodeSyncMakeFsm(pVnode);
+
+  pVnode->sync = syncOpen(&syncInfo);
+  if (pVnode->sync <= 0) {
+    vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
+    return -1;
+  }
+
+  setPingTimerMS(pVnode->sync, 3000);
+  setElectTimerMS(pVnode->sync, 500);
+  setHeartbeatTimerMS(pVnode->sync, 100);
+  return 0;
+}
+
+void vnodeSyncStart(SVnode *pVnode) {
+  syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
+  if (pVnode->config.standby) {
+    syncStartStandBy(pVnode->sync);
+  } else {
+    syncStart(pVnode->sync);
+  }
+}
+
+void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }

From 9e9381a6c51a4f1d6634b1a5ceabc7e34a5fd98c Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 20:28:24 +0800
Subject: [PATCH 40/63] fix bug

---
 source/client/src/clientImpl.c   | 3 ++-
 source/libs/parser/src/parUtil.c | 2 --
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 0cc4b2f5d8..6e86dc6522 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -704,7 +704,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       break;
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
-      pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+      pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;      
+      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
       break;
     default:
       break;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index e5902dff1d..192a4baf80 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -714,8 +714,6 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName,
   SVgroupInfo* pVg = NULL;
   int32_t      code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableVgroup, (void**)&pVg);
   if (TSDB_CODE_SUCCESS == code) {
-    ASSERT(0 != pVg->vgId);
-    ASSERT(0 != pVg->epSet.eps[pVg->epSet.inUse].fqdn[0]);
     memcpy(pVgroup, pVg, sizeof(SVgroupInfo));
   }
   return code;

From ea05f8af12142b547b9afe4b45e52593aadc5869 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 20:30:43 +0800
Subject: [PATCH 41/63] fix bugs

---
 source/libs/function/src/builtins.c     | 2 +-
 source/libs/function/src/builtinsimpl.c | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index f6414a6e20..e900d2d49a 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1463,7 +1463,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .classification = FUNC_MGT_AGG_FUNC,
     .translateFunc = translateHistogramMerge,
     .getEnvFunc   = getHistogramFuncEnv,
-    .initFunc     = histogramFunctionSetup,
+    .initFunc     = functionSetup,
     .processFunc  = histogramFunctionMerge,
     .finalizeFunc = histogramFinalize
   },
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 41414efba5..c279ceb01e 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -3345,11 +3345,12 @@ int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) {
   pInfo->numOfBins  = pInputInfo->numOfBins;
   pInfo->totalCount += pInputInfo->totalCount;
   for (int32_t k = 0; k < pInfo->numOfBins; ++k) {
+    pInfo->bins[k].lower = pInputInfo->bins[k].lower;
+    pInfo->bins[k].upper = pInputInfo->bins[k].upper;
     pInfo->bins[k].count += pInputInfo->bins[k].count;
   }
 
-  SET_VAL(GET_RES_INFO(pCtx), 1, 1);
-
+  SET_VAL(GET_RES_INFO(pCtx), pInfo->numOfBins, pInfo->numOfBins);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3404,7 +3405,7 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   colDataAppend(pCol, pBlock->info.rows, res, false);
 
   taosMemoryFree(res);
-  return pResInfo->numOfRes;
+  return 1;
 }
 
 bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {

From 594c370543dd0ea82fa23cf17293fe84261d92de Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 8 Jun 2022 20:39:46 +0800
Subject: [PATCH 42/63] fix error handle issue

---
 source/client/src/clientImpl.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 6e86dc6522..60dfaf1278 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -697,6 +697,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       if (TSDB_CODE_SUCCESS == code) {
         schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob,
                               pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest);
+      } else {
+        pRequest->body.queryFp(pRequest->body.param, pRequest, code);
       }
 
       //todo not to be released here

From 90024cb1302eb9463107588ffb53a9430e57a1e4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 8 Jun 2022 20:56:38 +0800
Subject: [PATCH 43/63] refactor: do some internal refactor.

---
 source/client/src/clientImpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 9da0c77653..48ca5073df 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1348,7 +1348,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     int32_t bytes = *(int32_t*) p;
     p += sizeof(int32_t);
 
-    ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
+//    ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
   }
 
   int32_t* colLength = (int32_t*)p;

From bc8e6b7fd7a2a7d02cda9d0d471e786bf1c6ecc9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 8 Jun 2022 21:17:17 +0800
Subject: [PATCH 44/63] refactor: adjust vnode sync

---
 include/libs/sync/syncTools.h          |  2 +-
 include/libs/transport/trpc.h          |  5 +-
 source/dnode/vnode/src/vnd/vnodeSync.c | 74 ++++++++++----------------
 source/libs/sync/src/syncMain.c        |  4 +-
 4 files changed, 34 insertions(+), 51 deletions(-)

diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index 4b160c9e61..bd396edf55 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -33,7 +33,7 @@ SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
 
 int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
-int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo);
 void    syncSetMsgCb(int64_t rid, const SMsgCb* msgcb);
 char*   sync2SimpleStr(int64_t rid);
 
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 839194da94..1535f9d290 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -33,7 +33,10 @@ extern int32_t tsRpcHeadSize;
 typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
-  char     user[TSDB_USER_LEN];
+  union {
+    char    user[TSDB_USER_LEN];
+    int64_t applyIndex;
+  };
 } SRpcConnInfo;
 
 typedef struct SRpcHandleInfo {
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index efff466c18..203eecd8ab 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -35,7 +35,6 @@ static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
 
   vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
   tsem_wait(&pVnode->syncSem);
-  vTrace("vgId:%d, ===> block finish, count:%d", pVnode->config.vgId, count);
 }
 
 static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
@@ -93,6 +92,11 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
 
     if (code == 0) {
       vnodeAccumBlockMsg(pVnode, pMsg->msgType);
+      if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+        // todo refactor
+        SRpcMsg rsp = {.code = code, .info = pMsg->info};
+        tmsgSendRsp(&rsp);
+      }
     } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
       SEpSet newEpSet = {0};
       syncGetEpSet(pVnode->sync, &newEpSet);
@@ -132,36 +136,22 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
     if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    vTrace("vgId:%d, msg:%p get from vnode-apply queue", vgId, pMsg);
+    vTrace("vgId:%d, msg:%p get from vnode-apply queue, handle:%p", vgId, pMsg, pMsg->info.handle);
 
-    // init response rpc msg
-    SRpcMsg rsp = {0};
-
-    // get original rpc msg
-    assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
-    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
-    SRpcMsg originalRpcMsg;
-    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
-
-    // apply data into tsdb
-    if (vnodeProcessWriteReq(pVnode, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
-      rsp.code = terrno;
-      vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
+    SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
+    if (rsp.code == 0) {
+      if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->conn.applyIndex, &rsp) < 0) {
+        rsp.code = terrno;
+        vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
+      }
     }
 
-    syncApplyMsgDestroy(pSyncApplyMsg);
-    rpcFreeCont(originalRpcMsg.pCont);
-
-    vnodePostBlockMsg(pVnode, originalRpcMsg.msgType);
-
-    // if leader, send response
-    if (pMsg->info.handle != NULL) {
-      rsp.info = pMsg->info;
+    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    if (rsp.info.handle != NULL) {
       tmsgSendRsp(&rsp);
     }
 
-    vTrace("vgId:%d, msg:%p is freed, code:0x%x handle:%p", vgId, pMsg, rsp.code, pMsg->info.handle);
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, rsp.code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
@@ -195,43 +185,33 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigC
   vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
 
   // todo rpc response here
+  // build rpc msg
+  // put into apply queue
 }
 
 static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  SVnode   *pVnode = pFsm->data;
+  SSnapshot snapshot = {0};
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  char      logBuf[256] = {0};
+
   if (pFsm->FpGetSnapshot != NULL) {
-    SSnapshot snapshot = {0};
-    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    (*pFsm->FpGetSnapshot)(pFsm, &snapshot);
     beginIndex = snapshot.lastApplyIndex;
   }
 
   if (cbMeta.index > beginIndex) {
-    char logBuf[256] = {0};
     snprintf(
         logBuf, sizeof(logBuf),
         "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
         pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
     syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 
-    SVnode       *pVnode = pFsm->data;
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
-    SRpcMsg       applyMsg;
-    syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
-    syncApplyMsgDestroy(pSyncApplyMsg);
-
-    // recover handle for response
-    SRpcMsg saveRpcMsg;
-    int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
-    if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
-      applyMsg.info = saveRpcMsg.info;
-    } else {
-      applyMsg.info.handle = NULL;
-      applyMsg.info.ahandle = NULL;
-    }
-
-    // put to applyQ
-    tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
-
+    SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index};
+    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+    memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
+    syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
+    tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
   } else {
     char logBuf[256] = {0};
     snprintf(logBuf, sizeof(logBuf),
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index d522b829dd..795d3e3c27 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -281,7 +281,7 @@ int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
   return ret;
 }
 
-int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
     return TAOS_SYNC_STATE_ERROR;
@@ -291,7 +291,7 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
   SRespStub stub;
   int32_t   ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub);
   if (ret == 1) {
-    memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg));
+    *pInfo = stub.rpcMsg.info;
   }
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);

From dcc78597bfbfbb1005423640df2f663bdc43c98b Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 8 Jun 2022 21:51:28 +0800
Subject: [PATCH 45/63] refactor index code

---
 source/libs/index/inc/indexCache.h         |  4 +-
 source/libs/index/inc/indexComm.h          |  4 +-
 source/libs/index/inc/indexFst.h           | 59 +++++++-------
 source/libs/index/inc/indexFstAutomation.h | 26 +++----
 source/libs/index/src/index.c              | 50 ++++++------
 source/libs/index/src/indexCache.c         | 89 +++++++++++-----------
 source/libs/index/src/indexComm.c          |  4 +-
 source/libs/index/src/indexFst.c           | 49 ++++++------
 source/libs/index/src/indexFstAutomation.c | 42 +++++-----
 source/libs/index/src/indexTfile.c         | 89 +++++++++-------------
 source/libs/index/test/fstTest.cc          | 72 ++++++++---------
 source/libs/index/test/fstUT.cc            | 16 ++--
 source/libs/index/test/indexTests.cc       | 14 ++--
 13 files changed, 248 insertions(+), 270 deletions(-)

diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h
index 6e68163d74..6c95eb987b 100644
--- a/source/libs/index/inc/indexCache.h
+++ b/source/libs/index/inc/indexCache.h
@@ -70,7 +70,7 @@ void indexCacheBroadcast(void* cache);
 void indexCacheWait(void* cache);
 
 Iterate* indexCacheIteratorCreate(IndexCache* cache);
-void     indexCacheIteratorDestroy(Iterate* iiter);
+void     idxCacheIteratorDestroy(Iterate* iiter);
 
 int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
 
@@ -82,7 +82,7 @@ void indexCacheUnRef(IndexCache* cache);
 
 void indexCacheDebug(IndexCache* cache);
 
-void indexCacheDestroyImm(IndexCache* cache);
+void idxCacheDestroyImm(IndexCache* cache);
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h
index bcf76b4f20..1e001ee5d7 100644
--- a/source/libs/index/inc/indexComm.h
+++ b/source/libs/index/inc/indexComm.h
@@ -26,8 +26,8 @@ extern "C" {
 extern char JSON_COLUMN[];
 extern char JSON_VALUE_DELIM;
 
-char* indexPackJsonData(SIndexTerm* itm);
-char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
+char* idxPackJsonData(SIndexTerm* itm);
+char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
 
 typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
diff --git a/source/libs/index/inc/indexFst.h b/source/libs/index/inc/indexFst.h
index 0a360c1c72..6fd12c110c 100644
--- a/source/libs/index/inc/indexFst.h
+++ b/source/libs/index/inc/indexFst.h
@@ -29,9 +29,9 @@ extern "C" {
 
 #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
 
-typedef struct Fst             Fst;
-typedef struct FstNode         FstNode;
-typedef struct StreamWithState StreamWithState;
+typedef struct Fst     Fst;
+typedef struct FstNode FstNode;
+typedef struct FStmSt  FStmSt;
 
 typedef enum { Included, Excluded, Unbounded } FstBound;
 
@@ -40,12 +40,12 @@ typedef struct FstBoundWithData {
   FstBound type;
 } FstBoundWithData;
 
-typedef struct FstStreamBuilder {
+typedef struct FStmBuilder {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   FstBoundWithData* min;
   FstBoundWithData* max;
-} FstStreamBuilder, FstStreamWithStateBuilder;
+} FStmBuilder, FStmStBuilder;
 
 typedef struct FstRange {
   uint64_t start;
@@ -267,17 +267,17 @@ typedef struct Fst {
 Fst* fstCreate(FstSlice* data);
 void fstDestroy(Fst* fst);
 
-bool              fstGet(Fst* fst, FstSlice* b, Output* out);
-FstNode*          fstGetNode(Fst* fst, CompiledAddr);
-FstNode*          fstGetRoot(Fst* fst);
-FstType           fstGetType(Fst* fst);
-CompiledAddr      fstGetRootAddr(Fst* fst);
-Output            fstEmptyFinalOutput(Fst* fst, bool* null);
-FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx);
+bool         fstGet(Fst* fst, FstSlice* b, Output* out);
+FstNode*     fstGetNode(Fst* fst, CompiledAddr);
+FstNode*     fstGetRoot(Fst* fst);
+FstType      fstGetType(Fst* fst);
+CompiledAddr fstGetRootAddr(Fst* fst);
+Output       fstEmptyFinalOutput(Fst* fst, bool* null);
+FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx);
 
-FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx);
+FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx);
 // into stream to expand later
-StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb);
+FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
 
 bool fstVerify(Fst* fst);
 
@@ -293,41 +293,40 @@ typedef struct StreamState {
 
 void streamStateDestroy(void* s);
 
-typedef struct StreamWithState {
+typedef struct FStmSt {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   SArray*           inp;
   FstOutput         emptyOutput;
   SArray*           stack;  // <StreamState>
   FstBoundWithData* endAt;
-} StreamWithState;
+} FStmSt;
 
-typedef struct StreamWithStateResult {
+typedef struct FStmStRslt {
   FstSlice  data;
   FstOutput out;
   void*     state;
-} StreamWithStateResult;
+} FStmStRslt;
 
-StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
-void                   swsResultDestroy(StreamWithStateResult* result);
+FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
+void        swsResultDestroy(FStmStRslt* result);
 
 typedef void* (*StreamCallback)(void*);
-StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
-                                       FstBoundWithData* max);
+FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
 
-void streamWithStateDestroy(StreamWithState* sws);
+void stmStDestroy(FStmSt* sws);
 
-bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min);
+bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min);
 
-StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback);
+FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback);
 
-FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
+FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut);
 
-void fstStreamBuilderDestroy(FstStreamBuilder* b);
+void stmBuilderDestroy(FStmBuilder* b);
 
 // set up bound range
 // refator later:  to simple code by marco
-void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
+void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/index/inc/indexFstAutomation.h b/source/libs/index/inc/indexFstAutomation.h
index 3a0f74ee76..583bbfedae 100644
--- a/source/libs/index/inc/indexFstAutomation.h
+++ b/source/libs/index/inc/indexFstAutomation.h
@@ -22,24 +22,24 @@ extern "C" {
 #include "indexFstUtil.h"
 #include "indexInt.h"
 
-typedef struct AutomationCtx AutomationCtx;
+typedef struct FAutoCtx FAutoCtx;
 
 typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
 
 typedef struct StartWith {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } StartWith;
 
 typedef struct Complement {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } Complement;
 
 // automation
-typedef struct AutomationCtx {
+typedef struct FAutoCtx {
   AutomationType type;
   void*          stdata;
   char*          data;
-} AutomationCtx;
+} FAutoCtx;
 
 typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
 typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
@@ -60,16 +60,16 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv);
 void                 startWithStateValueDestroy(void* sv);
 
 typedef struct AutomationFunc {
-  void* (*start)(AutomationCtx* ctx);
-  bool (*isMatch)(AutomationCtx* ctx, void*);
-  bool (*canMatch)(AutomationCtx* ctx, void* data);
-  bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state);
-  void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte);
-  void* (*acceptEof)(AutomationCtx* ct, void* state);
+  void* (*start)(FAutoCtx* ctx);
+  bool (*isMatch)(FAutoCtx* ctx, void*);
+  bool (*canMatch)(FAutoCtx* ctx, void* data);
+  bool (*willAlwaysMatch)(FAutoCtx* ctx, void* state);
+  void* (*accept)(FAutoCtx* ctx, void* state, uint8_t byte);
+  void* (*acceptEof)(FAutoCtx* ct, void* state);
 } AutomationFunc;
 
-AutomationCtx* automCtxCreate(void* data, AutomationType atype);
-void           automCtxDestroy(AutomationCtx* ctx);
+FAutoCtx* automCtxCreate(void* data, AutomationType atype);
+void      automCtxDestroy(FAutoCtx* ctx);
 
 extern AutomationFunc automFuncs[];
 #ifdef __cplusplus
diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index a5b158c00c..c4b85d11e0 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -77,15 +77,15 @@ typedef struct SIdxColInfo {
 
 static TdThreadOnce isInit = PTHREAD_ONCE_INIT;
 // static void           indexInit();
-static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
+static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
 
-static void indexInterResultsDestroy(SArray* results);
-static int  indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out);
+static void idxInterRsltDestroy(SArray* results);
+static int  idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out);
 
-static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
+static int idxGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
 
 // merge cache and tfile by opera type
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
+static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
 
 // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 // int32_t        indexSerialKey(ICacheKey* key, char* buf);
@@ -221,11 +221,11 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
   for (size_t i = 0; i < nQuery; i++) {
     SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i);
     SArray*          trslt = NULL;
-    indexTermSearch(index, qterm, &trslt);
+    idxTermSearch(index, qterm, &trslt);
     taosArrayPush(iRslts, (void*)&trslt);
   }
-  indexMergeFinalResults(iRslts, opera, result);
-  indexInterResultsDestroy(iRslts);
+  idxMergeFinalResults(iRslts, opera, result);
+  idxInterRsltDestroy(iRslts);
   return 0;
 }
 
@@ -319,7 +319,7 @@ void indexMultiTermDestroy(SIndexMultiTerm* terms) {
   taosArrayDestroy(terms);
 }
 
-static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
+static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
   SIndexTerm* term = query->term;
   const char* colName = term->colName;
   int32_t     nColName = term->nColName;
@@ -374,7 +374,7 @@ END:
   idxTRsltDestroy(tr);
   return -1;
 }
-static void indexInterResultsDestroy(SArray* results) {
+static void idxInterRsltDestroy(SArray* results) {
   if (results == NULL) {
     return;
   }
@@ -387,7 +387,7 @@ static void indexInterResultsDestroy(SArray* results) {
   taosArrayDestroy(results);
 }
 
-static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out) {
+static int idxMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray* out) {
   // refactor, merge interResults into fResults by oType
   for (int i = 0; i < taosArrayGetSize(in); i--) {
     SArray* t = taosArrayGetP(in, i);
@@ -407,7 +407,7 @@ static int indexMergeFinalResults(SArray* in, EIndexOperatorType oType, SArray*
   return 0;
 }
 
-static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
+static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
   int32_t sz = taosArrayGetSize(result);
   if (sz > 0) {
     TFileValue* lv = taosArrayGetP(result, sz - 1);
@@ -427,11 +427,11 @@ static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdx
     taosArrayPush(result, &tfv);
   }
 }
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
+static void idxMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
   char*       colVal = (cv != NULL) ? cv->colVal : tv->colVal;
   TFileValue* tfv = tfileValueCreate(colVal);
 
-  indexMayMergeTempToFinalResult(result, tfv, tr);
+  idxMayMergeTempToFinalRslt(result, tfv, tr);
 
   if (cv != NULL) {
     uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
@@ -446,7 +446,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateVal
     taosArrayAddAll(tr->total, tv->val);
   }
 }
-static void indexDestroyFinalResult(SArray* result) {
+static void idxDestroyFinalRslt(SArray* result) {
   int32_t sz = result ? taosArrayGetSize(result) : 0;
   for (size_t i = 0; i < sz; i++) {
     TFileValue* tv = taosArrayGetP(result, i);
@@ -475,7 +475,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
   Iterate* cacheIter = indexCacheIteratorCreate(pCache);
   if (cacheIter == NULL) {
     indexError("%p immtable is empty, ignore merge opera", pCache);
-    indexCacheDestroyImm(pCache);
+    idxCacheDestroyImm(pCache);
     tfileReaderUnRef(pReader);
     atomic_store_32(&pCache->merging, 0);
     if (quit) {
@@ -509,26 +509,26 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
       comp = 1;
     }
     if (comp == 0) {
-      indexMergeCacheAndTFile(result, cv, tv, tr);
+      idxMergeCacheAndTFile(result, cv, tv, tr);
       cn = cacheIter->next(cacheIter);
       tn = tfileIter->next(tfileIter);
     } else if (comp < 0) {
-      indexMergeCacheAndTFile(result, cv, NULL, tr);
+      idxMergeCacheAndTFile(result, cv, NULL, tr);
       cn = cacheIter->next(cacheIter);
     } else {
-      indexMergeCacheAndTFile(result, NULL, tv, tr);
+      idxMergeCacheAndTFile(result, NULL, tv, tr);
       tn = tfileIter->next(tfileIter);
     }
   }
-  indexMayMergeTempToFinalResult(result, NULL, tr);
+  idxMayMergeTempToFinalRslt(result, NULL, tr);
   idxTRsltDestroy(tr);
 
-  int ret = indexGenTFile(sIdx, pCache, result);
-  indexDestroyFinalResult(result);
+  int ret = idxGenTFile(sIdx, pCache, result);
+  idxDestroyFinalRslt(result);
 
-  indexCacheDestroyImm(pCache);
+  idxCacheDestroyImm(pCache);
 
-  indexCacheIteratorDestroy(cacheIter);
+  idxCacheIteratorDestroy(cacheIter);
   tfileIteratorDestroy(tfileIter);
 
   tfileReaderUnRef(pReader);
@@ -578,7 +578,7 @@ static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) {
   tfileReaderUnRef(rd);
   return ver;
 }
-static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
+static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   int64_t version = indexGetAvaialbleVer(sIdx, cache);
   indexInfo("file name version: %" PRId64 "", version);
   uint8_t colType = cache->type;
diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c
index c1fb086fa0..416c03bd0f 100644
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@@ -26,15 +26,15 @@
 #define MEM_SIGNAL_QUIT    MEM_THRESHOLD * 20
 #define MEM_ESTIMATE_RADIO 1.5
 
-static void indexMemRef(MemTable* tbl);
-static void indexMemUnRef(MemTable* tbl);
+static void idxMemRef(MemTable* tbl);
+static void idxMemUnRef(MemTable* tbl);
 
-static void    indexCacheTermDestroy(CacheTerm* ct);
-static int32_t indexCacheTermCompare(const void* l, const void* r);
-static int32_t indexCacheJsonTermCompare(const void* l, const void* r);
-static char*   indexCacheTermGet(const void* pData);
+static void    idxCacheTermDestroy(CacheTerm* ct);
+static int32_t idxCacheTermCompare(const void* l, const void* r);
+static int32_t idxCacheJsonTermCompare(const void* l, const void* r);
+static char*   idxCacheTermGet(const void* pData);
 
-static MemTable* indexInternalCacheCreate(int8_t type);
+static MemTable* idxInternalCacheCreate(int8_t type);
 
 static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
 static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
@@ -81,7 +81,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
   pCt->colVal = term->colVal;
   pCt->version = atomic_load_64(&pCache->version);
 
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -134,7 +134,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
   pCt->colType = term->colType;
   pCt->version = atomic_load_64(&pCache->version);
 
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
   while (tSkipListIterNext(iter)) {
@@ -188,10 +188,10 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
 
   char* exBuf = NULL;
   if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
-    exBuf = indexPackJsonData(term);
+    exBuf = idxPackJsonData(term);
     pCt->colVal = exBuf;
   }
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
   while (tSkipListIterNext(iter)) {
@@ -268,10 +268,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
   char*  exBuf = NULL;
 
   if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
-    exBuf = indexPackJsonDataPrefix(term, &skip);
+    exBuf = idxPackJsonDataPrefix(term, &skip);
     pCt->colVal = exBuf;
   }
-  char* key = indexCacheTermGet(pCt);
+  char* key = idxCacheTermGet(pCt);
 
   // SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
@@ -324,7 +324,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
     return NULL;
   };
 
-  cache->mem = indexInternalCacheCreate(type);
+  cache->mem = idxInternalCacheCreate(type);
   cache->mem->pCache = cache;
   cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
   cache->type = type;
@@ -347,7 +347,7 @@ void indexCacheDebug(IndexCache* cache) {
 
   taosThreadMutexLock(&cache->mtx);
   tbl = cache->mem;
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   taosThreadMutexUnlock(&cache->mtx);
 
   {
@@ -363,13 +363,13 @@ void indexCacheDebug(IndexCache* cache) {
     }
     tSkipListDestroyIter(iter);
 
-    indexMemUnRef(tbl);
+    idxMemUnRef(tbl);
   }
 
   {
     taosThreadMutexLock(&cache->mtx);
     tbl = cache->imm;
-    indexMemRef(tbl);
+    idxMemRef(tbl);
     taosThreadMutexUnlock(&cache->mtx);
     if (tbl != NULL) {
       SSkipList*         slt = tbl->mem;
@@ -385,7 +385,7 @@ void indexCacheDebug(IndexCache* cache) {
       tSkipListDestroyIter(iter);
     }
 
-    indexMemUnRef(tbl);
+    idxMemUnRef(tbl);
   }
 }
 
@@ -410,7 +410,7 @@ void indexCacheWait(void* cache) {
   IndexCache* pCache = cache;
   taosThreadCondWait(&pCache->finished, &pCache->mtx);
 }
-void indexCacheDestroyImm(IndexCache* cache) {
+void idxCacheDestroyImm(IndexCache* cache) {
   if (cache == NULL) {
     return;
   }
@@ -423,8 +423,8 @@ void indexCacheDestroyImm(IndexCache* cache) {
 
   taosThreadMutexUnlock(&cache->mtx);
 
-  indexMemUnRef(tbl);
-  indexMemUnRef(tbl);
+  idxMemUnRef(tbl);
+  idxMemUnRef(tbl);
 }
 void indexCacheDestroy(void* cache) {
   IndexCache* pCache = cache;
@@ -432,8 +432,8 @@ void indexCacheDestroy(void* cache) {
     return;
   }
 
-  indexMemUnRef(pCache->mem);
-  indexMemUnRef(pCache->imm);
+  idxMemUnRef(pCache->mem);
+  idxMemUnRef(pCache->imm);
   taosMemoryFree(pCache->colName);
 
   taosThreadMutexDestroy(&pCache->mtx);
@@ -454,7 +454,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   }
   taosThreadMutexLock(&cache->mtx);
 
-  indexMemRef(cache->imm);
+  idxMemRef(cache->imm);
 
   MemTable* tbl = cache->imm;
   iiter->val.val = taosArrayInit(1, sizeof(uint64_t));
@@ -467,7 +467,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
 
   return iiter;
 }
-void indexCacheIteratorDestroy(Iterate* iter) {
+void idxCacheIteratorDestroy(Iterate* iter) {
   if (iter == NULL) {
     return;
   }
@@ -501,7 +501,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
 
       indexCacheRef(cache);
       cache->imm = cache->mem;
-      cache->mem = indexInternalCacheCreate(cache->type);
+      cache->mem = idxInternalCacheCreate(cache->type);
       cache->mem->pCache = cache;
       cache->occupiedMem = 0;
       if (quit == false) {
@@ -529,7 +529,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
   // set up key
   ct->colType = term->colType;
   if (hasJson) {
-    ct->colVal = indexPackJsonData(term);
+    ct->colVal = idxPackJsonData(term);
   } else {
     ct->colVal = (char*)taosMemoryCalloc(1, sizeof(char) * (term->nColVal + 1));
     memcpy(ct->colVal, term->colVal, term->nColVal);
@@ -545,9 +545,9 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
   pCache->occupiedMem += estimate;
   indexCacheMakeRoomForWrite(pCache);
   MemTable* tbl = pCache->mem;
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   tSkipListPut(tbl->mem, (char*)ct);
-  indexMemUnRef(tbl);
+  idxMemUnRef(tbl);
 
   taosThreadMutexUnlock(&pCache->mtx);
 
@@ -598,8 +598,8 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
   taosThreadMutexLock(&pCache->mtx);
   mem = pCache->mem;
   imm = pCache->imm;
-  indexMemRef(mem);
-  indexMemRef(imm);
+  idxMemRef(mem);
+  idxMemRef(imm);
   taosThreadMutexUnlock(&pCache->mtx);
 
   int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0;
@@ -608,8 +608,8 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
     ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0;
   }
 
-  indexMemUnRef(mem);
-  indexMemUnRef(imm);
+  idxMemUnRef(mem);
+  idxMemUnRef(imm);
   indexInfo("cache search, time cost %" PRIu64 "us", taosGetTimestampUs() - st);
 
   return ret;
@@ -632,14 +632,14 @@ void indexCacheUnRef(IndexCache* cache) {
   }
 }
 
-void indexMemRef(MemTable* tbl) {
+void idxMemRef(MemTable* tbl) {
   if (tbl == NULL) {
     return;
   }
   int ref = T_REF_INC(tbl);
   UNUSED(ref);
 }
-void indexMemUnRef(MemTable* tbl) {
+void idxMemUnRef(MemTable* tbl) {
   if (tbl == NULL) {
     return;
   }
@@ -651,18 +651,18 @@ void indexMemUnRef(MemTable* tbl) {
   }
 }
 
-static void indexCacheTermDestroy(CacheTerm* ct) {
+static void idxCacheTermDestroy(CacheTerm* ct) {
   if (ct == NULL) {
     return;
   }
   taosMemoryFree(ct->colVal);
   taosMemoryFree(ct);
 }
-static char* indexCacheTermGet(const void* pData) {
+static char* idxCacheTermGet(const void* pData) {
   CacheTerm* p = (CacheTerm*)pData;
   return (char*)p;
 }
-static int32_t indexCacheTermCompare(const void* l, const void* r) {
+static int32_t idxCacheTermCompare(const void* l, const void* r) {
   CacheTerm* lt = (CacheTerm*)l;
   CacheTerm* rt = (CacheTerm*)r;
   // compare colVal
@@ -683,7 +683,7 @@ static int indexFindCh(char* a, char c) {
   }
   return p - a;
 }
-static int indexCacheJsonTermCompareImpl(char* a, char* b) {
+static int idxCacheJsonTermCompareImpl(char* a, char* b) {
   // int alen = indexFindCh(a, '&');
   // int blen = indexFindCh(b, '&');
 
@@ -703,7 +703,7 @@ static int indexCacheJsonTermCompareImpl(char* a, char* b) {
   //}
   return 0;
 }
-static int32_t indexCacheJsonTermCompare(const void* l, const void* r) {
+static int32_t idxCacheJsonTermCompare(const void* l, const void* r) {
   CacheTerm* lt = (CacheTerm*)l;
   CacheTerm* rt = (CacheTerm*)r;
   // compare colVal
@@ -713,16 +713,15 @@ static int32_t indexCacheJsonTermCompare(const void* l, const void* r) {
   }
   return cmp;
 }
-static MemTable* indexInternalCacheCreate(int8_t type) {
+static MemTable* idxInternalCacheCreate(int8_t type) {
   int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY;
   int32_t (*cmpFn)(const void* l, const void* r) =
-      INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? indexCacheJsonTermCompare : indexCacheTermCompare;
+      INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare;
 
   MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable));
-  indexMemRef(tbl);
+  idxMemRef(tbl);
   if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) {
-    tbl->mem =
-        tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, indexCacheTermGet);
+    tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet);
   }
   return tbl;
 }
diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index ca9d429e68..fdcc296574 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -201,7 +201,7 @@ static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {
 
 _cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
 
-char* indexPackJsonData(SIndexTerm* itm) {
+char* idxPackJsonData(SIndexTerm* itm) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
@@ -229,7 +229,7 @@ char* indexPackJsonData(SIndexTerm* itm) {
   return buf;
 }
 
-char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
+char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c
index 892716f387..b368c6faf3 100644
--- a/source/libs/index/src/indexFst.c
+++ b/source/libs/index/src/indexFst.c
@@ -1087,19 +1087,19 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
   *out = tOut;
   return true;
 }
-FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
+FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx) {
   // refactor later
-  return fstStreamBuilderCreate(fst, ctx);
+  return stmBuilderCreate(fst, ctx);
 }
-StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
+FStmSt* stmBuilderIntoStm(FStmBuilder* sb) {
   if (sb == NULL) {
     return NULL;
   }
-  return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
+  return stmStCreate(sb->fst, sb->aut, sb->min, sb->max);
 }
-FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
+FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx) {
   // refactor later
-  return fstStreamBuilderCreate(fst, ctx);
+  return stmBuilderCreate(fst, ctx);
 }
 
 FstNode* fstGetRoot(Fst* fst) {
@@ -1176,9 +1176,8 @@ bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type ==
 
 void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); }
 
-StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
-                                       FstBoundWithData* max) {
-  StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState));
+FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max) {
+  FStmSt* sws = taosMemoryCalloc(1, sizeof(FStmSt));
   if (sws == NULL) {
     return NULL;
   }
@@ -1192,11 +1191,11 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
 
   sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
   sws->endAt = max;
-  streamWithStateSeekMin(sws, min);
+  stmStSeekMin(sws, min);
 
   return sws;
 }
-void streamWithStateDestroy(StreamWithState* sws) {
+void stmStDestroy(FStmSt* sws) {
   if (sws == NULL) {
     return;
   }
@@ -1207,8 +1206,8 @@ void streamWithStateDestroy(StreamWithState* sws) {
   taosMemoryFree(sws);
 }
 
-bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
-  AutomationCtx* aut = sws->aut;
+bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
+  FAutoCtx* aut = sws->aut;
   if (fstBoundWithDataIsEmpty(min)) {
     if (fstBoundWithDataIsIncluded(min)) {
       sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
@@ -1301,9 +1300,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
 
   return false;
 }
-StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
-  AutomationCtx* aut = sws->aut;
-  FstOutput      output = sws->emptyOutput;
+FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
+  FAutoCtx* aut = sws->aut;
+  FstOutput output = sws->emptyOutput;
   if (output.null == false) {
     FstSlice emptySlice = fstSliceCreate(NULL, 0);
     if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) {
@@ -1367,8 +1366,8 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       return NULL;
     }
     if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
-      FstOutput              fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
-      StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
+      FstOutput   fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
+      FStmStRslt* result = swsResultCreate(&slice, fOutput, tState);
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
       taosArrayDestroy(nodes);
@@ -1382,8 +1381,8 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
   return NULL;
 }
 
-StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
-  StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult));
+FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
+  FStmStRslt* result = taosMemoryCalloc(1, sizeof(FStmStRslt));
   if (result == NULL) {
     return NULL;
   }
@@ -1393,7 +1392,7 @@ StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* sta
   result->state = state;
   return result;
 }
-void swsResultDestroy(StreamWithStateResult* result) {
+void swsResultDestroy(FStmStRslt* result) {
   if (NULL == result) {
     return;
   }
@@ -1411,8 +1410,8 @@ void streamStateDestroy(void* s) {
   fstNodeDestroy(ss->node);
 }
 
-FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
-  FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder));
+FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut) {
+  FStmBuilder* b = taosMemoryCalloc(1, sizeof(FStmBuilder));
   if (NULL == b) {
     return NULL;
   }
@@ -1423,14 +1422,14 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
   b->max = fstBoundStateCreate(Unbounded, NULL);
   return b;
 }
-void fstStreamBuilderDestroy(FstStreamBuilder* b) {
+void stmBuilderDestroy(FStmBuilder* b) {
   fstSliceDestroy(&b->min->data);
   fstSliceDestroy(&b->max->data);
   taosMemoryFreeClear(b->min);
   taosMemoryFreeClear(b->max);
   taosMemoryFree(b);
 }
-void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
+void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type) {
   if (b == NULL) {
     return;
   }
diff --git a/source/libs/index/src/indexFstAutomation.c b/source/libs/index/src/indexFstAutomation.c
index 0702e3b94e..385e832763 100644
--- a/source/libs/index/src/indexFstAutomation.c
+++ b/source/libs/index/src/indexFstAutomation.c
@@ -71,19 +71,19 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
 }
 
 // iterate fst
-static void* alwaysMatchStart(AutomationCtx* ctx) { return NULL; }
-static bool  alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { return true; }
-static bool  alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { return true; }
-static bool  alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
-static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; }
-static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* alwaysMatchStart(FAutoCtx* ctx) { return NULL; }
+static bool  alwaysMatchIsMatch(FAutoCtx* ctx, void* state) { return true; }
+static bool  alwaysMatchCanMatch(FAutoCtx* ctx, void* state) { return true; }
+static bool  alwaysMatchWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
+static void* alwaysMatchAccpet(FAutoCtx* ctx, void* state, uint8_t byte) { return NULL; }
+static void* alwaysMatchAccpetEof(FAutoCtx* ctx, void* state) { return NULL; }
 // prefix query, impl later
 
-static void* prefixStart(AutomationCtx* ctx) {
+static void* prefixStart(FAutoCtx* ctx) {
   StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata);
   return startWithStateValueDump(data);
 };
-static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
+static bool prefixIsMatch(FAutoCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
   if (ssv == NULL) {
     return false;
@@ -94,15 +94,15 @@ static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
     return false;
   }
 }
-static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
+static bool prefixCanMatch(FAutoCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
   if (ssv == NULL) {
     return false;
   }
   return ssv->val >= 0;
 }
-static bool  prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
-static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
+static bool  prefixWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
+static void* prefixAccept(FAutoCtx* ctx, void* state, uint8_t byte) {
   StartWithStateValue* ssv = (StartWithStateValue*)state;
   if (ssv == NULL || ctx == NULL) {
     return NULL;
@@ -125,18 +125,18 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
   }
   return NULL;
 }
-static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* prefixAcceptEof(FAutoCtx* ctx, void* state) { return NULL; }
 
 // pattern query, impl later
 
-static void* patternStart(AutomationCtx* ctx) { return NULL; }
-static bool  patternIsMatch(AutomationCtx* ctx, void* data) { return true; }
-static bool  patternCanMatch(AutomationCtx* ctx, void* data) { return true; }
-static bool  patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
+static void* patternStart(FAutoCtx* ctx) { return NULL; }
+static bool  patternIsMatch(FAutoCtx* ctx, void* data) { return true; }
+static bool  patternCanMatch(FAutoCtx* ctx, void* data) { return true; }
+static bool  patternWillAlwaysMatch(FAutoCtx* ctx, void* state) { return true; }
 
-static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; }
+static void* patternAccept(FAutoCtx* ctx, void* state, uint8_t byte) { return NULL; }
 
-static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; }
+static void* patternAcceptEof(FAutoCtx* ctx, void* state) { return NULL; }
 
 AutomationFunc automFuncs[] = {
     {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet,
@@ -146,8 +146,8 @@ AutomationFunc automFuncs[] = {
     // add more search type
 };
 
-AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
-  AutomationCtx* ctx = taosMemoryCalloc(1, sizeof(AutomationCtx));
+FAutoCtx* automCtxCreate(void* data, AutomationType atype) {
+  FAutoCtx* ctx = taosMemoryCalloc(1, sizeof(FAutoCtx));
   if (ctx == NULL) {
     return NULL;
   }
@@ -169,7 +169,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
   ctx->stdata = (void*)sv;
   return ctx;
 }
-void automCtxDestroy(AutomationCtx* ctx) {
+void automCtxDestroy(FAutoCtx* ctx) {
   startWithStateValueDestroy(ctx->stdata);
   taosMemoryFree(ctx->data);
   taosMemoryFree(ctx);
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 9600b1172e..6e5f4f7090 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -26,15 +26,14 @@
 const static uint64_t tfileMagicNumber = 0xdb4775248b80fb57ull;
 
 typedef struct TFileFstIter {
-  FstStreamBuilder* fb;
-  StreamWithState*  st;
-  AutomationCtx*    ctx;
-  TFileReader*      rdr;
+  FStmBuilder* fb;
+  FStmSt*      st;
+  FAutoCtx*    ctx;
+  TFileReader* rdr;
 } TFileFstIter;
 
 #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
 
-static int  tfileUidCompare(const void* a, const void* b);
 static int  tfileStrCompare(const void* a, const void* b);
 static int  tfileValueCompare(const void* a, const void* b, const void* param);
 static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds);
@@ -244,26 +243,21 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
 }
 
 static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
-  if (hasJson) {
-    p = indexPackJsonData(tem);
-    sz = strlen(p);
-  }
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*         ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
-  FstStreamBuilder*      sb = fstSearch(((TFileReader*)reader)->fst, ctx);
-  StreamWithState*       st = streamBuilderIntoStream(sb);
-  StreamWithStateResult* rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FStmSt*      st = stmBuilderIntoStm(sb);
+  FStmStRslt*  rt = NULL;
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     taosArrayPush(offsets, &(rt->out.out));
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmStDestroy(st);
+  stmBuilderDestroy(sb);
 
   int32_t ret = 0;
   for (int i = 0; i < taosArrayGetSize(offsets); i++) {
@@ -274,27 +268,14 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
       return TSDB_CODE_TDB_FILE_CORRUPTED;
     }
   }
-  if (hasJson) {
-    taosMemoryFree(p);
-  }
   return 0;
 }
 static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
-
   int      ret = 0;
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
-  if (hasJson) {
-    p = indexPackJsonData(tem);
-    sz = strlen(p);
-  }
   int64_t  st = taosGetTimestampUs();
   FstSlice key = fstSliceCreate(p, sz);
-  /*impl later*/
-  if (hasJson) {
-    taosMemoryFree(p);
-  }
   fstSliceDestroy(&key);
   return 0;
 }
@@ -305,7 +286,7 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
   if (hasJson) {
-    p = indexPackJsonData(tem);
+    p = idxPackJsonData(tem);
     sz = strlen(p);
   }
   int64_t  st = taosGetTimestampUs();
@@ -327,16 +308,16 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
-  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
   FstSlice h = fstSliceCreate((uint8_t*)p, skip);
-  fstStreamBuilderSetRange(sb, &h, type);
+  stmBuilderSetRange(sb, &h, type);
   fstSliceDestroy(&h);
 
-  StreamWithState*       st = streamBuilderIntoStream(sb);
-  StreamWithStateResult* rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  FStmSt*     st = stmBuilderIntoStm(sb);
+  FStmStRslt* rt = NULL;
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     FstSlice* s = &rt->data;
     char*     ch = (char*)fstSliceData(s, NULL);
     // if (0 != strncmp(ch, tem->colName, tem->nColName)) {
@@ -354,8 +335,8 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
     }
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmStDestroy(st);
+  stmBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
 static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
@@ -376,7 +357,7 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
   if (hasJson) {
-    p = indexPackJsonData(tem);
+    p = idxPackJsonData(tem);
     sz = strlen(p);
   }
   int64_t  st = taosGetTimestampUs();
@@ -401,7 +382,7 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
 }
 static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   int   ret = 0;
-  char* p = indexPackJsonData(tem);
+  char* p = idxPackJsonData(tem);
   int   sz = strlen(p);
 
   int64_t  st = taosGetTimestampUs();
@@ -456,18 +437,18 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
   int ret = 0;
   int skip = 0;
 
-  char* p = indexPackJsonDataPrefix(tem, &skip);
+  char* p = idxPackJsonDataPrefix(tem, &skip);
 
   _cache_range_compare cmpFn = indexGetCompare(ctype);
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
-  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
-  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  FAutoCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
+  FStmBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
-  StreamWithState*       st = streamBuilderIntoStream(sb);
-  StreamWithStateResult* rt = NULL;
-  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  FStmSt*     st = stmBuilderIntoStm(sb);
+  FStmStRslt* rt = NULL;
+  while ((rt = stmStNextWith(st, NULL)) != NULL) {
     FstSlice* s = &rt->data;
 
     int32_t sz = 0;
@@ -492,8 +473,8 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
     taosMemoryFree(tmp);
     swsResultDestroy(rt);
   }
-  streamWithStateDestroy(st);
-  fstStreamBuilderDestroy(sb);
+  stmStDestroy(st);
+  stmBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
 int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) {
@@ -704,8 +685,8 @@ static bool tfileIteratorNext(Iterate* iiter) {
   char*    colVal = NULL;
   uint64_t offset = 0;
 
-  TFileFstIter*          tIter = iiter->iter;
-  StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
+  TFileFstIter* tIter = iiter->iter;
+  FStmStRslt*   rt = stmStNextWith(tIter->st, NULL);
   if (rt == NULL) {
     return false;
   }
@@ -739,7 +720,7 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
 
   iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
   iter->fb = fstSearch(reader->fst, iter->ctx);
-  iter->st = streamBuilderIntoStream(iter->fb);
+  iter->st = stmBuilderIntoStm(iter->fb);
   iter->rdr = reader;
   return iter;
 }
@@ -770,8 +751,8 @@ void tfileIteratorDestroy(Iterate* iter) {
   iterateValueDestroy(iv, true);
 
   TFileFstIter* tIter = iter->iter;
-  streamWithStateDestroy(tIter->st);
-  fstStreamBuilderDestroy(tIter->fb);
+  stmStDestroy(tIter->st);
+  stmBuilderDestroy(tIter->fb);
   automCtxDestroy(tIter->ctx);
   taosMemoryFree(tIter);
 
diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc
index a2d7adf1c7..332b7370df 100644
--- a/source/libs/index/test/fstTest.cc
+++ b/source/libs/index/test/fstTest.cc
@@ -85,11 +85,11 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
-    StreamWithStateResult* rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
+    FStmBuilder* sb = fstSearch(_fst, ctx);
+    FStmSt*      st = stmBuilderIntoStm(sb);
+    FStmStRslt*  rt = NULL;
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -99,27 +99,27 @@ class FstReadMemory {
       result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    streamWithStateDestroy(st);
-    fstStreamBuilderDestroy(sb);
+    stmStDestroy(st);
+    stmBuilderDestroy(sb);
     return true;
   }
-  bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
+  bool SearchRange(FAutoCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
                    RangeType highType, std::vector<uint64_t>& result) {
-    FstStreamBuilder* sb = fstSearch(_fst, ctx);
+    FStmBuilder* sb = fstSearch(_fst, ctx);
 
     FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
     FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());
 
     // range [low, high);
-    fstStreamBuilderSetRange(sb, &l, lowType);
-    fstStreamBuilderSetRange(sb, &h, highType);
+    stmBuilderSetRange(sb, &l, lowType);
+    stmBuilderSetRange(sb, &h, highType);
 
     fstSliceDestroy(&l);
     fstSliceDestroy(&h);
 
-    StreamWithState*       st = streamBuilderIntoStream(sb);
-    StreamWithStateResult* rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    FStmSt*     st = stmBuilderIntoStm(sb);
+    FStmStRslt* rt = NULL;
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -129,11 +129,11 @@ class FstReadMemory {
       result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    streamWithStateDestroy(st);
-    fstStreamBuilderDestroy(sb);
+    stmStDestroy(st);
+    stmBuilderDestroy(sb);
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -253,7 +253,7 @@ void checkFstLongTerm() {
   // prefix search
   // std::vector<uint64_t> result;
 
-  // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  // FAutoCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
   // m->Search(ctx, result);
   // std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -286,7 +286,7 @@ void checkFstCheckIterator1() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  FAutoCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -321,7 +321,7 @@ void checkFstCheckIterator2() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  FAutoCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
@@ -361,7 +361,7 @@ void checkFstCheckIteratorPrefix() {
     // prefix search
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 1);
     automCtxDestroy(ctx);
@@ -370,7 +370,7 @@ void checkFstCheckIteratorPrefix() {
     // prefix search
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 2);
     automCtxDestroy(ctx);
@@ -378,7 +378,7 @@ void checkFstCheckIteratorPrefix() {
   {
     std::vector<uint64_t> result;
 
-    AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
+    FAutoCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
     m->Search(ctx, result);
     assert(result.size() == 1);
     automCtxDestroy(ctx);
@@ -412,7 +412,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "e", LT, result);
     assert(result.size() == 3);
@@ -421,7 +421,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "e", LT, result);
     assert(result.size() == 2);
@@ -430,7 +430,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "e", LE, result);
     assert(result.size() == 3);
@@ -439,7 +439,7 @@ void checkFstCheckIteratorRange1() {
   {
     // prefix search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "e", LE, result);
     assert(result.size() == 4);
@@ -473,7 +473,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "ed", LT, result);
     assert(result.size() == 4);
@@ -482,7 +482,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "bb", GE, "ed", LT, result);
     assert(result.size() == 3);
@@ -491,7 +491,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "ed", LE, result);
     assert(result.size() == 5);
@@ -501,7 +501,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "ed", LE, result);
     assert(result.size() == 4);
@@ -510,7 +510,7 @@ void checkFstCheckIteratorRange2() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GT, "ed", LT, result);
     assert(result.size() == 3);
@@ -544,7 +544,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "b", GE, "", (RangeType)10, result);
     assert(result.size() == 5);
@@ -553,7 +553,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result);
     assert(result.size() == 1);
@@ -563,7 +563,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result);
     assert(result.size() == 0);
@@ -572,7 +572,7 @@ void checkFstCheckIteratorRange3() {
   {
     // range  search
     std::vector<uint64_t> result;
-    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    FAutoCtx*             ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
     // [b, e)
     m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result);
     assert(result.size() == 0);
diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc
index 136c4dafec..b9388e62f7 100644
--- a/source/libs/index/test/fstUT.cc
+++ b/source/libs/index/test/fstUT.cc
@@ -106,11 +106,11 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
-    StreamWithStateResult* rt = NULL;
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
+    FStmBuilder* sb = fstSearch(_fst, ctx);
+    FStmSt*      st = stmBuilderIntoStm(sb);
+    FStmStRslt*  rt = NULL;
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       // result.push_back((uint64_t)(rt->out.out));
       FstSlice*   s = &rt->data;
       int32_t     sz = 0;
@@ -122,7 +122,7 @@ class FstReadMemory {
     std::cout << std::endl;
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -192,7 +192,7 @@ class TFst {
     }
     return fr->Get(k, v);
   }
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     // add more
     return fr->Search(ctx, result);
   }
@@ -231,7 +231,7 @@ TEST_F(FstEnv, writeNormal) {
   assert(val == 0);
 
   std::vector<uint64_t> rlt;
-  AutomationCtx*        ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  FAutoCtx*             ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
   assert(fst->Search(ctx, rlt) == true);
 }
 TEST_F(FstEnv, WriteMillonrRecord) {}
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index 327a4ae219..90dea3a377 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -106,17 +106,17 @@ class FstReadMemory {
     return ok;
   }
   // add later
-  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
-    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
-    StreamWithState*       st = streamBuilderIntoStream(sb);
-    StreamWithStateResult* rt = NULL;
+  bool Search(FAutoCtx* ctx, std::vector<uint64_t>& result) {
+    FStmBuilder* sb = fstSearch(_fst, ctx);
+    FStmSt*      st = stmBuilderIntoStm(sb);
+    FStmStRslt*  rt = NULL;
 
-    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    while ((rt = stmStNextWith(st, NULL)) != NULL) {
       result.push_back((uint64_t)(rt->out.out));
     }
     return true;
   }
-  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+  bool SearchWithTimeCostUs(FAutoCtx* ctx, std::vector<uint64_t>& result) {
     int64_t s = taosGetTimestampUs();
     bool    ok = this->Search(ctx, result);
     int64_t e = taosGetTimestampUs();
@@ -220,7 +220,7 @@ void checkFstPrefixSearch() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
+  FAutoCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
   m->Search(ctx, result);
   assert(result.size() == count);
   for (int i = 0; i < result.size(); i++) {

From 9c127e3e69cc1329ea07d038acdb824aa7654378 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 8 Jun 2022 22:00:57 +0800
Subject: [PATCH 46/63] fix(query): decode

---
 source/common/src/tdatablock.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 86d4f205c6..f825e41495 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -684,7 +684,8 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
  */
 size_t blockDataGetSerialMetaSize(uint32_t numOfCols) {
   // | total rows/total length | block group id | column schema | each column length |
-  return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t);
+  return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) +
+         numOfCols * sizeof(int32_t);
 }
 
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@@ -1892,12 +1893,12 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
   uint64_t* groupId = (uint64_t*)data;
   data += sizeof(uint64_t);
 
-  for(int32_t i = 0; i < numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    *((int16_t*) data) = pColInfoData->info.type;
+    *((int16_t*)data) = pColInfoData->info.type;
     data += sizeof(int16_t);
 
-    *((int32_t*) data) = pColInfoData->info.bytes;
+    *((int32_t*)data) = pColInfoData->info.bytes;
     data += sizeof(int32_t);
   }
 
@@ -1951,7 +1952,14 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
   pBlock->info.groupId = *(uint64_t*)pStart;
   pStart += sizeof(uint64_t);
 
-  for(int32_t i = 0; i < numOfCols; ++i) {
+  if (pBlock->pDataBlock == NULL) {
+    pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
+    taosArraySetSize(pBlock->pDataBlock, numOfCols);
+  }
+
+  ASSERT(pBlock->pDataBlock->size == numOfCols);
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
     pColInfoData->info.type = *(int16_t*)pStart;
     pStart += sizeof(int16_t);
@@ -1982,11 +1990,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
         pColInfoData->pData = taosMemoryMalloc(colLen[i]);
       }
     } else {
+      if (pColInfoData->nullbitmap == NULL) {
+        pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows));
+      }
       memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
       pStart += BitmapLen(numOfRows);
     }
 
     if (colLen[i] > 0) {
+      if (pColInfoData->pData == NULL) {
+        pColInfoData->pData = taosMemoryCalloc(1, colLen[i]);
+      }
       memcpy(pColInfoData->pData, pStart, colLen[i]);
     }
 

From 51c9a6fac77b0f42404f950098bc47784859676b Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 8 Jun 2022 22:42:10 +0800
Subject: [PATCH 47/63] fix(query): fix to_iso8601 function append format in
 "-hh:mm", "-hhmm" "-hh", "z" when digits exceeds 10 digits

---
 source/libs/scalar/src/sclfunc.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 2b15f5031e..2369779105 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -901,12 +901,22 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 
     if (hasFraction) {
       int32_t fracLen = (int32_t)strlen(fraction) + 1;
-      char *tzInfo = strchr(buf, '+');
-      if (tzInfo) {
+
+      char *tzInfo;
+      if (buf[len - 1] == 'z' || buf[len - 1] == 'Z') {
+        tzInfo = &buf[len - 1];
         memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
       } else {
-        tzInfo = strchr(buf, '-');
-        memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+        tzInfo = strchr(buf, '+');
+        if (tzInfo) {
+          memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+        } else {
+          //search '-' backwards
+          tzInfo = strrchr(buf, '-');
+          if (tzInfo) {
+            memmove(tzInfo + fracLen, tzInfo, strlen(tzInfo));
+          }
+        }
       }
 
       char tmp[32] = {0};

From 1783c971e9a7ccdb953aa309b54accc3403048fa Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Thu, 9 Jun 2022 08:47:19 +0800
Subject: [PATCH 48/63] fix: add  group id to multiway merge operator

---
 source/libs/executor/src/sortoperator.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 95f9514b07..1e57de8438 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -291,6 +291,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       pTupleHandle = tsortNextTuple(pHandle);
     } else {
       pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
       pInfo->prefetchedTuple = NULL;
     }
 
@@ -307,7 +308,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       appendOneRowToDataBlock(p, pTupleHandle);
     } else {
       pInfo->prefetchedTuple = pTupleHandle;
-      pInfo->groupId = tupleGroupId;
       break;
     }
 
@@ -330,6 +330,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);

From ef18a45d995f4c4c6c868c84ac4961eb2d933f50 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Thu, 9 Jun 2022 09:09:48 +0800
Subject: [PATCH 49/63] in data type convert

---
 source/libs/command/inc/commandInt.h          |  4 +-
 source/libs/command/src/explain.c             |  4 +-
 source/libs/parser/src/parTranslater.c        |  8 +++-
 source/libs/scalar/src/sclvector.c            | 10 ++++-
 .../2-query/query_cols_tags_and_or.py         | 40 +++++--------------
 5 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h
index 100e35bc3c..f1f199c6c6 100644
--- a/source/libs/command/inc/commandInt.h
+++ b/source/libs/command/inc/commandInt.h
@@ -112,9 +112,9 @@ typedef struct SExplainCtx {
 #define EXPLAIN_ROW_NEW(level, ...)                                                                               \
   do {                                                                                                            \
     if (isVerboseLine) {                                                                                          \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 2 + 3, "");       \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, "");       \
     } else {                                                                                                      \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> ");  \
     }                                                                                                             \
     tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__);         \
   } while (0)
diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c
index 98ae5e2fb1..6f93e3e21d 100644
--- a/source/libs/command/src/explain.c
+++ b/source/libs/command/src/explain.c
@@ -652,7 +652,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
       if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
         // sort key
-        EXPLAIN_ROW_NEW(level, "Sort Key: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
         if (pResNode->pExecInfo) {
           for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
             SOrderByExprNode *ptn = nodesListGetNode(pSortNode->pSortKeys, i);
@@ -664,7 +664,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
 
         // sort method
-        EXPLAIN_ROW_NEW(level, "Sort Method: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Method: ");
 
         int32_t           nodeNum = taosArrayGetSize(pResNode->pExecInfo);
         SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 225a626bc7..0d1cc13b60 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -836,9 +836,13 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo
       SDataType dt = ((SExprNode*)pNode)->resType;
       if (first) {
         targetDt = dt;
-        first = false;
-      } else if (!dataTypeEqual(&dt, &targetDt)) {
+        if (targetDt.type != TSDB_DATA_TYPE_NULL) {
+          first = false;
+        }
+      } else if (dt.type != targetDt.type && dt.type != TSDB_DATA_TYPE_NULL) {
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName);
+      } else if (dt.bytes > targetDt.bytes) {
+        targetDt.bytes = dt.bytes;
       }
     }
     pRight->dataType = targetDt;
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 1252e27554..1412dfe0d4 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -879,11 +879,17 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p
   }
 
   if (type != GET_PARAM_TYPE(param1)) {
-    return vectorConvertScalarParam(param1, paramOut1, type);
+    code = vectorConvertScalarParam(param1, paramOut1, type);
+    if (code) {
+      return code;
+    }
   }
   
   if (type != GET_PARAM_TYPE(param2)) {
-    return vectorConvertScalarParam(param2, paramOut2, type);
+    code = vectorConvertScalarParam(param2, paramOut2, type);
+    if (code) {
+      return code;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py
index 5dc5a2f123..f8a44f735f 100644
--- a/tests/system-test/2-query/query_cols_tags_and_or.py
+++ b/tests/system-test/2-query/query_cols_tags_and_or.py
@@ -1066,9 +1066,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c5 in (1, 6.6)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # not in
         query_sql = f'select {select_elm} from {tb_name} where c5 not in (2, 3)'
         tdSql.query(query_sql)
@@ -1076,14 +1074,10 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # and
         query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 and c5 < 7 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 6.6 and c5 is not null and c5 between 2 and 6.6 and c5 not between 1 and 2 and c5 in (2,6.6) and c5 not in (1,2)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # or
         query_sql = f'select {select_elm} from {tb_name} where c5 > 6 or c5 >= 6.6 or c5 < 1 or c5 <= 0 or c5 != 1.1 or c5 <> 1.1 or c5 = 5 or c5 is null or c5 between 4 and 5 or c5 not between 1 and 3 or c5 in (4,5) or c5 not in (1.1,3)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # and or
         query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 or c5 < 5 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 4 or c5 is not null and c5 between 2 and 4 and c5 not between 1 and 2 and c5 in (2,4) and c5 not in (1,2)'
         tdSql.query(query_sql)
@@ -1151,9 +1145,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c6 in (1, 7.7)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # not in
         query_sql = f'select {select_elm} from {tb_name} where c6 not in (2, 3)'
         tdSql.query(query_sql)
@@ -1161,14 +1153,10 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # and
         query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 and c6 < 8 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 7.7 and c6 is not null and c6 between 2 and 7.7 and c6 not between 1 and 2 and c6 in (2,7.7) and c6 not in (1,2)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # or
         query_sql = f'select {select_elm} from {tb_name} where c6 > 7 or c6 >= 7.7 or c6 < 1 or c6 <= 0 or c6 != 1.1 or c6 <> 1.1 or c6 = 5 or c6 is null or c6 between 4 and 5 or c6 not between 1 and 3 or c6 in (4,5) or c6 not in (1.1,3)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 7) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # and or
         query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 or c6 < 5 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 4 or c6 is not null and c6 between 2 and 4 and c6 not between 1 and 2 and c6 in (2,4) and c6 not in (1,2)'
         tdSql.query(query_sql)
@@ -1410,9 +1398,7 @@ class TDTestCase:
         tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
         # in
         query_sql = f'select {select_elm} from {tb_name} where c9 in ("binar", false)'
-        tdSql.query(query_sql)
-        tdSql.checkRows(2)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # # not in
         query_sql = f'select {select_elm} from {tb_name} where c9 not in (true)'
         tdSql.query(query_sql)
@@ -1421,19 +1407,13 @@ class TDTestCase:
 
         # # and
         query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" and c9 <> "binary" and c9 is not null and c9 in ("binary", true) and c9 not in ("binary")'
-        tdSql.query(query_sql)
-        tdSql.checkRows(9)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 9) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # # or
         query_sql = f'select {select_elm} from {tb_name} where c9 = true or c9 != "false" or c9 <> "binary" or c9 = "true" or c9 is not null or c9 in ("binary", true) or c9 not in ("binary")'
-        tdSql.query(query_sql)
-        tdSql.checkRows(11)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
+        tdSql.error(query_sql)
         # # and or
         query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" or c9 <> "binary" or c9 = "true" and c9 is not null or c9 in ("binary", true) or c9 not in ("binary")'
-        tdSql.query(query_sql)
-        tdSql.checkRows(11)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
+        tdSql.error(query_sql)
         query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2)'
         tdSql.query(query_sql)
         tdSql.checkRows(9)

From 9afcca0355ade10397e9f72694faca70a3a60d70 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Thu, 9 Jun 2022 09:36:14 +0800
Subject: [PATCH 50/63] fix: add group id to create initial source

---
 source/common/src/tdatablock.c   |  1 +
 source/libs/executor/src/tsort.c | 61 ++++++++++++++++++++------------
 2 files changed, 39 insertions(+), 23 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 704a319512..65b0e25275 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1217,6 +1217,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
   pBlock->info.numOfCols = numOfCols;
   pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
   pBlock->info.rowSize = pDataBlock->info.rowSize;
+  pBlock->info.groupId = pDataBlock->info.groupId;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 8fb1d1f302..9448ef15b3 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -526,16 +526,24 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
   if (pHandle->type == SORT_SINGLESOURCE_SORT) {
     SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0);
     taosArrayClear(pHandle->pOrderedSource);
-    
+
+    bool hasGroupId = false;
+    SSDataBlock* prefetchedDataBlock = NULL;
+
     while (1) {
-      SSDataBlock* pBlock = pHandle->fetchfp(source->param);
+      SSDataBlock* pBlock = NULL;
+      if (prefetchedDataBlock == NULL) {
+        pBlock = pHandle->fetchfp(source->param);
+      } else {
+        pBlock = prefetchedDataBlock;
+        prefetchedDataBlock = NULL;
+      }
+
       if (pBlock == NULL) {
         break;
       }
 
-      if (pHandle->pDataBlock == NULL) {
-        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
-
+      if (!hasGroupId) {
         // calculate the buffer pages according to the total available buffers.
         int32_t rowSize = blockDataGetRowSize(pBlock);
         if (rowSize * 4 > 4096) {
@@ -547,29 +555,36 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
         // todo!!
         pHandle->numOfPages = 1024;
         sortBufSize = pHandle->numOfPages * pHandle->pageSize;
+
+        hasGroupId = true;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
       }
 
-      // perform the scalar function calculation before apply the sort
-      if (pHandle->beforeFp != NULL) {
-        pHandle->beforeFp(pBlock, pHandle->param);
-      }
+      if (pHandle->pDataBlock->info.groupId == pBlock->info.groupId) {
+        // perform the scalar function calculation before apply the sort
+        if (pHandle->beforeFp != NULL) {
+          pHandle->beforeFp(pBlock, pHandle->param);
+        }
+        // todo relocate the columns
+        int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
+        if (code != 0) {
+          return code;
+        }
 
-      // todo relocate the columns
-      int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
-      if (code != 0) {
-        return code;
-      }
+        size_t size = blockDataGetSize(pHandle->pDataBlock);
+        if (size > sortBufSize) {
+          // Perform the in-memory sort and then flush data in the buffer into disk.
+          int64_t p = taosGetTimestampUs();
+          blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
 
-      size_t size = blockDataGetSize(pHandle->pDataBlock);
-      if (size > sortBufSize) {
-        // Perform the in-memory sort and then flush data in the buffer into disk.
-        int64_t p = taosGetTimestampUs();
-        blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+          int64_t el = taosGetTimestampUs() - p;
+          pHandle->sortElapsed += el;
 
-        int64_t el = taosGetTimestampUs() - p;
-        pHandle->sortElapsed += el;
-
-        doAddToBuf(pHandle->pDataBlock, pHandle);
+          doAddToBuf(pHandle->pDataBlock, pHandle);
+        }
+      } else {
+        prefetchedDataBlock = pBlock;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
       }
     }
 

From 30b54fcfb5ea0593b276a8fba2e3d0c38cbdaaf6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 9 Jun 2022 09:36:50 +0800
Subject: [PATCH 51/63] refactor: do some internal refactor.

---
 source/common/src/tdatablock.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index f825e41495..fd2b93309e 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1944,6 +1944,9 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
   blockDataEnsureCapacity(pBlock, numOfRows);
+  pBlock->info.rows = numOfRows;
+  pBlock->info.numOfCols = numOfCols;
+
   const char* pStart = pData;
 
   int32_t dataLen = *(int32_t*)pStart;
@@ -1966,6 +1969,10 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
 
     pColInfoData->info.bytes = *(int32_t*)pStart;
     pStart += sizeof(int32_t);
+
+    if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+      pBlock->info.hasVarCol = true;
+    }
   }
 
   blockDataEnsureCapacity(pBlock, numOfRows);

From e17b2e386b09e8e45fca207d59945a384f371cef Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 9 Jun 2022 09:42:58 +0800
Subject: [PATCH 52/63] enh: rm invalid file

---
 source/libs/index/src/.indexFilter.c.swo | Bin 16384 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 source/libs/index/src/.indexFilter.c.swo

diff --git a/source/libs/index/src/.indexFilter.c.swo b/source/libs/index/src/.indexFilter.c.swo
deleted file mode 100644
index c61d5e7793e4ca891ef2b8cc1e5b63c72f1431a8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeHNU5q4E6)r#pTo(|GBpSkHfK7MXGrha)!m`6I)b@1G6x&nX+g;5%L1?SHZci6m
zUDedj{IIMB4AEdBFFcxv3DFl;i80X^<8RccG2sFKP%#=!e31u?8g<d{+*>vMGd;s1
zCPuB^eA`p^=iGb0d(OG{oa*-e`?O_tWbS~1&ozqjrTd?0|54wkyzrEw_?{P1@#}r=
zxV^&<n|Jc=?ROmBt?h?MU}4?!&lHRW?ylM{F9crXcX+|E+d;v0Exu6*{EqyqY&#+M
z=Q_KIOzTnxQU)%Dfhg?G9@(qhcJRP{YJT&~P3*dBn-|kFZId#PGLSNmGLSNmGLSNm
zGLSNmGVnjbKsdNu`7qXUX}lgc{=Q`5`ycVgiOao-`q$$6eq8?2ME!-hp2y`&@c_v<
ze~;@o;`;Lw^?!@&*W>zI;|VAIFN@2|@%JYu>f_?{D`g;MAY~wBAY~wBAY~wBAY~wB
zAY~wBAZ6hHk^$3Fl%rVrRTMAd{eKewU$|OPz5#p{2!R~%yQ>uCH^9@tNnjsvGw|cR
zit-fjao|Z{9rz${FK`ENBd`Z}^&N`xE8u0|`+yF-xJOaiKovL)FyKaDFYuSQE6SgN
zmw|5r&jB9;9t3hg7I+tM1#mg=%9V=pQ{Xdz1{?xz0ImbB1-^ZSqI?>79B2ZEfjz)0
zmn+J*fX@Pt180DD1FyYJQC<MP44em^03HK4&;i=OHNe%t@860w0nY+9a0|eI8-Z8f
zq9{KCUIe}Xd;&NF*gy|B2z(6*j4qG`-V0n0>;bL>Uc}yf9XJm>0Xzn*00)3s;GMwL
z0PW#k;7xqaDUA6%jC_|FjfE4fVs)X^YBY<*Qll}CI;x-2;#7U5rRn#n6>S0EwWdLB
z?Jmo7+}YzD&oMR!oZWPc6|0rzvfeUQYNeK;E@s(Tw0q++*|B3xZ&oU@f1$uk%VK@c
z=d9D?oij=hnxWlcwj17duobev?%roQwq=IA=s8i}%{1h<?pd7WIxfi>S^2?jD?LCZ
z^r_b5<h4diuNtkUep0WV)@eTBCVTYJt@>q6Z#HDHoa993vRY}Bws(ug?p!-Lpdx7=
zofdf6Q81A<wbH0Jcb?8z<rH!kc~~8Azh?SoUyu5k5<Qur=ZkF*H;DnkZ)keKk{R@h
z4maIskQq>q+2aHGOP?%}Oih+K_HwiAbDp&5c&>#ljQCa?)%VTK1s;UHx0$r^c@Q}v
zn#8OK@w8m2D^joLlLf4KwneHz<A|we8d_QW!E&Ofn0BFXPJx24hGDMCX(o%)+^&a~
z;_Dd8?9?DDWX+S1SS`iCmYn)jsTt8?I0<wG0a$9(2|>^*>xy4SF)yZ>InZpMMhIbN
z7Vlyt(H09jZ8~SLsD83)EV$K;+>SI0J#w5fo|xna%q0>M+sWjHJ+dPVIlLR@*^n{c
zUJZMr1zr>mBC#1Gmi4B6$~nfJfD5<5Sk4MU%=}cd1ou;`m&)4vh2_dN<bLp5GTQ&d
zJnn5&b+|O$P&K`g*yWg%dTFurIzlSi$r73RgzRP2C@xJY*G0wD!HiOUdG`sIPBqoo
z%S|k|0*BlZ)w@fmR8N=cvf<P@2sy@*s$Yy4YByENV!fndH5W6Xv8KBUOJ+4?BNHLX
zgtj|;_#{&7vhVd94?7bs30s$EC5?V=cq-!lrtnjw2Q(k$9I16N+#p6^Qw`19C_7;T
zXG#>nvZ%Nr`Y!X(RQ*+E`m0pf_B@AeeO#}!!8`Lp1gw!CWe-JAR~wcJYbArW?9ldH
z7Q+AJndw^M6zjq8Sm-g1&JkQF8#ty*Ba+{&wjW|S8evf90YpQ14$98^9G&c7#2gC>
z4u<&nqjn4fT6})XyoU|~3*B0@@P0e2Pl1S@=?hunKOF12+2{AiHeG4y>T;=7De0Kv
zLH0g&=mC;6-p6VWyWn<_^bjpE-P;erEy+(1*_I@a&UWeGXc11!wvtA8(kvPzLf<1v
zDK)}bL3=8fGQNGZi{gY3GXtAjXNzbp&J~uEM~P@RWIjHprm-lP1CQjYv0<yTUC-~E
zA@f}2W;kU0(v>!+RjSup_tr~Brjuos$IgmffeVA4h}#m~M{rHr<|d(+42t8s9yO+&
zWUC&uq!piO>>U2B_iTr=%>L}ykfWm|8qp#V!(rv)>y9g$&10bWg1GfOePCMazkTq@
zp+{G}jr(pF@rmtroCvF(S#z=Ix!%l3QHU#{*>=3nnep<VV>+fkUiL6K3K9{Fmrp4Y
zzhl<iaJ-=?EqeX_lwvAUSPoX3W2WP|t8Cp4do1k1s0Y5cifhsw)4I&{aOeh{v%7j>
zI5=7;tgo-nt-8^i=da>7)E({yJSZHWBRMNx#Dbm|ITq`gYaF}Q;r1Gbn}G%mHksFj
ze9jj2CR4jz?t5&JyKqhpt3_?dVihS5vPfK6GjyCetX!=#mDSX`p%t4Iwa#kIdac?h
zi7{1ug`HOGbyYW3=EWLt4PbnY!_8qVyFYMj%wXNb?ajrF8FN@J)r&Y*4E2Oo(To-5
z`81j_vU2ilmJN?(x1$ifZ7HmW>kp*D$m|e9ISy-c76rT;IeCVb?6hVqRhvdUf4EZN
zMObD=U944RvhcYemzEmzAho?IrbxFz*6_L^=Ezyui(Kdk+_YF82EsG<MN8OYKe%WX
z^K6a#0r~serme3_!v^uNN7*L!u4B5ioIEyexO#C=6tI@QXW8A&cx^@OCoW*U5CK<~
z9o&E5PG+dp23vr7=b7eq=Ga{i^>DSmhpyM8^m84re_Sc#6pH`9hxqk1#Jd##m*e2^
zhluyT3w#H79tZ#rco6sia6fQ8@G9c}p95b6o(Db;d<3`wxD5CK;{IoWXMj%uHqZmA
zfDZf>Ie^~+9|ax&R)BkeyMY|=3i1Gd0Db|y1pEwm9N@qrPy|i@w*Y@a4&ZaZc_0Fg
z0ha=o06#(e{~Yi%um*%c0Nf3{2e^Pdz^{Q{0$%~10v3TXuoqAO;^+eLlzvhMQU+26
zQU+26{;L@nNzEjItqfA=;*P`*MbS>Ugt#G-X~=A8HjaYmDbMyHW|j#z5h)Kl%M_L7
z+98GD)OaTevSps_CqXh?_7EHOJ=|Fl;)r`A#MsE-J|eDSjPh4>CzGukGA$?5adBoW
zNoP$YmmB4ZA@a7Gz97>)6m^X(XqYTK3*nQIKO&|{3FA#{YeO<7lLZN7BLm0;gOFVo
z(b4u$k4P9xOYRdH#4U1ggA+ljYFVn|WMrh8XQI*ERti*RQZJ%mJeMR|!^lO`=|pr_
z-f73=e8hyd5`dx%1f88jHU!_fzD$nfSv)6{$&}!^(KtIULnYlz#dJw$GDi^R*pG0p
zo1xgA@&f-+HQP<cK;RB<Fh<^+?4oB$P;Plrk&zP4h;%QDEp9}8D8Wiut0eO@WVs=@
zOA;5`)QkdtX{W?9<&x@1vaj(>C)}8$Y*L=ZS?BD&fi&$}RnsY_cVfjTHN<9*X(cpG
z3d~XR4VkfH2vmaE<MN62ShA9&J;<XBm7QJ~skIs<M5=WmobpvzS;_Z3KQn_w#~Qt|
zA(<5L^)TZ+!e5IY&D=Ub>_&O&%&k^-X8Xjc6=i=iVj<G25dS1<>t%qn?<C(Zs*uCq
zdI6A4Xo5Yz0gfh&iO3x?PxA^WA79!S_%$yGt3F~$IZIM^VR%%}eK=P1mHh6YT*M%v
z+g7#QS}rYD>nlY0IEjML+1Mp^O=O2G;Zb7^AVZrG+!H&=tD!_8NQICbi`64GP*NOW
zpNFsm5Ce7=c-O>R7pkFzEBR%)&kL%d%9mv)v_+^AHmz$Sd7o!XYGVm*xK*hZPm1xu
zdl6|!42E&e3d-=ue)tZVMWxY^LWSqRly>1`>MKa01>+l^m|UC&8>;1&#gZ0}zd;;1
z3KP4Ob~1hh#MZEr0VdN=)azfc!im!Xsw?#*5xw2>4;F%O_Ba}1pD3LGE0quE_)ol9
Bh%x{G


From eb6dc7feaeea8b2fc43c40ef8b2547d566265f94 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 9 Jun 2022 09:46:01 +0800
Subject: [PATCH 53/63] refactor: adjust vnode queue

---
 source/dnode/mgmt/mgmt_vnode/inc/vmInt.h    |  1 +
 source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 26 +++++++++++----------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
index 0f7daf0e1d..7e5379b0f8 100644
--- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
+++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
@@ -34,6 +34,7 @@ typedef struct SVnodeMgmt {
   SQWorkerPool   fetchPool;
   SWWorkerPool   syncPool;
   SWWorkerPool   writePool;
+  SWWorkerPool   applyPool;
   SWWorkerPool   mergePool;
   SSingleWorker  mgmtWorker;
   SSingleWorker  monitorWorker;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index b6913f93f2..88831384d4 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -16,10 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
-#include "sync.h"
-#include "syncTools.h"
-
 static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
+  if (pMsg->info.handle == NULL) return;
   SRpcMsg rsp = {
       .code = code,
       .pCont = pMsg->info.rsp,
@@ -55,7 +53,7 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   if (IsReq(pMsg)) {
     if (code != 0) {
       if (terrno != 0) code = terrno;
-      dError("msg:%p failed to process since %s", pMsg, terrstr());
+      dError("msg:%p, failed to process since %s", pMsg, terrstr());
     }
     vmSendRsp(pMsg, code);
   }
@@ -107,11 +105,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
     int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
     if (code != 0) {
+      if (terrno != 0) code = terrno;
       dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
-      if (pMsg->info.handle != NULL) {
-        if (terrno != 0) code = terrno;
-        vmSendRsp(pMsg, code);
-      }
+      vmSendRsp(pMsg, code);
     }
 
     dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
@@ -130,8 +126,8 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
     if (code != 0) {
-      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       if (terrno != 0) code = terrno;
+      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       vmSendRsp(pMsg, code);
     }
 
@@ -150,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dError("vgId:%d, failed to put msg:%p into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
+    dError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
            TMSG_INFO(pMsg->msgType));
     return terrno != 0 ? terrno : -1;
   }
@@ -260,7 +256,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
 int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg);
   pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
-  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeApplyMsg);
+  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyMsg);
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
   pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
   pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue);
@@ -277,7 +273,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
 void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
-  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
+  tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
   tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
   tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
@@ -309,6 +305,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
   pWPool->max = tsNumOfVnodeWriteThreads;
   if (tWWorkerInit(pWPool) != 0) return -1;
 
+  SWWorkerPool *pAPool = &pMgmt->applyPool;
+  pAPool->name = "vnode-apply";
+  pAPool->max = tsNumOfVnodeWriteThreads;
+  if (tWWorkerInit(pAPool) != 0) return -1;
+
   SWWorkerPool *pSPool = &pMgmt->syncPool;
   pSPool->name = "vnode-sync";
   pSPool->max = tsNumOfVnodeSyncThreads;
@@ -345,6 +346,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
   tWWorkerCleanup(&pMgmt->writePool);
+  tWWorkerCleanup(&pMgmt->applyPool);
   tWWorkerCleanup(&pMgmt->syncPool);
   tQWorkerCleanup(&pMgmt->queryPool);
   tQWorkerCleanup(&pMgmt->fetchPool);

From 414b3242a7d683b4bc0719d6261a847238997bc2 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Thu, 9 Jun 2022 09:51:00 +0800
Subject: [PATCH 54/63] fix: add group id to sort operator

---
 source/libs/executor/inc/executorimpl.h |  6 ++-
 source/libs/executor/src/sortoperator.c | 68 +++++++++++++++----------
 2 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 4f02c559b1..ce1ae318dc 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -700,6 +700,10 @@ typedef struct SSortOperatorInfo {
 
   int64_t      startTs;       // sort start time
   uint64_t     sortElapsed;   // sort elapsed time, time to flush to disk not included.
+
+  STupleHandle *prefetchedTuple;
+  bool  hasGroupId;
+  uint64_t groupId;
 } SSortOperatorInfo;
 
 typedef struct STagFilterOperatorInfo {
@@ -759,7 +763,7 @@ void    setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo);
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo);
 SSDataBlock* loadNextDataBlock(void* param);
 
 void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 1e57de8438..dcaa95e28a 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -42,6 +42,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
 
   pInfo->pSortInfo = pSortInfo;
   pInfo->pColMatchInfo = pColMatchColInfo;
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->name = "SortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
   pOperator->blocking = true;
@@ -81,8 +83,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   pBlock->info.rows += 1;
 }
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                SArray* pColMatchInfo) {
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
+                                SSortOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -93,14 +95,33 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-    STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
     if (pTupleHandle == NULL) {
       break;
     }
 
-    appendOneRowToDataBlock(p, pTupleHandle);
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
     if (p->info.rows >= capacity) {
-      return pDataBlock;
+      break;
     }
   }
 
@@ -117,6 +138,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);
@@ -188,8 +210,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
-      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+  SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
+                                           pInfo->pColMatchInfo, pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -230,11 +252,11 @@ typedef struct SMultiwaySortMergeOperatorInfo {
   SArray*      pColMatchInfo;  // for index map from table scan output
 
   SSDataBlock* pInputBlock;
-  int64_t startTs;  // sort start time
+  int64_t      startTs;  // sort start time
 
-  bool  hasGroupId;
-  uint64_t groupId;
-  STupleHandle *prefetchedTuple;
+  bool          hasGroupId;
+  uint64_t      groupId;
+  STupleHandle* prefetchedTuple;
 } SMultiwaySortMergeOperatorInfo;
 
 int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
@@ -274,7 +296,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
 }
 
 SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
+                                        SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -285,7 +307,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-
     STupleHandle* pTupleHandle = NULL;
     if (pInfo->prefetchedTuple == NULL) {
       pTupleHandle = tsortNextTuple(pHandle);
@@ -314,7 +335,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
     if (p->info.rows >= capacity) {
       break;
     }
-
   }
 
   if (p->info.rows > 0) {
@@ -337,7 +357,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
-
 SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -351,12 +370,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
-      getMultiwaySortedBlockData(pInfo->pSortHandle,
-                         pInfo->binfo.pRes,
-                         pOperator->resultInfo.capacity,
-                         pInfo->pColMatchInfo,
-                                 pInfo);
+  SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
+                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -367,7 +382,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
 }
 
 void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
-  SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param;
+  SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
   pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
 
@@ -387,9 +402,9 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
-                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
-                                                   SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
+                                                   SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
+                                                   SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
   SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
   SOperatorInfo*                  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   int32_t                         rowSize = pResBlock->info.rowSize;
@@ -413,7 +428,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
 
   pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2;
   pInfo->sortBufSize = pInfo->bufPageSize * 16;
-
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->pTaskInfo = pTaskInfo;
   pOperator->fpSet =
       createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,

From 83f5e5eedcbfa8705966fd61abeb83c45bc4c5ff Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 9 Jun 2022 09:56:45 +0800
Subject: [PATCH 55/63] fix(stream): fix no executor stream task

---
 source/client/src/tmq.c             | 181 +---------------------------
 source/libs/stream/src/stream.c     |  13 +-
 source/libs/stream/src/streamExec.c |   2 +-
 3 files changed, 16 insertions(+), 180 deletions(-)

diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index cccb1fa6d0..b5011a19a2 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -494,7 +494,6 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
       tmqAskEp(tmq, true);
       taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
-      /*tmq_commit(tmq, NULL, true);*/
       tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam);
       taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
@@ -667,94 +666,6 @@ FAIL:
 
 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
   return tmqCommitInner(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam);
-#if 0
-  // TODO: add read write lock
-  SRequestObj*   pRequest = NULL;
-  tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
-  // build msg
-  // send to mnode
-  SMqCMCommitOffsetReq req;
-  SArray*              pOffsets = NULL;
-
-  if (offsets == NULL) {
-    pOffsets = taosArrayInit(0, sizeof(SMqOffset));
-    for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-      for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-        SMqOffset    offset;
-        strcpy(offset.topicName, pTopic->topicName);
-        strcpy(offset.cgroup, tmq->groupId);
-        offset.vgId = pVg->vgId;
-        offset.offset = pVg->currentOffset;
-        taosArrayPush(pOffsets, &offset);
-      }
-    }
-    req.num = pOffsets->size;
-    req.offsets = pOffsets->pData;
-  } else {
-    req.num = taosArrayGetSize(&offsets->container);
-    req.offsets = (SMqOffset*)offsets->container.pData;
-  }
-
-  SEncoder encoder;
-
-  tEncoderInit(&encoder, NULL, 0);
-  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
-  int32_t tlen = encoder.pos;
-  void*   buf = taosMemoryMalloc(tlen);
-  if (buf == NULL) {
-    tEncoderClear(&encoder);
-    return -1;
-  }
-  tEncoderClear(&encoder);
-
-  tEncoderInit(&encoder, buf, tlen);
-  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
-  tEncoderClear(&encoder);
-
-  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET);
-  if (pRequest == NULL) {
-    tscError("failed to malloc request");
-  }
-
-  SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
-  if (pParam == NULL) {
-    return -1;
-  }
-  pParam->tmq = tmq;
-  tsem_init(&pParam->rspSem, 0, 0);
-  pParam->async = async;
-  pParam->offsets = pOffsets;
-
-  pRequest->body.requestMsg = (SDataBuf){
-      .pData = buf,
-      .len = tlen,
-      .handle = NULL,
-  };
-
-  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  sendInfo->requestObjRefId = 0;
-  sendInfo->param = pParam;
-  sendInfo->fp = tmqCommitCb;
-  SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
-
-  int64_t transporterId = 0;
-  asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
-
-  if (!async) {
-    tsem_wait(&pParam->rspSem);
-    resp = pParam->rspErr;
-    tsem_destroy(&pParam->rspSem);
-    taosMemoryFree(pParam);
-
-    if (pOffsets) {
-      taosArrayDestroy(pOffsets);
-    }
-  }
-
-  return resp;
-#endif
 }
 
 tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
@@ -859,93 +770,6 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para
   conf->commitCbUserParam = param;
 }
 
-#if 0
-TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
-  STscObj*     pTscObj = (STscObj*)taos;
-  SRequestObj* pRequest = NULL;
-  SQuery*      pQueryNode = NULL;
-  char*        astStr = NULL;
-  int32_t      sqlLen;
-
-  terrno = TSDB_CODE_SUCCESS;
-  if (taos == NULL || streamName == NULL || sql == NULL) {
-    tscError("invalid parameters for creating stream, connObj:%p, stream name:%s, sql:%s", taos, streamName, sql);
-    terrno = TSDB_CODE_TSC_INVALID_INPUT;
-    goto _return;
-  }
-  sqlLen = strlen(sql);
-
-  if (strlen(tbName) >= TSDB_TABLE_NAME_LEN) {
-    tscError("output tb name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
-    terrno = TSDB_CODE_TSC_INVALID_INPUT;
-    goto _return;
-  }
-
-  if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) {
-    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
-    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
-    goto _return;
-  }
-
-  tscDebug("start to create stream: %s", streamName);
-
-  int32_t code = 0;
-  CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return);
-  CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
-
-  /*printf("%s\n", pStr);*/
-
-  SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
-  strcpy(name.dbname, pRequest->pDb);
-  strcpy(name.tname, streamName);
-
-  SCMCreateStreamReq req = {
-      .igExists = 1,
-      .ast = astStr,
-      .sql = (char*)sql,
-  };
-  tNameExtractFullName(&name, req.name);
-  strcpy(req.targetStbFullName, tbName);
-
-  int   tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
-  void* buf = taosMemoryMalloc(tlen);
-  if (buf == NULL) {
-    goto _return;
-  }
-
-  tSerializeSCMCreateStreamReq(buf, tlen, &req);
-
-  pRequest->body.requestMsg = (SDataBuf){
-      .pData = buf,
-      .len = tlen,
-      .handle = NULL,
-  };
-  pRequest->type = TDMT_MND_CREATE_STREAM;
-
-  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  SEpSet        epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
-
-  int64_t transporterId = 0;
-  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
-
-  tsem_wait(&pRequest->body.rspSem);
-
-_return:
-  taosMemoryFreeClear(astStr);
-  qDestroyQuery(pQueryNode);
-  /*if (sendInfo != NULL) {*/
-  /*destroySendMsgInfo(sendInfo);*/
-  /*}*/
-
-  if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
-    pRequest->code = terrno;
-  }
-
-  return pRequest;
-}
-#endif
-
 #if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
@@ -1540,10 +1364,11 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   }
   return NULL;
 }
-DLL_EXPORT void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
+
+void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
   tmqCommitInner(tmq, offsets, 0, 1, cb, param);
 }
 
-DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
+tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
   return tmqCommitInner(tmq, offsets, 0, 0, NULL, NULL);
 }
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 823ce6d2fe..528d8a1580 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -78,7 +78,18 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDisp
   // 2.1. idle: exec
   // 2.2. executing: return
   // 2.3. closing: keep trying
-  streamExec(pTask, pMsgCb);
+  if (pTask->execType != TASK_EXEC__NONE) {
+    streamExec(pTask, pMsgCb);
+  } else {
+    ASSERT(pTask->sinkType != TASK_SINK__NONE);
+    while (1) {
+      void* data = streamQueueNextItem(pTask->inputQueue);
+      if (data == NULL) return 0;
+      if (streamTaskOutput(pTask, data) < 0) {
+        ASSERT(0);
+      }
+    }
+  }
 
   // 3. handle output
   // 3.1 check and set status
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 72df516e0d..00365aded5 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -76,7 +76,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
         streamDataSubmitRefDec((SStreamDataSubmit*)data);
         taosFreeQitem(data);
       } else {
-        taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
+        /*taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);*/
         taosFreeQitem(data);
       }
       streamQueueProcessSuccess(pTask->inputQueue);

From 80a29a6a315edf66ea63eafe8f806fe59a532dc6 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Thu, 9 Jun 2022 10:09:49 +0800
Subject: [PATCH 56/63] fix: add group id to sorted merge operator

---
 source/libs/executor/inc/executorimpl.h |  4 ++
 source/libs/executor/src/executorimpl.c | 64 ++++++++++++++++++++++++-
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index ce1ae318dc..68d71a0c24 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -688,6 +688,10 @@ typedef struct SSortedMergeOperatorInfo {
   int32_t          numOfResPerPage;
   char**           groupVal;
   SArray          *groupInfo;
+
+  bool hasGroupId;
+  uint64_t groupId;
+  STupleHandle* prefetchedTuple;
 } SSortedMergeOperatorInfo;
 
 typedef struct SSortOperatorInfo {
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 3d7faa1c83..fe536673aa 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3133,6 +3133,68 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
   return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL;
 }
 
+SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
+                                SSortedMergeOperatorInfo *pInfo) {
+  blockDataCleanup(pDataBlock);
+
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
+
+  while (1) {
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
+    if (p->info.rows >= capacity) {
+      break;
+    }
+  }
+
+  if (p->info.rows > 0) {
+    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+
+      SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      colDataAssign(pDst, pSrc, p->info.rows);
+    }
+
+    pDataBlock->info.rows = p->info.rows;
+    pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
+  }
+
+  blockDataDestroy(p);
+  return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
+}
+
 static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -3141,7 +3203,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
   SSortedMergeOperatorInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL);
+    return getSortedMergeBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL, pInfo);
   }
 
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;

From 4e6010f59be0438636494a68a9303e12cf5dd0ca Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 9 Jun 2022 10:15:09 +0800
Subject: [PATCH 57/63] fix: avoid invalid conn

---
 source/dnode/mgmt/node_mgmt/src/dmTransport.c | 10 ++++----
 source/libs/transport/src/transSvr.c          | 25 +++++++++++++------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 0ad1c508b2..08c1ff90ef 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -23,7 +23,7 @@ static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg);
 
 static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) {
   SRpcConnInfo connInfo = {0};
-  if (IsReq(pRpc) && rpcGetConnInfo(pRpc->info.handle, &connInfo) != 0) {
+  if (IsReq(pRpc) && rpcGetConnInfo(&pRpc->info, &connInfo) != 0) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
     dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->info.ahandle, pRpc->info.handle);
     return -1;
@@ -49,9 +49,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
 }
 
 static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
-  SDnodeTrans  *pTrans = &pDnode->trans;
+  SDnodeTrans * pTrans = &pDnode->trans;
   int32_t       code = -1;
-  SRpcMsg      *pMsg = NULL;
+  SRpcMsg *     pMsg = NULL;
   SMgmtWrapper *pWrapper = NULL;
   SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
 
@@ -167,11 +167,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
 
   for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
-    SArray       *pArray = (*pWrapper->func.getHandlesFp)();
+    SArray *      pArray = (*pWrapper->func.getHandlesFp)();
     if (pArray == NULL) return -1;
 
     for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-      SMgmtHandle  *pMgmt = taosArrayGet(pArray, i);
+      SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
       SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
       if (pMgmt->needCheckVgId) {
         pHandle->needCheckVgId = pMgmt->needCheckVgId;
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 50f99128b2..d73d85b90f 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -1153,23 +1153,34 @@ _return2:
   rpcFreeCont(msg->pCont);
 }
 
-int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) {
+int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) {
   if (thandle == NULL) {
     tTrace("invalid handle %p, failed to Get Conn info", thandle);
     return -1;
   }
-  SExHandle* ex = thandle;
-  SSvrConn*  pConn = ex->handle;
+  SRpcHandleInfo* pInfo = thandle;
+  SExHandle*      exh = pInfo->handle;
+  int64_t         refId = pInfo->refId;
+  ASYNC_CHECK_HANDLE(exh, refId);
+
+  // SExHandle* ex = thandle;
+  SSvrConn* pConn = exh->handle;
   if (pConn == NULL) {
     tTrace("invalid handle %p, failed to Get Conn info", thandle);
+    transReleaseExHandle(refMgt, refId);
     return -1;
   }
-
   struct sockaddr_in addr = pConn->addr;
-  pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);
-  pInfo->clientPort = ntohs(addr.sin_port);
-  tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
+  pConnInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);
+  pConnInfo->clientPort = ntohs(addr.sin_port);
+  tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user));
+  transReleaseExHandle(refMgt, refId);
   return 0;
+_return1:
+  transReleaseExHandle(refMgt, refId);
+  return -1;
+_return2:
+  return -1;
 }
 
 #endif

From c7bbd9354b7a4c83132a830d73ccdebcbb3365cb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 9 Jun 2022 10:17:43 +0800
Subject: [PATCH 58/63] fix(query): set correct column number value for
 ssdatablock.

---
 source/common/src/tdatablock.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index fd2b93309e..a3a94c50f2 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1945,7 +1945,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
   blockDataEnsureCapacity(pBlock, numOfRows);
   pBlock->info.rows = numOfRows;
-  pBlock->info.numOfCols = numOfCols;
 
   const char* pStart = pData;
 
@@ -1960,7 +1959,8 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
     taosArraySetSize(pBlock->pDataBlock, numOfCols);
   }
 
-  ASSERT(pBlock->pDataBlock->size == numOfCols);
+  pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  ASSERT(pBlock->pDataBlock->size >= numOfCols);
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

From 840074dfa3c08fd05b6a350e598f221a5b6cf900 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 9 Jun 2022 11:40:49 +0800
Subject: [PATCH 59/63] refactor: do some internal refactor, and add a new api

---
 include/client/taos.h          |  1 +
 source/client/inc/clientInt.h  |  3 ++-
 source/client/src/clientEnv.c  |  2 ++
 source/client/src/clientImpl.c |  6 +++++-
 source/client/src/clientMain.c | 25 ++++++++++++++++++-------
 5 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/include/client/taos.h b/include/client/taos.h
index 5bb528bd7f..b4e5a41ccf 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -194,6 +194,7 @@ DLL_EXPORT int         taos_errno(TAOS_RES *tres);
 
 DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
 DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
+DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES* res, __taos_async_fn_t fp, void* param);
 
 // Shuduo: temporary enable for app build
 #if 1
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 3a6f3badc0..6aa83e9575 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -171,6 +171,7 @@ typedef struct SReqResultInfo {
   uint32_t       current;
   bool           completed;
   int32_t        precision;
+  bool           convertUcs4;
   int32_t        payloadLen;
 } SReqResultInfo;
 
@@ -222,7 +223,7 @@ typedef struct SSyncQueryParam {
   SRequestObj* pRequest;
 } SSyncQueryParam;
 
-void*   doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void*   doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
 void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
 
 void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 75e1884360..171f06c257 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -191,6 +191,8 @@ void *createRequest(STscObj *pObj, int32_t type) {
   pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampUs();
 
+  pRequest->body.resInfo.convertUcs4 = true;  // convert ucs4 by default
+
   pRequest->type = type;
   pRequest->pTscObj = pObj;
   pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index e15624bc0d..9025121854 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1114,7 +1114,7 @@ static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
   tsem_post(&pParam->sem);
 }
 
-void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
+void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
   assert(pRequest != NULL);
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@@ -1126,6 +1126,10 @@ void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUc
     }
 
     SSyncQueryParam* pParam = pRequest->body.param;
+
+    // convert ucs4 to native multi-bytes string
+    pResultInfo->convertUcs4 = convertUcs4;
+
     taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
     tsem_wait(&pParam->sem);
   }
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 1a1925e244..5e442c4bf1 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -219,14 +219,13 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
 
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
+#if SYNC_ON_TOP_OF_ASYNC
+    return doAsyncFetchRows(pRequest, true, true);
+#else
     if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
         pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
       return NULL;
     }
-
-#if SYNC_ON_TOP_OF_ASYNC
-    return doAsyncFetchRow(pRequest, true, true);
-#else
     return doFetchRows(pRequest, true, true);
 #endif
 
@@ -489,6 +488,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
   if (res == NULL) {
     return 0;
   }
+
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
 
@@ -501,7 +501,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     }
 
 #if SYNC_ON_TOP_OF_ASYNC
-    doAsyncFetchRow(pRequest, false, true);
+    doAsyncFetchRows(pRequest, false, true);
 #else
     doFetchRows(pRequest, true, true);
 #endif
@@ -552,7 +552,11 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
     return 0;
   }
 
+#if SYNC_ON_TOP_OF_ASYNC
+  doAsyncFetchRows(pRequest, false, false);
+#else
   doFetchRows(pRequest, false, false);
+#endif
 
   SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
@@ -771,11 +775,11 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
-    pRequest->code = code;
     pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    return;
   }
 
-  pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResultInfo->pData, true, false);
+  pRequest->code = setQueryResultFromRsp(pResultInfo, (SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, false);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     pResultInfo->numOfRows = 0;
     pRequest->code = code;
@@ -815,6 +819,13 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   schedulerAsyncFetchRows(pRequest->body.queryJob, fetchCallback, pRequest);
 }
 
+void taos_fetch_raw_block_a(TAOS_RES* res, __taos_async_fn_t fp, void* param) {
+  ASSERT(res != NULL && fp != NULL);
+  SRequestObj *pRequest = res;
+  pRequest->body.resInfo.convertUcs4 = false;
+  taos_fetch_rows_a(res, fp, param);
+}
+
 TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp,
                          void *param, int interval) {
   // TODO

From 9e4b3adc7c2cef3f0e88c431f46e65de0a7ddc11 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 9 Jun 2022 11:45:06 +0800
Subject: [PATCH 60/63] fix(stream): msg dispatch

---
 include/libs/stream/tstream.h               | 1 +
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 ++
 source/dnode/vnode/src/tq/tq.c              | 2 +-
 source/libs/executor/src/scanoperator.c     | 6 +++---
 source/libs/stream/src/stream.c             | 2 +-
 source/libs/stream/src/streamDispatch.c     | 9 ++++++++-
 source/libs/stream/src/streamExec.c         | 3 ++-
 7 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index ddd7e1cd02..6b5eb3b491 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -339,6 +339,7 @@ typedef struct {
   int32_t sourceTaskId;
   int32_t sourceVg;
   int32_t sourceChildId;
+  int32_t upstreamNodeId;
 #if 0
   int64_t sourceVer;
 #endif
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 4c5a32536f..0bc9568caf 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -353,7 +353,9 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 181cddee47..a1b8d81d58 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -448,7 +448,7 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
-  SStreamDispatchRsp* pRsp = pMsg->pCont;
+  SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t             taskId = pRsp->taskId;
   SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
   streamProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index cc76cc29e4..1e2a26386b 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
   SSDataBlock* pSDB = pInfo->pUpdateRes;
   STimeWindow  win = {
-      .skey = INT64_MIN,
-      .ekey = INT64_MAX,
+       .skey = INT64_MIN,
+       .ekey = INT64_MAX,
   };
   bool needRead = false;
   if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
@@ -846,7 +846,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
   if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
     if (pInfo->validBlockIndex >= total) {
-      doClearBufferedBlocks(pInfo);
+      /*doClearBufferedBlocks(pInfo);*/
       pOperator->status = OP_EXEC_DONE;
       return NULL;
     }
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 528d8a1580..069595390d 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -59,7 +59,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
 
   // rsp by input status
   void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
-  ((SMsgHead*)buf)->vgId = htonl(pReq->sourceVg);
+  ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId);
   SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
   pCont->inputStatus = status;
   pCont->streamId = pReq->streamId;
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index d1e3fa0799..16da418677 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -22,6 +22,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
   if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
   ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
   ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum);
@@ -42,6 +43,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
   if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
   ASSERT(pReq->blockNum > 0);
   pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*));
@@ -94,6 +96,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
       .sourceTaskId = pTask->taskId,
       .sourceVg = data->sourceVg,
       .sourceChildId = pTask->childId,
+      .upstreamNodeId = pTask->nodeId,
       .blockNum = blockNum,
   };
 
@@ -184,13 +187,17 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
 #endif
 
   SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue);
-  if (pBlock == NULL) return 0;
+  if (pBlock == NULL) {
+    atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
+    return 0;
+  }
   ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK);
 
   SRpcMsg dispatchMsg = {0};
   SEpSet* pEpSet = NULL;
   if (streamBuildDispatchMsg(pTask, pBlock, &dispatchMsg, &pEpSet) < 0) {
     ASSERT(0);
+    atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
     return -1;
   }
 
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 00365aded5..fe1a857743 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -65,6 +65,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
       }
       qRes->type = STREAM_INPUT__DATA_BLOCK;
       qRes->blocks = pRes;
+      /*qRes->sourceVg = pTask->nodeId;*/
       if (streamTaskOutput(pTask, qRes) < 0) {
         streamQueueProcessFail(pTask->inputQueue);
         taosArrayDestroy(pRes);
@@ -76,7 +77,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
         streamDataSubmitRefDec((SStreamDataSubmit*)data);
         taosFreeQitem(data);
       } else {
-        /*taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);*/
+        taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
         taosFreeQitem(data);
       }
       streamQueueProcessSuccess(pTask->inputQueue);

From c9b0b7cd7602f6f26038ed1b69a9a7a77ca616c0 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 9 Jun 2022 11:53:50 +0800
Subject: [PATCH 61/63] feat: add test case for tsrow

---
 source/common/src/tdataformat.c       |  12 +-
 source/common/test/CMakeLists.txt     |  14 +-
 source/common/test/dataformatTest.cpp | 482 +++++++++++++++++++++++++-
 3 files changed, 500 insertions(+), 8 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 6037b011af..84a2cf0544 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -244,7 +244,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     }
   }
 
-  ASSERT(flags);
+  // ASSERT(flags); // only 1 column(ts)
 
   // decide
   uint32_t nData = 0;
@@ -268,7 +268,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
         nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv;
         break;
       default:
-        ASSERT(0);
+        break; // only ts column
+        // ASSERT(0);
     }
 
     uint8_t tflags = 0;
@@ -283,7 +284,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
       tflags |= TSROW_KV_BIG;
     }
 
-    if (nDataT < nDataK) {
+    if (nDataT <= nDataK) {
       nData = nDataT;
     } else {
       nData = nDataK;
@@ -373,7 +374,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
           ptv = pf + pTSchema->flen;
           break;
         default:
-          ASSERT(0);
+          // ASSERT(0);
+          break;
       }
     } else {
       pTSKVRow = (STSKVRow *)(*ppRow)->pData;
@@ -495,7 +497,7 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal
   SValue    value;
 
   ASSERT(iCol < pTSchema->numOfCols);
-  ASSERT(flags);
+  // ASSERT(flags); // only 1 ts column
   ASSERT(pRow->sver == pTSchema->version);
 
   if (iCol == 0) {
diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt
index 0535b08be7..eb79e79afa 100644
--- a/source/common/test/CMakeLists.txt
+++ b/source/common/test/CMakeLists.txt
@@ -5,7 +5,12 @@ MESSAGE(STATUS "build parser unit test")
 SET(CMAKE_CXX_STANDARD 11)
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-ADD_EXECUTABLE(commonTest ${SOURCE_LIST})
+ADD_EXECUTABLE(commonTest "")
+TARGET_SOURCES(
+    commonTest
+    PRIVATE
+    "commonTests.cpp"
+)
 TARGET_LINK_LIBRARIES(
         commonTest
         PUBLIC os util common gtest
@@ -24,7 +29,12 @@ target_sources(
     PRIVATE
     "dataformatTest.cpp"
 )
-target_link_libraries(dataformatTest gtest gtest_main util)
+target_link_libraries(dataformatTest gtest gtest_main util common)
+target_include_directories(
+        dataformatTest
+        PUBLIC "${TD_SOURCE_DIR}/include/common"
+        PUBLIC "${TD_SOURCE_DIR}/include/util"
+)
 
 # tmsg test
 # add_executable(tmsgTest "")
diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
index 3497014c22..81e91da0d6 100644
--- a/source/common/test/dataformatTest.cpp
+++ b/source/common/test/dataformatTest.cpp
@@ -1 +1,481 @@
-#include "gtest/gtest.h"
\ No newline at end of file
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tdataformat.h>
+#include <tglobal.h>
+#include <tmsg.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#define NONE_CSTR "no"
+#define NULL_CSTR "nu"
+#define NONE_LEN  2
+#define NULL_LEN  2
+const static int16_t MAX_COLS = 14;
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+STSchema *genSTSchema(int16_t nCols) {
+  EXPECT_LE(nCols, MAX_COLS);
+  SSchema *pSchema = (SSchema *)taosMemoryCalloc(nCols, sizeof(SSchema));
+  EXPECT_NE(pSchema, nullptr);
+
+  for (int16_t i = 0; i < nCols; ++i) {
+    pSchema[i].colId = PRIMARYKEY_TIMESTAMP_COL_ID + i;
+    char colName[TSDB_COL_NAME_LEN] = {0};
+    snprintf(colName, TSDB_COL_NAME_LEN, "c%" PRIi16, i);
+    strncpy(pSchema[i].name, colName, TSDB_COL_NAME_LEN);
+
+    switch (i) {
+      case 0: {
+        pSchema[0].type = TSDB_DATA_TYPE_TIMESTAMP;
+        pSchema[0].bytes = TYPE_BYTES[pSchema[0].type];
+      } break;
+      case 1: {
+        pSchema[1].type = TSDB_DATA_TYPE_INT;
+        pSchema[1].bytes = TYPE_BYTES[pSchema[1].type];
+        ;
+      } break;
+      case 2: {
+        pSchema[2].type = TSDB_DATA_TYPE_BIGINT;
+        pSchema[2].bytes = TYPE_BYTES[pSchema[2].type];
+      } break;
+      case 3: {
+        pSchema[3].type = TSDB_DATA_TYPE_FLOAT;
+        pSchema[3].bytes = TYPE_BYTES[pSchema[3].type];
+      } break;
+      case 4: {
+        pSchema[4].type = TSDB_DATA_TYPE_DOUBLE;
+        pSchema[4].bytes = TYPE_BYTES[pSchema[4].type];
+      } break;
+      case 5: {
+        pSchema[5].type = TSDB_DATA_TYPE_BINARY;
+        pSchema[5].bytes = 12;
+      } break;
+      case 6: {
+        pSchema[6].type = TSDB_DATA_TYPE_NCHAR;
+        pSchema[6].bytes = 42;
+      } break;
+      case 7: {
+        pSchema[7].type = TSDB_DATA_TYPE_TINYINT;
+        pSchema[7].bytes = TYPE_BYTES[pSchema[7].type];
+      } break;
+      case 8: {
+        pSchema[8].type = TSDB_DATA_TYPE_SMALLINT;
+        pSchema[8].bytes = TYPE_BYTES[pSchema[8].type];
+      } break;
+      case 9: {
+        pSchema[9].type = TSDB_DATA_TYPE_BOOL;
+        pSchema[9].bytes = TYPE_BYTES[pSchema[9].type];
+      } break;
+      case 10: {
+        pSchema[10].type = TSDB_DATA_TYPE_UTINYINT;
+        pSchema[10].bytes = TYPE_BYTES[pSchema[10].type];
+      } break;
+      case 11: {
+        pSchema[11].type = TSDB_DATA_TYPE_USMALLINT;
+        pSchema[11].bytes = TYPE_BYTES[pSchema[11].type];
+      } break;
+      case 12: {
+        pSchema[12].type = TSDB_DATA_TYPE_UINT;
+        pSchema[12].bytes = TYPE_BYTES[pSchema[12].type];
+      } break;
+      case 13: {
+        pSchema[13].type = TSDB_DATA_TYPE_UBIGINT;
+        pSchema[13].bytes = TYPE_BYTES[pSchema[13].type];
+      } break;
+
+      default:
+        ASSERT(0);
+        break;
+    }
+  }
+
+  STSchema *pResult = NULL;
+  pResult = tdGetSTSChemaFromSSChema(&pSchema, nCols);
+
+  taosMemoryFree(pSchema);
+  return pResult;
+}
+
+// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
+  if (!(*pArray)) {
+    *pArray = taosArrayInit(nCols, sizeof(SColVal));
+    if (!(*pArray)) return -1;
+  }
+
+  for (int16_t i = 0; i < nCols; ++i) {
+    SColVal colVal = {0};
+    colVal.cid = PRIMARYKEY_TIMESTAMP_COL_ID + i;
+    if (strncasecmp(data[i], NONE_CSTR, NONE_LEN) == 0) {
+      colVal.isNone = 1;
+      taosArrayPush(*pArray, &colVal);
+      continue;
+    } else if (strncasecmp(data[i], NULL_CSTR, NULL_LEN) == 0) {
+      colVal.isNull = 1;
+      taosArrayPush(*pArray, &colVal);
+      continue;
+    }
+
+    switch (i) {
+      case 0:
+        sscanf(data[i], "%" PRIi64, &colVal.value.ts);
+        break;
+      case 1: {
+        sscanf(data[i], "%" PRIi32, &colVal.value.i32);
+      } break;
+      case 2:
+        sscanf(data[i], "%" PRIi64, &colVal.value.i64);
+        break;
+      case 3:
+        sscanf(data[i], "%f", &colVal.value.f);
+        break;
+      case 4:
+        sscanf(data[i], "%lf", &colVal.value.d);
+        break;
+      case 5: {
+        int16_t dataLen = strlen(data[i]) + 1;
+        colVal.value.nData = dataLen < 10 ? dataLen : 10;
+        colVal.value.pData = (uint8_t *)data[i];
+      } break;
+      case 6: {
+        int16_t dataLen = strlen(data[i]) + 1;
+        colVal.value.nData = dataLen < 40 ? dataLen : 40;
+        colVal.value.pData = (uint8_t *)data[i];  // just for test, not real nchar
+      } break;
+      case 7:
+      case 9: {
+        int32_t d8;
+        sscanf(data[i], "%" PRId32, &d8);
+        colVal.value.i8 = (int8_t)d8;
+      } break;
+      case 8: {
+        int32_t d16;
+        sscanf(data[i], "%" PRId32, &d16);
+        colVal.value.i16 = (int16_t)d16;
+      } break;
+      case 10: {
+        uint32_t u8;
+        sscanf(data[i], "%" PRId32, &u8);
+        colVal.value.u8 = (uint8_t)u8;
+      } break;
+      case 11: {
+        uint32_t u16;
+        sscanf(data[i], "%" PRId32, &u16);
+        colVal.value.u16 = (uint16_t)u16;
+      } break;
+      case 12: {
+        sscanf(data[i], "%" PRIu32, &colVal.value.u32);
+      } break;
+      case 13: {
+        sscanf(data[i], "%" PRIu64, &colVal.value.u64);
+      } break;
+      default:
+        ASSERT(0);
+    }
+    taosArrayPush(*pArray, &colVal);
+  }
+  return 0;
+}
+
+int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
+  if (cv->isNone) {
+    printf("None ");
+    return 0;
+  }
+  if (cv->isNull) {
+    printf("Null ");
+    return 0;
+  }
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+      printf("%s ", cv->value.i8 == 0 ? "false" : "true");
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      printf("%" PRIi8 " ", cv->value.i8);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      printf("%" PRIi16 " ", cv->value.i16);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      printf("%" PRIi32 " ", cv->value.i32);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      printf("%" PRIi64 " ", cv->value.i64);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      printf("%f ", cv->value.f);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      printf("%lf ", cv->value.d);
+      break;
+    case TSDB_DATA_TYPE_VARCHAR: {
+      char tv[15] = {0};
+      snprintf(tv, 15, "%s", cv->value.pData);
+      printf("%s ", tv);
+    } break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      printf("%" PRIi64 " ", cv->value.i64);
+      break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      char tv[15] = {0};
+      snprintf(tv, 15, "%s", cv->value.pData);
+      printf("%s ", tv);
+    } break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      printf("%" PRIu8 " ", cv->value.u8);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      printf("%" PRIu16 " ", cv->value.u16);
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      printf("%" PRIu32 " ", cv->value.u32);
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      printf("%" PRIu64 " ", cv->value.u64);
+      break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+  return 0;
+}
+
+void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
+  printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+    SColVal cv = {0};
+    tTSRowGet(row, pTSchema, i, &cv);
+    debugPrintSColVal(&cv, pTSchema->columns[i].type);
+  }
+  printf("\n");
+  fflush(stdout);
+}
+
+static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
+  ASSERT(rawVal);
+
+  if (cv->isNone) {
+    EXPECT_STRCASEEQ(rawVal, NONE_CSTR);
+    return 0;
+  }
+  if (cv->isNull) {
+    EXPECT_STRCASEEQ(rawVal, NULL_CSTR);
+    return 0;
+  }
+
+  SValue rawSVal = {0};
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT: {
+      int32_t d8;
+      sscanf(rawVal, "%" PRId32, &d8);
+      EXPECT_EQ(cv->value.i8, (int8_t)d8);
+    } break;
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int32_t d16;
+      sscanf(rawVal, "%" PRId32, &d16);
+      EXPECT_EQ(cv->value.i16, (int16_t)d16);
+    } break;
+    case TSDB_DATA_TYPE_INT: {
+      sscanf(rawVal, "%" PRId32, &rawSVal.i32);
+      EXPECT_EQ(cv->value.i32, rawSVal.i32);
+    } break;
+    case TSDB_DATA_TYPE_BIGINT: {
+      sscanf(rawVal, "%" PRIi64, &rawSVal.i64);
+      EXPECT_EQ(cv->value.i64, rawSVal.i64);
+    } break;
+    case TSDB_DATA_TYPE_FLOAT: {
+      sscanf(rawVal, "%f", &rawSVal.f);
+      EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f);
+    } break;
+    case TSDB_DATA_TYPE_DOUBLE: {
+      sscanf(rawVal, "%lf", &rawSVal.d);
+      EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d);
+    } break;
+    case TSDB_DATA_TYPE_VARCHAR: {
+      EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);
+    } break;
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      sscanf(rawVal, "%" PRIi64, &rawSVal.ts);
+      EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts);
+    } break;
+    case TSDB_DATA_TYPE_NCHAR: {
+      EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);  // informal nchar comparsion
+    } break;
+    case TSDB_DATA_TYPE_UTINYINT: {
+      uint32_t u8;
+      sscanf(rawVal, "%" PRIu32, &u8);
+      EXPECT_EQ(cv->value.u8, (uint8_t)u8);
+    } break;
+    case TSDB_DATA_TYPE_USMALLINT: {
+      uint32_t u16;
+      sscanf(rawVal, "%" PRIu32, &u16);
+      EXPECT_EQ(cv->value.u16, (uint16_t)u16);
+    } break;
+    case TSDB_DATA_TYPE_UINT: {
+      sscanf(rawVal, "%" PRIu32, &rawSVal.u32);
+      EXPECT_EQ(cv->value.u32, rawSVal.u32);
+    } break;
+    case TSDB_DATA_TYPE_UBIGINT: {
+      sscanf(rawVal, "%" PRIu64, &rawSVal.u64);
+      EXPECT_EQ(cv->value.u64, rawSVal.u64);
+    } break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+  return 0;
+}
+
+static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
+  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+    SColVal cv = {0};
+    tTSRowGet(row, pTSchema, i, &cv);
+    checkSColVal(data[i], &cv, pTSchema->columns[i].type);
+  }
+}
+
+TEST(testCase, AllNormTest) {
+  int16_t  nCols = 1;
+  STSRow2 *row = nullptr;
+  SArray  *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  EXPECT_NE(pArray, nullptr);
+
+  STSchema *pTSchema = genSTSchema(nCols);
+  EXPECT_NE(pTSchema, nullptr);
+
+  // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
+  // c9 bool
+  char *data[10] = {"1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "1"};
+
+  genTestData((const char **)&data, nCols, &pArray);
+
+  tTSRowNew(NULL, pArray, pTSchema, &row);
+
+  debugPrintTSRow(row, pTSchema, __func__, __LINE__);
+  checkTSRow((const char **)&data, row, pTSchema);
+
+  taosArrayDestroy(pArray);
+  taosMemoryFree(pTSchema);
+}
+
+#if 1
+TEST(testCase, NoneTest) {
+  const static int nCols = 14;
+  const static int nRows = 20;
+  STSRow2         *row = nullptr;
+  SArray          *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  EXPECT_NE(pArray, nullptr);
+
+  STSchema *pTSchema = genSTSchema(nCols);
+  EXPECT_NE(pTSchema, nullptr);
+
+  // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
+  // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
+  const char *data[nRows][nCols] = {
+      {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
+      {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220002", "no", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "nu"},
+      {"1653694220003", "nu", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+      {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+      {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+      {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+      {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
+      {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
+      {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
+      {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
+      {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
+      {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
+       "18446744073709551615"},
+      {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+      {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
+      {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
+       "4294967295", "100"},
+      {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
+       "no"}};
+
+
+  for (int r = 0; r < nRows; ++r) {
+    genTestData((const char **)&data[r], nCols, &pArray);
+    tTSRowNew(NULL, pArray, pTSchema, &row);
+    debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
+    checkTSRow((const char **)&data[r], row, pTSchema);  // check
+    taosMemoryFreeClear(row);
+    taosArrayClear(pArray);
+  }
+
+  taosArrayDestroy(pArray);
+  taosMemoryFree(pTSchema);
+}
+#endif
\ No newline at end of file

From d9c83f6357ca9039a1abf85176939192151cb8ca Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 9 Jun 2022 13:50:18 +0800
Subject: [PATCH 62/63] enh: change rpcGetConnInfo

---
 include/libs/index/index.h                    |  8 +--
 include/libs/transport/trpc.h                 |  1 +
 source/dnode/mgmt/node_mgmt/src/dmTransport.c | 18 ++---
 source/dnode/vnode/src/meta/metaTable.c       |  2 +-
 source/libs/index/inc/indexInt.h              |  2 +-
 source/libs/index/src/index.c                 |  2 +-
 source/libs/index/src/indexCache.c            |  8 +--
 source/libs/index/src/indexFilter.c           |  2 +-
 source/libs/index/src/indexJson.c             |  8 +--
 source/libs/index/test/jsonUT.cc              | 70 +++++++++----------
 source/libs/transport/src/transSvr.c          | 37 +++-------
 11 files changed, 69 insertions(+), 89 deletions(-)

diff --git a/include/libs/index/index.h b/include/libs/index/index.h
index bd601f1d9f..9e71c941d3 100644
--- a/include/libs/index/index.h
+++ b/include/libs/index/index.h
@@ -136,14 +136,14 @@ int indexRebuild(SIndex* index, SIndexOpts* opt);
  * @param index (output, index json object)
  * @return error code
  */
-int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index);
+int indexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index);
 /*
  * close index
  * @param index (input, index to be closed)
  * @return void
  */
 
-void tIndexJsonClose(SIndexJson* index);
+void indexJsonClose(SIndexJson* index);
 
 /*
  * insert terms into index
@@ -152,7 +152,7 @@ void tIndexJsonClose(SIndexJson* index);
  * @param uid  (input, uid of terms)
  * @return error code
  */
-int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
+int indexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
 /*
  * search index
  * @param index (input, index object)
@@ -161,7 +161,7 @@ int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
  * @return error code
  */
 
-int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result);
+int indexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result);
 /*
  * @param
  * @param
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 1535f9d290..fd57eef83a 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -46,6 +46,7 @@ typedef struct SRpcHandleInfo {
   int32_t noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int32_t persistHandle;  // persist handle or not
 
+  SRpcConnInfo connInfo;
   // app info
   void *ahandle;  // app handle set by client
   void *wrapper;  // wrapper handle
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 08c1ff90ef..b66e559370 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -22,17 +22,17 @@ static void dmSendRsp(SRpcMsg *pMsg);
 static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg);
 
 static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) {
-  SRpcConnInfo connInfo = {0};
-  if (IsReq(pRpc) && rpcGetConnInfo(&pRpc->info, &connInfo) != 0) {
-    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->info.ahandle, pRpc->info.handle);
-    return -1;
-  }
+  SRpcConnInfo *pConnInfo = &(pRpc->info.connInfo);
+  // if (IsReq(pRpc)) {
+  //  terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+  //  dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->info.ahandle, pRpc->info.handle);
+  //  return -1;
+  //}
 
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
-  memcpy(pMsg->conn.user, connInfo.user, TSDB_USER_LEN);
-  pMsg->conn.clientIp = connInfo.clientIp;
-  pMsg->conn.clientPort = connInfo.clientPort;
+  memcpy(pMsg->conn.user, pConnInfo->user, TSDB_USER_LEN);
+  pMsg->conn.clientIp = pConnInfo->clientIp;
+  pMsg->conn.clientPort = pConnInfo->clientPort;
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 8d4762a656..9a05f9e5a0 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -99,7 +99,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
       indexMultiTermAdd(terms, term);
     }
   }
-  tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
+  indexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
   indexMultiTermDestroy(terms);
 #endif
   return 0;
diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h
index f556a738dd..47f7260d3a 100644
--- a/source/libs/index/inc/indexInt.h
+++ b/source/libs/index/inc/indexInt.h
@@ -131,7 +131,7 @@ typedef struct TFileCacheKey {
   char*    colName;
   int32_t  nColName;
 } ICacheKey;
-int indexFlushCacheToTFile(SIndex* sIdx, void*, bool quit);
+int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit);
 
 int64_t indexAddRef(void* p);
 int32_t indexRemoveRef(int64_t ref);
diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index c4b85d11e0..9b8bee5623 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -455,7 +455,7 @@ static void idxDestroyFinalRslt(SArray* result) {
   taosArrayDestroy(result);
 }
 
-int indexFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
+int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
   if (sIdx == NULL) {
     return -1;
   }
diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c
index 44a05832a3..20cd9c8b4c 100644
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@@ -69,7 +69,7 @@ static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRsl
      cacheSearchRange_JSON}};
 
 static void doMergeWork(SSchedMsg* msg);
-static bool indexCacheIteratorNext(Iterate* itera);
+static bool idxCacheIteratorNext(Iterate* itera);
 
 static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   if (cache == NULL) {
@@ -476,7 +476,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   iiter->val.val = taosArrayInit(1, sizeof(uint64_t));
   iiter->val.colVal = NULL;
   iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL;
-  iiter->next = indexCacheIteratorNext;
+  iiter->next = idxCacheIteratorNext;
   iiter->getValue = indexCacheIteratorGetValue;
 
   taosThreadMutexUnlock(&cache->mtx);
@@ -748,9 +748,9 @@ static void doMergeWork(SSchedMsg* msg) {
 
   int quit = msg->thandle ? true : false;
   taosMemoryFree(msg->thandle);
-  indexFlushCacheToTFile(sidx, pCache, quit);
+  idxFlushCacheToTFile(sidx, pCache, quit);
 }
-static bool indexCacheIteratorNext(Iterate* itera) {
+static bool idxCacheIteratorNext(Iterate* itera) {
   SSkipListIterator* iter = itera->iter;
   if (iter == NULL) {
     return false;
diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c
index 66ed88077c..e4af4a7a3f 100644
--- a/source/libs/index/src/indexFilter.c
+++ b/source/libs/index/src/indexFilter.c
@@ -355,7 +355,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
 
     SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
     indexMultiTermQueryAdd(mtm, tm, qtype);
-    ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result);
+    ret = indexJsonSearch(arg->ivtIdx, mtm, output->result);
   } else {
     bool   reverse;
     Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c
index a4f3be3829..88b3d907bb 100644
--- a/source/libs/index/src/indexJson.c
+++ b/source/libs/index/src/indexJson.c
@@ -15,11 +15,11 @@
 #include "index.h"
 #include "indexInt.h"
 
-int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
+int indexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
   // handle
   return indexOpen(opts, path, index);
 }
-int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
+int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
   for (int i = 0; i < taosArrayGetSize(terms); i++) {
     SIndexJsonTerm *p = taosArrayGetP(terms, i);
     if (p->colType == TSDB_DATA_TYPE_BOOL) {
@@ -36,7 +36,7 @@ int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
   return indexPut(index, terms, uid);
 }
 
-int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) {
+int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) {
   SArray *terms = tq->query;
   for (int i = 0; i < taosArrayGetSize(terms); i++) {
     SIndexJsonTerm *p = taosArrayGetP(terms, i);
@@ -54,7 +54,7 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re
   return indexSearch(index, tq, result);
 }
 
-void tIndexJsonClose(SIndexJson *index) {
+void indexJsonClose(SIndexJson *index) {
   // handle close
   return indexClose(index);
 }
diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc
index d6bc31a801..134fb53d2b 100644
--- a/source/libs/index/test/jsonUT.cc
+++ b/source/libs/index/test/jsonUT.cc
@@ -56,11 +56,11 @@ class JsonEnv : public ::testing::Test {
 
     initLog();
     opts = indexOptsCreate();
-    int ret = tIndexJsonOpen(opts, dir.c_str(), &index);
+    int ret = indexJsonOpen(opts, dir.c_str(), &index);
     assert(ret == 0);
   }
   virtual void TearDown() {
-    tIndexJsonClose(index);
+    indexJsonClose(index);
     indexOptsDestroy(opts);
     printf("destory\n");
     taosMsleep(1000);
@@ -75,7 +75,7 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
                                       (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
-  tIndexJsonPut(index, terms, (int64_t)tableId);
+  indexJsonPut(index, terms, (int64_t)tableId);
 
   indexMultiTermDestroy(terms);
 }
@@ -86,7 +86,7 @@ static void delData(SIndexJson* index, const std::string& colName, int8_t dtype,
                                       (const char*)data, dlen);
   SIndexMultiTerm* terms = indexMultiTermCreate();
   indexMultiTermAdd(terms, term);
-  tIndexJsonPut(index, terms, (int64_t)tableId);
+  indexJsonPut(index, terms, (int64_t)tableId);
 
   indexMultiTermDestroy(terms);
 }
@@ -99,7 +99,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v
 
   SArray* res = taosArrayInit(1, sizeof(uint64_t));
   indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType);
-  tIndexJsonSearch(index, mq, res);
+  indexJsonSearch(index, mq, res);
   indexMultiTermQueryDestroy(mq);
   *result = res;
 }
@@ -112,7 +112,7 @@ TEST_F(JsonEnv, testWrite) {
                                           colVal.c_str(), colVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -125,7 +125,7 @@ TEST_F(JsonEnv, testWrite) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -138,7 +138,7 @@ TEST_F(JsonEnv, testWrite) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -152,7 +152,7 @@ TEST_F(JsonEnv, testWrite) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(100, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -167,7 +167,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -182,7 +182,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
         SIndexMultiTerm* terms = indexMultiTermCreate();
         indexMultiTermAdd(terms, term);
-        tIndexJsonPut(index, terms, i);
+        indexJsonPut(index, terms, i);
         indexMultiTermDestroy(terms);
       }
     }
@@ -196,7 +196,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -210,7 +210,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(10, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -225,7 +225,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
       SArray* result = taosArrayInit(1, sizeof(uint64_t));
       indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
-      tIndexJsonSearch(index, mq, result);
+      indexJsonSearch(index, mq, result);
       EXPECT_EQ(0, taosArrayGetSize(result));
       indexMultiTermQueryDestroy(mq);
     }
@@ -240,7 +240,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
 
         SArray* result = taosArrayInit(1, sizeof(uint64_t));
         indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
-        tIndexJsonSearch(index, mq, result);
+        indexJsonSearch(index, mq, result);
         EXPECT_EQ(10, taosArrayGetSize(result));
         indexMultiTermQueryDestroy(mq);
       }
@@ -258,7 +258,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -271,7 +271,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -284,7 +284,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -297,7 +297,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -310,7 +310,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -324,7 +324,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -339,7 +339,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -354,7 +354,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -369,7 +369,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -385,7 +385,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -398,7 +398,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -412,7 +412,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_TERM);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -426,7 +426,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -441,7 +441,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -455,7 +455,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -469,7 +469,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(1000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -483,7 +483,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i);
+      indexJsonPut(index, terms, i);
       indexMultiTermDestroy(terms);
     }
   }
@@ -498,7 +498,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(0, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
@@ -511,7 +511,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
-      tIndexJsonPut(index, terms, i + 1000);
+      indexJsonPut(index, terms, i + 1000);
       indexMultiTermDestroy(terms);
     }
   }
@@ -526,7 +526,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) {
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
     indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
-    tIndexJsonSearch(index, mq, result);
+    indexJsonSearch(index, mq, result);
     EXPECT_EQ(2000, taosArrayGetSize(result));
     indexMultiTermQueryDestroy(mq);
   }
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index d73d85b90f..cfd711e6eb 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -307,6 +307,13 @@ static void uvHandleReq(SSvrConn* pConn) {
   if (pHead->noResp == 1) {
     transMsg.info.refId = -1;
   }
+
+  // set up conn info
+  SRpcConnInfo* pConnInfo = &(transMsg.info.connInfo);
+  pConnInfo->clientIp = (uint32_t)(pConn->addr.sin_addr.s_addr);
+  pConnInfo->clientPort = ntohs(pConn->addr.sin_port);
+  tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user));
+
   transReleaseExHandle(refMgt, pConn->refId);
 
   STrans* pTransInst = pConn->pTransInst;
@@ -1153,34 +1160,6 @@ _return2:
   rpcFreeCont(msg->pCont);
 }
 
-int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) {
-  if (thandle == NULL) {
-    tTrace("invalid handle %p, failed to Get Conn info", thandle);
-    return -1;
-  }
-  SRpcHandleInfo* pInfo = thandle;
-  SExHandle*      exh = pInfo->handle;
-  int64_t         refId = pInfo->refId;
-  ASYNC_CHECK_HANDLE(exh, refId);
-
-  // SExHandle* ex = thandle;
-  SSvrConn* pConn = exh->handle;
-  if (pConn == NULL) {
-    tTrace("invalid handle %p, failed to Get Conn info", thandle);
-    transReleaseExHandle(refMgt, refId);
-    return -1;
-  }
-  struct sockaddr_in addr = pConn->addr;
-  pConnInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);
-  pConnInfo->clientPort = ntohs(addr.sin_port);
-  tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user));
-  transReleaseExHandle(refMgt, refId);
-  return 0;
-_return1:
-  transReleaseExHandle(refMgt, refId);
-  return -1;
-_return2:
-  return -1;
-}
+int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; }
 
 #endif

From ba205a18e9bb3b79557c9bdec76212eed1c60d6c Mon Sep 17 00:00:00 2001
From: Huo Linhe <linhehuo@gmail.com>
Date: Thu, 9 Jun 2022 13:08:31 +0800
Subject: [PATCH 63/63] docs(grafana): update for one-liner installer script of
 grafana plugin

Ref: [TD-16379](https://jira.taosdata.com:18080/browse/TD-16379)
---
 docs-cn/20-third-party/01-grafana.mdx | 43 ++++++++++++++++++++-----
 docs-en/20-third-party/01-grafana.mdx | 45 ++++++++++++++++++++++-----
 2 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/docs-cn/20-third-party/01-grafana.mdx b/docs-cn/20-third-party/01-grafana.mdx
index 40b5c0ff4f..2daaebc32d 100644
--- a/docs-cn/20-third-party/01-grafana.mdx
+++ b/docs-cn/20-third-party/01-grafana.mdx
@@ -12,12 +12,38 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/
 - TDengine 集群已经部署并正常运行
 - taosAdapter 已经安装并正常运行。具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)
 
+记录以下信息：
+
+- TDengine 集群 REST API 地址，如：`http://tdengine.local:6041`。
+- TDengine 集群认证信息，可使用用户名及密码。
+
 ## 安装 Grafana
 
-目前 TDengine 支持 Grafana 7.0 以上的版本。用户可以根据当前的操作系统，到 Grafana 官网下载安装包，并执行安装。下载地址如下：<https://grafana.com/grafana/download>。
+目前 TDengine 支持 Grafana 7.5 以上的版本。用户可以根据当前的操作系统，到 Grafana 官网下载安装包，并执行安装。下载地址如下：<https://grafana.com/grafana/download>。
 
 ## 配置 Grafana
 
+### 安装方式一：安装脚本
+
+将集群信息设置为环境变量（也可以使用 [`.env`(dotenv) 文件](https://hexdocs.pm/dotenvy/dotenv-file-format.html) ）：
+
+```sh
+export TDENGINE_API=http://tdengine.local:6041
+# user + password
+export TDENGINE_USER=user
+export TDENGINE_PASSWORD=password
+```
+
+运行安装脚本：
+
+```sh
+bash -c "$(curl -fsSL https://raw.githubusercontent.com/taosdata/grafanaplugin/master/install.sh)"
+```
+
+该脚本将自动安装 Grafana 插件并配置数据源。安装完毕后，需要重启 Grafana 服务后生效。
+
+### 安装方式二：手动安装 TDengine 数据源插件
+
 使用 [`grafana-cli` 命令行工具](https://grafana.com/docs/grafana/latest/administration/cli/) 进行插件[安装](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation)。
 
 ```bash
@@ -48,11 +74,7 @@ sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/
 GF_INSTALL_PLUGINS=tdengine-datasource
 ```
 
-## 使用 Grafana
-
-### 配置数据源
-
-用户可以直接通过 <http://localhost:3000> 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示：
+之后，用户可以直接通过 <http://localhost:3000> 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示：
 
 ![TDengine Database Grafana plugin add data source](./add_datasource1.webp)
 
@@ -92,4 +114,11 @@ GF_INSTALL_PLUGINS=tdengine-datasource
 
 ### 导入 Dashboard
 
-在 2.3.3.0 及以上版本，您可以导入 TDinsight Dashboard (Grafana Dashboard ID： [15167](https://grafana.com/grafana/dashboards/15167)) 作为 TDengine 集群的监控可视化工具。安装和使用说明请见 [TDinsight 用户手册](/reference/tdinsight/)。
+在数据源配置页面，您可以为该数据源导入 TDinsight 面板，作为 TDengine 集群的监控可视化工具。该 Dashboard 已发布在 Grafana：[Dashboard 15167 - TDinsight](https://grafana.com/grafana/dashboards/15167)) 。其他安装方式和相关使用说明请见 [TDinsight 用户手册](/reference/tdinsight/)。
+
+使用 TDengine 作为数据源的其他面板，可以[在此搜索](https://grafana.com/grafana/dashboards/?dataSource=tdengine-datasource)。以下是一份不完全列表：
+
+- [15146](https://grafana.com/grafana/dashboards/15146): 监控多个 TDengine 集群
+- [15155](https://grafana.com/grafana/dashboards/15155): TDengine 告警示例
+- [15167](https://grafana.com/grafana/dashboards/15167): TDinsight
+- [16388](https://grafana.com/grafana/dashboards/16388): Telegraf 采集节点信息的数据展示
diff --git a/docs-en/20-third-party/01-grafana.mdx b/docs-en/20-third-party/01-grafana.mdx
index 1a84e02c66..1e1b49703e 100644
--- a/docs-en/20-third-party/01-grafana.mdx
+++ b/docs-en/20-third-party/01-grafana.mdx
@@ -14,12 +14,40 @@ In order for Grafana to add the TDengine data source successfully, the following
 1. The TDengine cluster is deployed and functioning properly
 2. taosAdapter is installed and running properly. Please refer to the taosAdapter manual for details.
 
+Record these values:
+
+- TDengine REST API url: `http://tdengine.local:6041`.
+- TDengine cluster authorization, with user + password.
+
 ## Installing Grafana
 
-TDengine currently supports Grafana versions 7.0 and above. Users can go to the Grafana official website to download the installation package and execute the installation according to the current operating system. The download address is as follows: <https://grafana.com/grafana/download>.
+TDengine currently supports Grafana versions 7.5 and above. Users can go to the Grafana official website to download the installation package and execute the installation according to the current operating system. The download address is as follows: <https://grafana.com/grafana/download>.
 
 ## Configuring Grafana
 
+### Option 1: Install with `install.sh`
+
+Set the url and authorization environment variables by `export` or a [`.env`(dotenv) file](https://hexdocs.pm/dotenvy/dotenv-file-format.html):
+
+```sh
+export TDENGINE_API=http://tdengine.local:6041
+# user + password
+export TDENGINE_USER=user
+export TDENGINE_PASSWORD=password
+```
+
+Run `install.sh`:
+
+```sh
+bash -c "$(curl -fsSL https://raw.githubusercontent.com/taosdata/grafanaplugin/master/install.sh)"
+```
+
+With this script, TDengine data source plugin and the Grafana data source will be installed and created automatically with Grafana provisioning configurations.
+
+And then, restart Grafana service and open Grafana in web-browser, usually <http://localhost:3000>.
+
+### Option 2: Install Plugin Manually
+
 Follow the installation steps in [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) with the [``grafana-cli`` command-line tool](https://grafana.com/docs/grafana/latest/administration/cli/) for plugin installation.
 
 ```bash
@@ -50,11 +78,7 @@ If Grafana is running in a Docker environment, the TDengine plugin can be automa
 GF_INSTALL_PLUGINS=tdengine-datasource
 ```
 
-## Using Grafana
-
-### Configuring Data Sources
-
-Users can log in to the Grafana server (username/password: admin/admin) directly through the URL `http://localhost:3000` and add a datasource through `Configuration -> Data Sources` on the left side, as shown in the following figure.
+Now users can log in to the Grafana server (username/password: admin/admin) directly through the URL `http://localhost:3000` and add a datasource through `Configuration -> Data Sources` on the left side, as shown in the following figure.
 
 ![TDengine Database TDinsight plugin add datasource 1](./grafana/add_datasource1.webp)
 
@@ -94,4 +118,11 @@ Follow the default prompt to query the average system memory usage for the speci
 
 ### Importing the Dashboard
 
-In version 2.3.3.0 and above, you can import the TDinsight Dashboard (Grafana Dashboard ID: [15168](https://grafana.com/grafana/dashboards/15167)) as a monitoring visualization tool for TDengine clusters. You can find installation and usage instructions in the TDinsight User Manual (/reference/tdinsight/).
+You can install TDinsight dashboard in data source configuration page (like `http://localhost:3000/datasources/edit/1/dashboards`) as a monitoring visualization tool for TDengine cluster. The dashboard is published in Grafana as [Dashboard 15167 - TDinsight](https://grafana.com/grafana/dashboards/15167). Check the [TDinsight User Manual](/reference/tdinsight/) for the details.
+
+For more dashboards using TDengine data source, [search here in Grafana](https://grafana.com/grafana/dashboards/?dataSource=tdengine-datasource). Here is a sub list:
+
+- [15146](https://grafana.com/grafana/dashboards/15146): Monitor multiple TDengine clusters.
+- [15155](https://grafana.com/grafana/dashboards/15155): TDengine alert demo.
+- [15167](https://grafana.com/grafana/dashboards/15167): TDinsight.
+- [16388](https://grafana.com/grafana/dashboards/16388): Telegraf node metrics dashboard using TDengine data source.