From 7cb3112d77fcfca0872e3ef6f4b4c768ab83f5a8 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 27 Oct 2022 15:52:48 +0800
Subject: [PATCH 01/42] refactor(sync): modify function name

---
 include/libs/sync/syncTools.h          |  4 ++--
 source/dnode/mnode/impl/src/mndMain.c  |  4 ++--
 source/dnode/vnode/src/vnd/vnodeSync.c |  4 ++--
 source/libs/sync/src/syncMain.c        | 27 +++++++++++++-------------
 source/libs/sync/src/syncReplication.c |  2 +-
 5 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index d5c015bfb2..1caf21b6a7 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -679,8 +679,8 @@ void syncReconfigFinishLog(const SyncReconfigFinish* pMsg);
 void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg);
 
 // on message ----------------------
-int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
-int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnPing(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReply(SSyncNode* ths, SyncPingReply* pMsg);
 int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg);
 
 int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg);
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index fbfa1b73be..94c4ba8c79 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -540,12 +540,12 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
 
   } else if (pMsg->msgType == TDMT_SYNC_PING) {
     SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
-    code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
+    code = syncNodeOnPing(pSyncNode, pSyncMsg);
     syncPingDestroy(pSyncMsg);
 
   } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
     SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
-    code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+    code = syncNodeOnPingReply(pSyncNode, pSyncMsg);
     syncPingReplyDestroy(pSyncMsg);
 
   } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index d3ae1015d0..7acf5b4003 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -301,13 +301,13 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   } else if (pMsg->msgType == TDMT_SYNC_PING) {
     SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
     ASSERT(pSyncMsg != NULL);
-    code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
+    code = syncNodeOnPing(pSyncNode, pSyncMsg);
     syncPingDestroy(pSyncMsg);
 
   } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
     SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
     ASSERT(pSyncMsg != NULL);
-    code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+    code = syncNodeOnPingReply(pSyncNode, pSyncMsg);
     syncPingReplyDestroy(pSyncMsg);
 
   } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index f1c900de50..bf76762757 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -49,8 +49,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths);
 static void    syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId);
 
 // process message ----
-int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
-int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnPing(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReply(SSyncNode* ths, SyncPingReply* pMsg);
 
 // ---------------------------------
 static void syncNodeFreeCb(void* param) {
@@ -1327,8 +1327,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   }
 
   // init callback
-  pSyncNode->FpOnPing = syncNodeOnPingCb;
-  pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb;
+  pSyncNode->FpOnPing = syncNodeOnPing;
+  pSyncNode->FpOnPingReply = syncNodeOnPingReply;
   pSyncNode->FpOnClientRequest = syncNodeOnClientRequest;
   pSyncNode->FpOnTimeout = syncNodeOnTimer;
   pSyncNode->FpOnSnapshot = syncNodeOnSnapshot;
@@ -3003,18 +3003,19 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 }
 
 // on message ----
-int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
-  // log state
+int32_t syncNodeOnPing(SSyncNode* ths, SyncPing* pMsg) {
+  sTrace("vgId:%d, recv sync-ping", ths->vgId);
+
   SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId, ths->vgId);
   SRpcMsg        rpcMsg;
   syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
 
   /*
-  // htonl
-  SMsgHead* pHead = rpcMsg.pCont;
-  pHead->contLen = htonl(pHead->contLen);
-  pHead->vgId = htonl(pHead->vgId);
-*/
+    // htonl
+    SMsgHead* pHead = rpcMsg.pCont;
+    pHead->contLen = htonl(pHead->contLen);
+    pHead->vgId = htonl(pHead->vgId);
+  */
 
   syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
   syncPingReplyDestroy(pMsgReply);
@@ -3022,9 +3023,9 @@ int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
   return 0;
 }
 
-int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
+int32_t syncNodeOnPingReply(SSyncNode* ths, SyncPingReply* pMsg) {
   int32_t ret = 0;
-  syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg);
+  sTrace("vgId:%d, recv sync-ping-reply", ths->vgId);
   return ret;
 }
 
diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c
index 7296e30e02..af53123421 100644
--- a/source/libs/sync/src/syncReplication.c
+++ b/source/libs/sync/src/syncReplication.c
@@ -57,7 +57,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) {
   SyncIndex logEndIndex = pSyncNode->pLogStore->syncLogEndIndex(pSyncNode->pLogStore);
   if (nextIndex < logStartIndex || nextIndex - 1 > logEndIndex) {
     char logBuf[128];
-    snprintf(logBuf, sizeof(logBuf), "start snapshot for next-index:%" PRId64 ", start:%" PRId64 ", end:%" PRId64,
+    snprintf(logBuf, sizeof(logBuf), "maybe start snapshot for next-index:%" PRId64 ", start:%" PRId64 ", end:%" PRId64,
              nextIndex, logStartIndex, logEndIndex);
     syncNodeEventLog(pSyncNode, logBuf);
 

From 04c06841495bc5859ebda47abe3e10c467484c1d Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 27 Oct 2022 16:21:54 +0800
Subject: [PATCH 02/42] refactor(sync): add SyncLocalCmd

---
 include/libs/sync/syncTools.h | 41 +++++++++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index 1caf21b6a7..b4eabfcbcd 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -678,24 +678,57 @@ void syncReconfigFinishPrint2(char* s, const SyncReconfigFinish* pMsg);
 void syncReconfigFinishLog(const SyncReconfigFinish* pMsg);
 void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncLocalCmd {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  int8_t   cmd;
+  SyncTerm sdNewTerm  // step down new term
+
+} SyncLocalCmd;
+
+SyncLocalCmd* syncLocalCmdBuild(uint32_t dataLen, int32_t vgId);
+void          syncLocalCmdDestroy(SyncLocalCmd* pMsg);
+void          syncLocalCmdSerialize(const SyncLocalCmd* pMsg, char* buf, uint32_t bufLen);
+void          syncLocalCmdDeserialize(const char* buf, uint32_t len, SyncLocalCmd* pMsg);
+char*         syncLocalCmdSerialize2(const SyncLocalCmd* pMsg, uint32_t* len);
+SyncLocalCmd* syncLocalCmdDeserialize2(const char* buf, uint32_t len);
+void          syncLocalCmd2RpcMsg(const SyncLocalCmd* pMsg, SRpcMsg* pRpcMsg);
+void          syncLocalCmdFromRpcMsg(const SRpcMsg* pRpcMsg, SyncLocalCmd* pMsg);
+SyncLocalCmd* syncLocalCmdFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*        syncLocalCmd2Json(const SyncLocalCmd* pMsg);
+char*         syncLocalCmd2Str(const SyncLocalCmd* pMsg);
+
+// for debug ----------------------
+void syncLocalCmdPrint(const SyncLocalCmd* pMsg);
+void syncLocalCmdPrint2(char* s, const SyncLocalCmd* pMsg);
+void syncLocalCmdLog(const SyncLocalCmd* pMsg);
+void syncLocalCmdLog2(char* s, const SyncLocalCmd* pMsg);
+
 // on message ----------------------
 int32_t syncNodeOnPing(SSyncNode* ths, SyncPing* pMsg);
 int32_t syncNodeOnPingReply(SSyncNode* ths, SyncPingReply* pMsg);
-int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg);
 
-int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg);
-int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg);
-int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex);
 int32_t syncNodeOnRequestVote(SSyncNode* ths, SyncRequestVote* pMsg);
 int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+
 int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg);
 int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
 int32_t syncNodeOnSnapshot(SSyncNode* ths, SyncSnapshotSend* pMsg);
 int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg);
 
 int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg);
 int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg);
 
+int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex);
+int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg);
+
 // -----------------------------------------
 typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg);
 typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg);

From 8b5d005db70f73dca410b6112cf0c7593b0ac65a Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 27 Oct 2022 17:16:46 +0800
Subject: [PATCH 03/42] refactor(sync): add SyncLocalCmd

---
 include/common/tmsgdef.h           |   1 +
 include/libs/sync/syncTools.h      |   8 +-
 source/libs/sync/src/syncMessage.c | 150 +++++++++++++++++++++++++++++
 3 files changed, 157 insertions(+), 2 deletions(-)

diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index be4bf0e4d2..571d14fe3c 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -269,6 +269,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL) // no longer used
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   
 #if defined(TD_MSG_NUMBER_)
diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index b4eabfcbcd..1c219f98bd 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -679,6 +679,10 @@ void syncReconfigFinishLog(const SyncReconfigFinish* pMsg);
 void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg);
 
 // ---------------------------------------------
+typedef enum {
+  SYNC_STEP_DOWN = 0,
+} ESyncLocalCmd;
+
 typedef struct SyncLocalCmd {
   uint32_t bytes;
   int32_t  vgId;
@@ -686,8 +690,8 @@ typedef struct SyncLocalCmd {
   SRaftId  srcId;
   SRaftId  destId;
 
-  int8_t   cmd;
-  SyncTerm sdNewTerm  // step down new term
+  int32_t  cmd;
+  SyncTerm sdNewTerm;  // step down new term
 
 } SyncLocalCmd;
 
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index 7a8ea0c4d7..b62c564fcd 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -3095,3 +3095,153 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg) {
     taosMemoryFree(serialized);
   }
 }
+
+// ---------------------------------------------
+SyncLocalCmd* syncLocalCmdBuild(uint32_t dataLen, int32_t vgId) {
+  uint32_t      bytes = sizeof(SyncLocalCmd);
+  SyncLocalCmd* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_LOCAL_CMD;
+  return pMsg;
+}
+
+void syncLocalCmdDestroy(SyncLocalCmd* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncLocalCmdSerialize(const SyncLocalCmd* pMsg, char* buf, uint32_t bufLen) {
+  ASSERT(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncLocalCmdDeserialize(const char* buf, uint32_t len, SyncLocalCmd* pMsg) {
+  memcpy(pMsg, buf, len);
+  ASSERT(len == pMsg->bytes);
+}
+
+char* syncLocalCmdSerialize2(const SyncLocalCmd* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  ASSERT(buf != NULL);
+  syncLocalCmdSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncLocalCmd* syncLocalCmdDeserialize2(const char* buf, uint32_t len) {
+  uint32_t      bytes = *((uint32_t*)buf);
+  SyncLocalCmd* pMsg = taosMemoryMalloc(bytes);
+  ASSERT(pMsg != NULL);
+  syncLocalCmdDeserialize(buf, len, pMsg);
+  ASSERT(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncLocalCmd2RpcMsg(const SyncLocalCmd* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncLocalCmdSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncLocalCmdFromRpcMsg(const SRpcMsg* pRpcMsg, SyncLocalCmd* pMsg) {
+  syncLocalCmdDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncLocalCmd* syncLocalCmdFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncLocalCmd* pMsg = syncLocalCmdDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  ASSERT(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncLocalCmd2Json(const SyncLocalCmd* pMsg) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    cJSON_AddNumberToObject(pRoot, "cmd", pMsg->cmd);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->sdNewTerm);
+    cJSON_AddStringToObject(pRoot, "sd-new-term", u64buf);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncSnapshotRsp", pRoot);
+  return pJson;
+}
+
+char* syncLocalCmd2Str(const SyncLocalCmd* pMsg) {
+  cJSON* pJson = syncLocalCmd2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// for debug ----------------------
+void syncLocalCmdPrint(const SyncLocalCmd* pMsg) {
+  char* serialized = syncLocalCmd2Str(pMsg);
+  printf("syncLocalCmdPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncLocalCmdPrint2(char* s, const SyncLocalCmd* pMsg) {
+  char* serialized = syncLocalCmd2Str(pMsg);
+  printf("syncLocalCmdPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncLocalCmdLog(const SyncLocalCmd* pMsg) {
+  char* serialized = syncLocalCmd2Str(pMsg);
+  sTrace("syncLocalCmdLog | len:%d | %s", (int32_t)strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncLocalCmdLog2(char* s, const SyncLocalCmd* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncLocalCmd2Str(pMsg);
+    sTrace("syncLocalCmdLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
\ No newline at end of file

From 54dfc8cd524b6e11ba07d2b19076dae2641f16fc Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 27 Oct 2022 17:38:33 +0800
Subject: [PATCH 04/42] enh(sync): add test

---
 source/libs/sync/test/CMakeLists.txt          |  14 +++
 .../test/syncConfigChangeSnapshotTest.cpp     |   2 +-
 .../libs/sync/test/syncConfigChangeTest.cpp   |   2 +-
 source/libs/sync/test/syncEncodeTest.cpp      |   2 +-
 source/libs/sync/test/syncIndexTest.cpp       |   2 +-
 source/libs/sync/test/syncLocalCmdTest.cpp    | 100 ++++++++++++++++++
 source/libs/sync/test/syncRaftIdCheck.cpp     |   2 +-
 source/libs/sync/test/syncRaftLogTest3.cpp    |  16 +--
 source/libs/sync/test/syncReplicateTest.cpp   |   2 +-
 source/libs/sync/test/syncRespMgrTest.cpp     |   6 +-
 source/libs/sync/test/syncTestTool.cpp        |   4 +-
 11 files changed, 133 insertions(+), 19 deletions(-)
 create mode 100644 source/libs/sync/test/syncLocalCmdTest.cpp

diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index b9cc7a391d..26dd32942b 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -59,6 +59,7 @@ add_executable(syncRestoreFromSnapshot "")
 add_executable(syncRaftCfgIndexTest "")
 add_executable(syncHeartbeatTest "")
 add_executable(syncHeartbeatReplyTest "")
+add_executable(syncLocalCmdTest "")
 
 
 target_sources(syncTest
@@ -305,6 +306,10 @@ target_sources(syncHeartbeatReplyTest
     PRIVATE
     "syncHeartbeatReplyTest.cpp"
 )
+target_sources(syncLocalCmdTest
+    PRIVATE
+    "syncLocalCmdTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -612,6 +617,11 @@ target_include_directories(syncHeartbeatReplyTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncLocalCmdTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -858,6 +868,10 @@ target_link_libraries(syncHeartbeatReplyTest
     sync
     gtest_main
 )
+target_link_libraries(syncLocalCmdTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp
index 95677e592b..8f16be27e7 100644
--- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp
+++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp
@@ -270,7 +270,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
   pMsg->msgType = 9999;
   pMsg->contLen = 256;
   pMsg->pCont = rpcMallocCont(pMsg->contLen);
-  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count,
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%" PRId64, myIndex, i, count,
            taosGetTimestampMs());
   return pMsg;
 }
diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp
index c04ab9b000..d1244546c9 100644
--- a/source/libs/sync/test/syncConfigChangeTest.cpp
+++ b/source/libs/sync/test/syncConfigChangeTest.cpp
@@ -191,7 +191,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
   pMsg->msgType = 9999;
   pMsg->contLen = 256;
   pMsg->pCont = rpcMallocCont(pMsg->contLen);
-  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count,
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%" PRId64, myIndex, i, count,
            taosGetTimestampMs());
   return pMsg;
 }
diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp
index 8b209c4c9e..9f1a81e7ed 100644
--- a/source/libs/sync/test/syncEncodeTest.cpp
+++ b/source/libs/sync/test/syncEncodeTest.cpp
@@ -186,7 +186,7 @@ int main(int argc, char **argv) {
   int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, pEntry->index, &pEntry);
   ASSERT(code == 0);
 
-  syncEntryLog2((char *)"==pEntry2==", pEntry2);
+  syncEntryLog2((char *)"==pEntry==", pEntry);
 
   // step5
   uint32_t len;
diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp
index 93cd5f79f0..763117c0c9 100644
--- a/source/libs/sync/test/syncIndexTest.cpp
+++ b/source/libs/sync/test/syncIndexTest.cpp
@@ -13,7 +13,7 @@ void print(SHashObj *pNextIndex) {
 
     SRaftId *pRaftId = (SRaftId *)key;
 
-    printf("key:<" PRIu64 ", %d>, value:%" PRIu64 " \n", pRaftId->addr, pRaftId->vgId, *p);
+    printf("key:<%" PRIu64 ", %d>, value:%" PRIu64 " \n", pRaftId->addr, pRaftId->vgId, *p);
     p = (uint64_t *)taosHashIterate(pNextIndex, p);
   }
 }
diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp
new file mode 100644
index 0000000000..256c13e267
--- /dev/null
+++ b/source/libs/sync/test/syncLocalCmdTest.cpp
@@ -0,0 +1,100 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncAppendEntries *createMsg() {
+  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000);
+  pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+  pMsg->srcId.vgId = 100;
+  pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+  pMsg->destId.vgId = 100;
+  pMsg->prevLogIndex = 11;
+  pMsg->prevLogTerm = 22;
+  pMsg->commitIndex = 33;
+  pMsg->privateTerm = 44;
+  strcpy(pMsg->data, "hello world");
+  return pMsg;
+}
+
+void test1() {
+  SyncAppendEntries *pMsg = createMsg();
+  syncAppendEntriesLog2((char *)"test1:", pMsg);
+  syncAppendEntriesDestroy(pMsg);
+}
+
+void test2() {
+  SyncAppendEntries *pMsg = createMsg();
+  uint32_t           len = pMsg->bytes;
+  char              *serialized = (char *)taosMemoryMalloc(len);
+  syncAppendEntriesSerialize(pMsg, serialized, len);
+  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen, 1000);
+  syncAppendEntriesDeserialize(serialized, len, pMsg2);
+  syncAppendEntriesLog2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncAppendEntriesDestroy(pMsg);
+  syncAppendEntriesDestroy(pMsg2);
+}
+
+void test3() {
+  SyncAppendEntries *pMsg = createMsg();
+  uint32_t           len;
+  char              *serialized = syncAppendEntriesSerialize2(pMsg, &len);
+  SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len);
+  syncAppendEntriesLog2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncAppendEntriesDestroy(pMsg);
+  syncAppendEntriesDestroy(pMsg2);
+}
+
+void test4() {
+  SyncAppendEntries *pMsg = createMsg();
+  SRpcMsg            rpcMsg;
+  syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
+  SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen);
+  syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2);
+  syncAppendEntriesLog2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncAppendEntriesDestroy(pMsg);
+  syncAppendEntriesDestroy(pMsg2);
+}
+
+void test5() {
+  SyncAppendEntries *pMsg = createMsg();
+  SRpcMsg            rpcMsg;
+  syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
+  SyncAppendEntries *pMsg2 = syncAppendEntriesFromRpcMsg2(&rpcMsg);
+  syncAppendEntriesLog2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncAppendEntriesDestroy(pMsg);
+  syncAppendEntriesDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+
+  return 0;
+}
diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp
index 65da0f6631..e7ef69da20 100644
--- a/source/libs/sync/test/syncRaftIdCheck.cpp
+++ b/source/libs/sync/test/syncRaftIdCheck.cpp
@@ -15,7 +15,7 @@ int main(int argc, char** argv) {
     char     host[128];
     uint16_t port;
     syncUtilU642Addr(u64, host, sizeof(host), &port);
-    printf("" PRIu64 " -> %s:%d \n", u64, host, port);
+    printf("%" PRIu64 " -> %s:%d \n", u64, host, port);
 
   } else if (argc == 3) {
     uint64_t u64;
diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp
index fd4cade31c..1171ef8785 100644
--- a/source/libs/sync/test/syncRaftLogTest3.cpp
+++ b/source/libs/sync/test/syncRaftLogTest3.cpp
@@ -97,8 +97,8 @@ void test1() {
   sTrace("lastIndex: %" PRId64, lastIndex);
   sTrace("lastTerm: %" PRIu64, lastTerm);
   sTrace("syncStartIndex: %" PRId64, syncStartIndex);
-  sTrace("" PRId64 "'s preIndex: %" PRId64, testIndex, preIndex);
-  sTrace("" PRId64 "'s preTerm: %" PRIu64, testIndex, preTerm);
+  sTrace("testIndex: %" PRId64 " preIndex: %" PRId64, testIndex, preIndex);
+  sTrace("testIndex: %" PRId64 " preTerm: %" PRIu64, testIndex, preTerm);
 
   if (gAssert) {
     assert(lastIndex == -1);
@@ -170,8 +170,8 @@ void test2() {
     SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i);
     SyncTerm  preTerm = syncNodeGetPreTerm(pSyncNode, i);
 
-    sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex);
-    sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm);
+    sTrace("i: %" PRId64 " preIndex: %" PRId64, i, preIndex);
+    sTrace("i: %" PRId64 " preTerm: %" PRIu64, i, preTerm);
 
     if (gAssert) {
       SyncIndex preIndexArr[12] = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
@@ -292,8 +292,8 @@ void test4() {
     SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i);
     SyncTerm  preTerm = syncNodeGetPreTerm(pSyncNode, i);
 
-    sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex);
-    sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm);
+    sTrace("i: %" PRId64 " preIndex: %" PRId64, i, preIndex);
+    sTrace("i: %" PRId64 " preTerm: %" PRIu64, i, preTerm);
   }
 
   logStoreDestory(pLogStore);
@@ -354,8 +354,8 @@ void test5() {
     SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i);
     SyncTerm  preTerm = syncNodeGetPreTerm(pSyncNode, i);
 
-    sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex);
-    sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm);
+    sTrace("i: %" PRId64 " preIndex: %" PRId64, i, preIndex);
+    sTrace("i: %" PRId64 " preTerm: %" PRIu64, i, preTerm);
 
     if (gAssert) {
       SyncIndex preIndexArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 5, 6, 7, 8, 9, 10};
diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp
index d3ba4bc136..adb3deb22d 100644
--- a/source/libs/sync/test/syncReplicateTest.cpp
+++ b/source/libs/sync/test/syncReplicateTest.cpp
@@ -145,7 +145,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
   pMsg->msgType = 9999;
   pMsg->contLen = 256;
   pMsg->pCont = rpcMallocCont(pMsg->contLen);
-  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count,
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%" PRId64, myIndex, i, count,
            taosGetTimestampMs());
   return pMsg;
 }
diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp
index 35daff796f..cad6eec91d 100644
--- a/source/libs/sync/test/syncRespMgrTest.cpp
+++ b/source/libs/sync/test/syncRespMgrTest.cpp
@@ -58,18 +58,18 @@ void syncRespMgrGetTest(uint64_t i) {
   if (ret == 1) {
     printStub(&stub);
   } else if (ret == 0) {
-    printf("" PRId64 " notFound \n", i);
+    printf("%" PRId64 " notFound \n", i);
   }
 }
 
 void syncRespMgrGetAndDelTest(uint64_t i) {
-  printf("------syncRespMgrGetAndDelTest-------" PRIu64 "-- \n", i);
+  printf("------syncRespMgrGetAndDelTest-------%" PRIu64 "-- \n", i);
   SRespStub stub;
   int32_t   ret = syncRespMgrGetAndDel(pMgr, i, &stub);
   if (ret == 1) {
     printStub(&stub);
   } else if (ret == 0) {
-    printf("" PRId64 " notFound \n", i);
+    printf("%" PRId64 " notFound \n", i);
   }
 }
 
diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp
index 1cdecfe5b3..bdb4d7d2d8 100644
--- a/source/libs/sync/test/syncTestTool.cpp
+++ b/source/libs/sync/test/syncTestTool.cpp
@@ -154,7 +154,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_
 void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb== pFsm:%p", pFsm); }
 
 void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta* cbMeta) {
-  char* s = syncCfg2Str(&(cbMeta.newCfg));
+  char* s = syncCfg2Str(&(cbMeta->newCfg));
   sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%" PRId64 ", code:%d, currentTerm:%" PRIu64 ", term:%" PRIu64
          ", newCfg:%s",
          cbMeta->flag, cbMeta->index, cbMeta->code, cbMeta->currentTerm, cbMeta->term, s);
@@ -308,7 +308,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
   pMsg->msgType = TDMT_VND_SUBMIT;
   pMsg->contLen = 256;
   pMsg->pCont = rpcMallocCont(pMsg->contLen);
-  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count,
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%" PRId64, myIndex, i, count,
            taosGetTimestampMs());
   return pMsg;
 }

From d7c866ee123a49b221cae2c5efd07fc8357a1191 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 27 Oct 2022 17:59:16 +0800
Subject: [PATCH 05/42] enh(sync): add test

---
 include/libs/sync/syncTools.h              |  4 +-
 source/libs/sync/src/syncMessage.c         |  4 +-
 source/libs/sync/test/syncLocalCmdTest.cpp | 72 +++++++++++-----------
 3 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index 1c219f98bd..eedc403493 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -680,7 +680,7 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg);
 
 // ---------------------------------------------
 typedef enum {
-  SYNC_STEP_DOWN = 0,
+  SYNC_LOCAL_CMD_STEP_DOWN = 100,
 } ESyncLocalCmd;
 
 typedef struct SyncLocalCmd {
@@ -695,7 +695,7 @@ typedef struct SyncLocalCmd {
 
 } SyncLocalCmd;
 
-SyncLocalCmd* syncLocalCmdBuild(uint32_t dataLen, int32_t vgId);
+SyncLocalCmd* syncLocalCmdBuild(int32_t vgId);
 void          syncLocalCmdDestroy(SyncLocalCmd* pMsg);
 void          syncLocalCmdSerialize(const SyncLocalCmd* pMsg, char* buf, uint32_t bufLen);
 void          syncLocalCmdDeserialize(const char* buf, uint32_t len, SyncLocalCmd* pMsg);
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index b62c564fcd..3c36633fe8 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -3097,7 +3097,7 @@ void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg) {
 }
 
 // ---------------------------------------------
-SyncLocalCmd* syncLocalCmdBuild(uint32_t dataLen, int32_t vgId) {
+SyncLocalCmd* syncLocalCmdBuild(int32_t vgId) {
   uint32_t      bytes = sizeof(SyncLocalCmd);
   SyncLocalCmd* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
@@ -3206,7 +3206,7 @@ cJSON* syncLocalCmd2Json(const SyncLocalCmd* pMsg) {
   }
 
   cJSON* pJson = cJSON_CreateObject();
-  cJSON_AddItemToObject(pJson, "SyncSnapshotRsp", pRoot);
+  cJSON_AddItemToObject(pJson, "SyncLocalCmd2Json", pRoot);
   return pJson;
 }
 
diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp
index 256c13e267..de908bf9c1 100644
--- a/source/libs/sync/test/syncLocalCmdTest.cpp
+++ b/source/libs/sync/test/syncLocalCmdTest.cpp
@@ -14,78 +14,78 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-SyncAppendEntries *createMsg() {
-  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000);
+SyncLocalCmd *createMsg() {
+  SyncLocalCmd *pMsg = syncLocalCmdBuild(1000);
   pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
   pMsg->destId.vgId = 100;
-  pMsg->prevLogIndex = 11;
-  pMsg->prevLogTerm = 22;
-  pMsg->commitIndex = 33;
-  pMsg->privateTerm = 44;
-  strcpy(pMsg->data, "hello world");
+  pMsg->sdNewTerm = 123;
+  pMsg->cmd = SYNC_LOCAL_CMD_STEP_DOWN;
+
   return pMsg;
 }
 
 void test1() {
-  SyncAppendEntries *pMsg = createMsg();
-  syncAppendEntriesLog2((char *)"test1:", pMsg);
-  syncAppendEntriesDestroy(pMsg);
+  SyncLocalCmd *pMsg = createMsg();
+  syncLocalCmdLog2((char *)"test1:", pMsg);
+  syncLocalCmdDestroy(pMsg);
 }
 
 void test2() {
-  SyncAppendEntries *pMsg = createMsg();
+  SyncLocalCmd *pMsg = createMsg();
   uint32_t           len = pMsg->bytes;
   char              *serialized = (char *)taosMemoryMalloc(len);
-  syncAppendEntriesSerialize(pMsg, serialized, len);
-  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen, 1000);
-  syncAppendEntriesDeserialize(serialized, len, pMsg2);
-  syncAppendEntriesLog2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
+  syncLocalCmdSerialize(pMsg, serialized, len);
+  SyncLocalCmd *pMsg2 = syncLocalCmdBuild(1000);
+  syncLocalCmdDeserialize(serialized, len, pMsg2);
+  syncLocalCmdLog2((char *)"test2: syncLocalCmdSerialize -> syncLocalCmdDeserialize ", pMsg2);
 
   taosMemoryFree(serialized);
-  syncAppendEntriesDestroy(pMsg);
-  syncAppendEntriesDestroy(pMsg2);
+  syncLocalCmdDestroy(pMsg);
+  syncLocalCmdDestroy(pMsg2);
 }
 
 void test3() {
-  SyncAppendEntries *pMsg = createMsg();
+  SyncLocalCmd *pMsg = createMsg();
   uint32_t           len;
-  char              *serialized = syncAppendEntriesSerialize2(pMsg, &len);
-  SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len);
-  syncAppendEntriesLog2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
+  char              *serialized = syncLocalCmdSerialize2(pMsg, &len);
+  SyncLocalCmd *pMsg2 = syncLocalCmdDeserialize2(serialized, len);
+  syncLocalCmdLog2((char *)"test3: syncLocalCmdSerialize3 -> syncLocalCmdDeserialize2 ", pMsg2);
 
   taosMemoryFree(serialized);
-  syncAppendEntriesDestroy(pMsg);
-  syncAppendEntriesDestroy(pMsg2);
+  syncLocalCmdDestroy(pMsg);
+  syncLocalCmdDestroy(pMsg2);
 }
 
 void test4() {
-  SyncAppendEntries *pMsg = createMsg();
+  SyncLocalCmd *pMsg = createMsg();
   SRpcMsg            rpcMsg;
-  syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
-  SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen);
-  syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2);
-  syncAppendEntriesLog2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
+  syncLocalCmd2RpcMsg(pMsg, &rpcMsg);
+  SyncLocalCmd *pMsg2 = (SyncLocalCmd *)taosMemoryMalloc(rpcMsg.contLen);
+  syncLocalCmdFromRpcMsg(&rpcMsg, pMsg2);
+  syncLocalCmdLog2((char *)"test4: syncLocalCmd2RpcMsg -> syncLocalCmdFromRpcMsg ", pMsg2);
 
   rpcFreeCont(rpcMsg.pCont);
-  syncAppendEntriesDestroy(pMsg);
-  syncAppendEntriesDestroy(pMsg2);
+  syncLocalCmdDestroy(pMsg);
+  syncLocalCmdDestroy(pMsg2);
 }
 
 void test5() {
-  SyncAppendEntries *pMsg = createMsg();
+  SyncLocalCmd *pMsg = createMsg();
   SRpcMsg            rpcMsg;
-  syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
-  SyncAppendEntries *pMsg2 = syncAppendEntriesFromRpcMsg2(&rpcMsg);
-  syncAppendEntriesLog2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
+  syncLocalCmd2RpcMsg(pMsg, &rpcMsg);
+  SyncLocalCmd *pMsg2 = syncLocalCmdFromRpcMsg2(&rpcMsg);
+  syncLocalCmdLog2((char *)"test5: syncLocalCmd2RpcMsg -> syncLocalCmdFromRpcMsg2 ", pMsg2);
 
   rpcFreeCont(rpcMsg.pCont);
-  syncAppendEntriesDestroy(pMsg);
-  syncAppendEntriesDestroy(pMsg2);
+  syncLocalCmdDestroy(pMsg);
+  syncLocalCmdDestroy(pMsg2);
 }
 
 int main() {
+  gRaftDetailLog = true;
+
   tsAsyncLog = 0;
   sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();

From 438786501b20560eaf8b0a4fe1428a943a7a5e38 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Fri, 28 Oct 2022 15:31:34 +0800
Subject: [PATCH 06/42] refactor(sync): adjust elect timer ms

---
 source/libs/sync/inc/syncEnv.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h
index 2a37e000e2..06da0eb3df 100644
--- a/source/libs/sync/inc/syncEnv.h
+++ b/source/libs/sync/inc/syncEnv.h
@@ -28,13 +28,13 @@ extern "C" {
 #include "trpc.h"
 #include "ttimer.h"
 
-#define TIMER_MAX_MS 0x7FFFFFFF
-#define ENV_TICK_TIMER_MS 1000
-#define PING_TIMER_MS 5000
-#define ELECT_TIMER_MS_MIN 5000
-#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
+#define TIMER_MAX_MS         0x7FFFFFFF
+#define ENV_TICK_TIMER_MS    1000
+#define PING_TIMER_MS        5000
+#define ELECT_TIMER_MS_MIN   2500
+#define ELECT_TIMER_MS_MAX   (ELECT_TIMER_MS_MIN * 2)
 #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
-#define HEARTBEAT_TIMER_MS 900
+#define HEARTBEAT_TIMER_MS   1000
 
 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})
 

From a7c2e0e6883f6bf9f80c33d1e078c501432bf102 Mon Sep 17 00:00:00 2001
From: huolibo <huolibo@qq.com>
Date: Fri, 28 Oct 2022 15:31:25 +0800
Subject: [PATCH 07/42] docs: what's updated on new version (2.0.41, 3.0.2)

---
 docs/en/14-reference/03-connector/04-java.mdx |   8 +
 docs/zh/08-connector/14-java.mdx              | 238 ++++++++++--------
 2 files changed, 136 insertions(+), 110 deletions(-)

diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx
index ecf632c624..abae520b43 100644
--- a/docs/en/14-reference/03-connector/04-java.mdx
+++ b/docs/en/14-reference/03-connector/04-java.mdx
@@ -878,7 +878,9 @@ The source code of the sample application is under `TDengine/examples/JDBC`:
 
 | taos-jdbcdriver version |                 major changes                  |
 | :---------------------: | :--------------------------------------------: |
+|       3.0.1 - 3.0.2        |  fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use 3.0.2 in the JDK 8 environment    |
 |       3.0.0        |   Support for TDengine 3.0    |
+|       2.0.41        |   fix decode method of username and password in REST connection |
 |     2.0.39 - 2.0.40     | Add REST connection/request timeout parameters |
 |         2.0.38          |  JDBC REST connections add bulk pull function  |
 |         2.0.37          |               Support json tags                |
@@ -910,6 +912,12 @@ The source code of the sample application is under `TDengine/examples/JDBC`:
 
   **Solution**: Use taos-jdbcdriver 2.* with your TDengine 2.* deployment.
 
+5. java.lang.NoSuchMethodError: java.nio.ByteBuffer.position(I)Ljava/nio/ByteBuffer; ... taos-jdbcdriver-3.0.1.jar
+
+**Cause**：taos-jdbcdriver 3.0.1 is compiled on JDK 11.
+
+**Solution**： Use taos-jdbcdriver 3.0.2.
+
 For additional troubleshooting, see [FAQ](../../../train-faq/faq).
 
 ## API Reference
diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx
index d13e21ad54..74ab629101 100644
--- a/docs/zh/08-connector/14-java.mdx
+++ b/docs/zh/08-connector/14-java.mdx
@@ -68,38 +68,39 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对
 ### 安装连接器
 
 <Tabs defaultValue="maven">
-<TabItem value="maven" label="使用 Maven 安装">
+  <TabItem value="maven" label="使用 Maven 安装">
 
-目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 仓库，且各大仓库都已同步。
+    目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver)
+    仓库，且各大仓库都已同步。
 
-- [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver)
-- [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver)
-- [maven.aliyun](https://maven.aliyun.com/mvn/search)
+    - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver)
+    - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver)
+    - [maven.aliyun](https://maven.aliyun.com/mvn/search)
 
-Maven 项目中，在 pom.xml 中添加以下依赖：
+    Maven 项目中，在 pom.xml 中添加以下依赖：
 
-```xml-dtd
-<dependency>
- <groupId>com.taosdata.jdbc</groupId>
- <artifactId>taos-jdbcdriver</artifactId>
- <version>3.0.0</version>
-</dependency>
-```
+    ```xml-dtd
+    <dependency>
+      <groupId>com.taosdata.jdbc</groupId>
+      <artifactId>taos-jdbcdriver</artifactId>
+      <version>3.0.0</version>
+    </dependency>
+    ```
 
-</TabItem>
-<TabItem value="source" label="使用源码编译安装">
+  </TabItem>
+  <TabItem value="source" label="使用源码编译安装">
 
-可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector
+    可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector
 
-```shell
-git clone https://github.com/taosdata/taos-connector-jdbc.git
-cd taos-connector-jdbc
-mvn clean install -Dmaven.test.skip=true
-```
+    ```shell
+    git clone https://github.com/taosdata/taos-connector-jdbc.git
+    cd taos-connector-jdbc
+    mvn clean install -Dmaven.test.skip=true
+    ```
 
-编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。
+    编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。
 
-</TabItem>
+  </TabItem>
 </Tabs>
 
 ## 建立连接
@@ -110,116 +111,125 @@ TDengine 的 JDBC URL 规范格式为：
 对于建立连接，原生连接与 REST 连接有细微不同。
 
 <Tabs defaultValue="rest">
-<TabItem value="native" label="原生连接">
+  <TabItem value="native" label="原生连接">
 
-```java
-Class.forName("com.taosdata.jdbc.TSDBDriver");
-String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
-Connection conn = DriverManager.getConnection(jdbcUrl);
-```
+    ```java
+    Class.forName("com.taosdata.jdbc.TSDBDriver");
+    String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
+    Connection conn = DriverManager.getConnection(jdbcUrl);
+    ```
 
-以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。
+    以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL
+    中指定用户名（user）为 root，密码（password）为 taosdata。
 
-**注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。
+    **注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。
 
-url 中的配置参数如下：
+    url 中的配置参数如下：
 
-- user：登录 TDengine 用户名，默认值 'root'。
-- password：用户登录密码，默认值 'taosdata'。
-- cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。
-- charset：客户端使用的字符集，默认值为系统字符集。
-- locale：客户端语言环境，默认值系统当前 locale。
-- timezone：客户端使用的时区，默认值为系统当前时区。
-- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。
-- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL 后的任何语句。默认值为：false。
+    - user：登录 TDengine 用户名，默认值 'root'。
+    - password：用户登录密码，默认值 'taosdata'。
+    - cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。
+    - charset：客户端使用的字符集，默认值为系统字符集。
+    - locale：客户端语言环境，默认值系统当前 locale。
+    - timezone：客户端使用的时区，默认值为系统当前时区。
+    - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。
+    - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL
+    后的任何语句。默认值为：false。
 
-JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。
+    JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。
 
-**使用 TDengine 客户端驱动配置文件建立连接 **
+    **使用 TDengine 客户端驱动配置文件建立连接 **
 
-当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示：
+    当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示：
 
-1. 在 Java 应用中不指定 hostname 和 port
+    1. 在 Java 应用中不指定 hostname 和 port
 
-```java
-public Connection getConn() throws Exception{
-  Class.forName("com.taosdata.jdbc.TSDBDriver");
-  String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
-  Properties connProps = new Properties();
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
-  connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-  Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
-  return conn;
-}
-```
+    ```java
+    public Connection getConn() throws Exception{
+    Class.forName("com.taosdata.jdbc.TSDBDriver");
+    String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
+    Properties connProps = new Properties();
+    connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+    connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+    connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+    Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+    return conn;
+  }
+    ```
 
-2. 在配置文件中指定 firstEp 和 secondEp
+    2. 在配置文件中指定 firstEp 和 secondEp
 
-```shell
-# first fully qualified domain name (FQDN) for TDengine system
-firstEp               cluster_node1:6030
+    ```shell
+    # first fully qualified domain name (FQDN) for TDengine system
+    firstEp cluster_node1:6030
 
-# second fully qualified domain name (FQDN) for TDengine system, for cluster only
-secondEp              cluster_node2:6030
+    # second fully qualified domain name (FQDN) for TDengine system, for cluster only
+    secondEp cluster_node2:6030
 
-# default system charset
-# charset               UTF-8
+    # default system charset
+    # charset UTF-8
 
-# system locale
-# locale                en_US.UTF-8
-```
+    # system locale
+    # locale en_US.UTF-8
+    ```
 
-以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。
+    以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp
+    连接集群。
 
-TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。
+    TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。
 
-> **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
+    > **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值
+    C://TDengine/cfg/taos.cfg。
 
-</TabItem>
-<TabItem value="rest" label="REST 连接">
+  </TabItem>
+  <TabItem value="rest" label="REST 连接">
 
-```java
-Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
-String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata";
-Connection conn = DriverManager.getConnection(jdbcUrl);
-```
+    ```java
+    Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
+    String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata";
+    Connection conn = DriverManager.getConnection(jdbcUrl);
+    ```
 
-以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。
+    以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为
+    root，密码（password）为 taosdata。
 
-使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要：
+    使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要：
 
-1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”；
-2. jdbcUrl 以“jdbc:TAOS-RS://”开头；
-3. 使用 6041 作为连接端口。
+    1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”；
+    2. jdbcUrl 以“jdbc:TAOS-RS://”开头；
+    3. 使用 6041 作为连接端口。
 
-url 中的配置参数如下：
+    url 中的配置参数如下：
 
-- user：登录 TDengine 用户名，默认值 'root'。
-- password：用户登录密码，默认值 'taosdata'。
-- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。
-- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。
-- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。
-- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。
-- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。
-- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。
-- useSSL: 连接中是否使用 SSL。
+    - user：登录 TDengine 用户名，默认值 'root'。
+    - password：用户登录密码，默认值 'taosdata'。
+    - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST
+    连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。
+    - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。
+    - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL
+    后的任何语句。默认值为：false。
+    - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。
+    - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。
+    - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。
+    - useSSL: 连接中是否使用 SSL。
 
-**注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。
+    **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。
 
-:::note
+    :::note
 
-- 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如：
+    - 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如：
 
-```sql
-INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6);
-```
+    ```sql
+    INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6);
+    ```
 
-- 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6);
+    - 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为
+    jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature)
+    tags('California.SanFrancisco') values(now, 24.6);
 
-:::
+    :::
 
-</TabItem>
+  </TabItem>
 </Tabs>
 
 ### 指定 URL 和 Properties 获取连接
@@ -271,7 +281,7 @@ properties 中的配置参数如下：
 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
 - TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。
-  此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。
+此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。
 
 ### 配置参数的优先级
 
@@ -880,7 +890,9 @@ public static void main(String[] args) throws Exception {
 
 | taos-jdbcdriver 版本 |            主要变化            |
 | :------------------: | :----------------------------: |
+|       3.0.1 - 3.0.2        |  修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用 3.0.2 版本    |
 |       3.0.0        |   支持 TDengine 3.0    |
+|       2.0.41        |   修正 REST 连接中用户名和密码转码方式    |
 |   2.0.39 - 2.0.40    |  增加 REST 连接/请求 超时设置  |
 |        2.0.38        | JDBC REST 连接增加批量拉取功能 |
 |        2.0.37        |      增加对 json tag 支持      |
@@ -890,27 +902,33 @@ public static void main(String[] args) throws Exception {
 
 1. 使用 Statement 的 `addBatch()` 和 `executeBatch()` 来执行“批量写入/更新”，为什么没有带来性能上的提升？
 
-   **原因**：TDengine 的 JDBC 实现中，通过 `addBatch` 方法提交的 SQL 语句，会按照添加的顺序，依次执行，这种方式没有减少与服务端的交互次数，不会带来性能上的提升。
+**原因**：TDengine 的 JDBC 实现中，通过 `addBatch` 方法提交的 SQL 语句，会按照添加的顺序，依次执行，这种方式没有减少与服务端的交互次数，不会带来性能上的提升。
 
-   **解决方法**：1. 在一条 insert 语句中拼接多个 values 值；2. 使用多线程的方式并发插入；3. 使用参数绑定的写入方式
+**解决方法**：1. 在一条 insert 语句中拼接多个 values 值；2. 使用多线程的方式并发插入；3. 使用参数绑定的写入方式
 
 2. java.lang.UnsatisfiedLinkError: no taos in java.library.path
 
-   **原因**：程序没有找到依赖的本地函数库 taos。
+**原因**：程序没有找到依赖的本地函数库 taos。
 
-   **解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib`。
+**解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib`。
 
 3. java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform
 
-   **原因**：目前 TDengine 只支持 64 位 JDK。
+**原因**：目前 TDengine 只支持 64 位 JDK。
 
-   **解决方法**：重新安装 64 位 JDK。
+**解决方法**：重新安装 64 位 JDK。
 
 4. java.lang.NoSuchMethodError: setByteArray
 
-  **原因**：taos-jdbcdriver 3.* 版本仅支持 TDengine 3.0 及以上版本。
+**原因**：taos-jdbcdriver 3.* 版本仅支持 TDengine 3.0 及以上版本。
 
-  **解决方法**： 使用 taos-jdbcdriver 2.* 版本连接 TDengine 2.* 版本。
+**解决方法**： 使用 taos-jdbcdriver 2.* 版本连接 TDengine 2.* 版本。
+
+5. java.lang.NoSuchMethodError: java.nio.ByteBuffer.position(I)Ljava/nio/ByteBuffer; ... taos-jdbcdriver-3.0.1.jar
+
+**原因**：taos-jdbcdriver 3.0.1 版本需要在 JDK 11+ 环境使用。
+
+**解决方法**： 更换 taos-jdbcdriver 3.0.2 版本。
 
 其它问题请参考 [FAQ](../../../train-faq/faq)
 

From d625cb56b7a5936dfc0d7459239aaf33f2a30dce Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Fri, 28 Oct 2022 15:47:55 +0800
Subject: [PATCH 08/42] docs: fix few typos

---
 docs/en/14-reference/03-connector/04-java.mdx | 8 ++++----
 docs/zh/08-connector/14-java.mdx              | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx
index abae520b43..7b3440ebac 100644
--- a/docs/en/14-reference/03-connector/04-java.mdx
+++ b/docs/en/14-reference/03-connector/04-java.mdx
@@ -201,7 +201,7 @@ The configuration parameters in the URL are as follows:
 - httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms.
 - httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false.
 - messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true.
-- useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection.
+- useSSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection.
 
 **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection.
 
@@ -268,7 +268,7 @@ The configuration parameters in properties are as follows.
 - TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection.
 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false.
 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
-- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. It only takes effect when using JDBC REST connection.
+- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection.
   For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only).
 
 ### Priority of configuration parameters
@@ -824,7 +824,7 @@ Example usage is as follows.
     //query or insert
     // ...
 
-    connection.close(); // put back to conneciton pool
+    connection.close(); // put back to connection pool
 }
 ```
 
@@ -856,7 +856,7 @@ public static void main(String[] args) throws Exception {
     //query or insert
     // ...
 
-    connection.close(); // put back to conneciton pool
+    connection.close(); // put back to connection pool
 }
 ```
 
diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx
index 74ab629101..1ee59d2df4 100644
--- a/docs/zh/08-connector/14-java.mdx
+++ b/docs/zh/08-connector/14-java.mdx
@@ -836,7 +836,7 @@ public abstract class ConsumerLoop {
     //query or insert
     // ...
 
-    connection.close(); // put back to conneciton pool
+    connection.close(); // put back to connection pool
 }
 ```
 
@@ -868,7 +868,7 @@ public static void main(String[] args) throws Exception {
     //query or insert
     // ...
 
-    connection.close(); // put back to conneciton pool
+    connection.close(); // put back to connection pool
 }
 ```
 

From 030195e72ac35b105545abdf4399fc6a78389405 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 28 Oct 2022 16:42:58 +0800
Subject: [PATCH 09/42] support macos

---
 source/libs/transport/src/transSvr.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 6819068b64..025eab3f3e 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -655,7 +655,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) {
   uv_tcp_init(pObj->loop, cli);
 
   if (uv_accept(stream, (uv_stream_t*)cli) == 0) {
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(DARWIN)
     if (pObj->numOfWorkerReady < pObj->numOfThreads) {
       tError("worker-threads are not ready for all, need %d instead of %d.", pObj->numOfThreads,
              pObj->numOfWorkerReady);
@@ -779,7 +779,7 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) {
     return false;
   }
 
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(DARWIN)
   uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
 #else
   uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
@@ -799,7 +799,7 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) {
   QUEUE_INIT(&pThrd->conn);
 
   pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 5, pThrd, uvWorkerAsyncCb);
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(DARWIN)
   uv_pipe_connect(&pThrd->connect_req, pThrd->pipe, pipeName, uvOnPipeConnectionCb);
 #else
   uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
@@ -976,17 +976,18 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
   uv_loop_init(srv->loop);
 
   char pipeName[PATH_MAX];
-#ifdef WINDOWS
+#if defined(WINDOWS) || defined(DARWIN)
   int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0);
   if (ret != 0) {
     tError("failed to init pipe, errmsg: %s", uv_err_name(ret));
     goto End;
   }
-
+#if defined(WINDOWS)
   snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%d-%" PRIu64, taosSafeRand(), GetCurrentProcessId());
-  // char pipeName[PATH_MAX] = {0};
-  // snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08d-%" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(),
-  //          taosGetSelfPthreadId());
+#elif defined(DARWIN)
+  snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08d-%" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(),
+           taosGetSelfPthreadId());
+#endif
 
   ret = uv_pipe_bind(&srv->pipeListen, pipeName);
   if (ret != 0) {
@@ -1059,8 +1060,8 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
       goto End;
     }
   }
-
 #endif
+
   if (false == taosValidIpAndPort(srv->ip, srv->port)) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr());

From 6c19e886df8b44f97cdc95f13adb3f270cad3388 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 17:58:59 +0800
Subject: [PATCH 10/42] fix(query): move the init function for different
 reader.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index f83755fc4f..5e5e287cf9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3601,16 +3601,6 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
-
-      code = doOpenReaderImpl(pNextReader);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      }
-
-      code = doOpenReaderImpl(pReader);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      }
     }
   }
 
@@ -3741,9 +3731,14 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
 
   if (pReader->innerReader[0] != NULL && pReader->step == 0) {
     bool ret = doTsdbNextDataBlock(pReader->innerReader[0]);
-    resetDataBlockScanInfo(pReader->innerReader[0]->status.pTableMap, pReader->innerReader[0]->window.ekey);
-    pReader->step = EXTERNAL_ROWS_PREV;
 
+    // prepare for the main scan
+    int32_t code = doOpenReaderImpl(pReader);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    pReader->step = EXTERNAL_ROWS_PREV;
     if (ret) {
       return ret;
     }
@@ -3759,7 +3754,12 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   }
 
   if (pReader->innerReader[1] != NULL && pReader->step == EXTERNAL_ROWS_MAIN) {
-    resetDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey);
+    // prepare for the next row scan
+    int32_t code = doOpenReaderImpl(pReader->innerReader[1]);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
     bool ret1 = doTsdbNextDataBlock(pReader->innerReader[1]);
     pReader->step = EXTERNAL_ROWS_NEXT;
     if (ret1) {

From 4a01bc53c7b8e8d891932d4c23958cd7503b8a16 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 28 Oct 2022 18:29:59 +0800
Subject: [PATCH 11/42] rm log

---
 source/libs/index/src/indexTfile.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 4fdf6d9e57..e002ff9c32 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -617,8 +617,6 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
       indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
                  (int)taosArrayGetSize(v->tableId));
     } else {
-      indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
-                (int)taosArrayGetSize(v->tableId));
     }
   }
   fstBuilderDestroy(tw->fb);

From 9ccf7addb3200732b3bef7d6e0d23430c8ddcd9b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 18:46:08 +0800
Subject: [PATCH 12/42] other:merge 3.0

---
 source/dnode/vnode/src/sma/smaRollup.c  |  10 +-
 source/libs/executor/inc/executorimpl.h |   1 +
 source/libs/executor/src/scanoperator.c | 269 +++++++++++++++++++++++-
 3 files changed, 271 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 19a0fbd629..af21ba83b7 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -701,6 +701,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
 #endif
     for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
       SSDataBlock *output = taosArrayGetP(pResList, i);
+      smaError("uid:%ld, groupid:%ld", output->info.uid, output->info.groupId);
+
       STsdb       *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
       SSubmitReq  *pReq = NULL;
 
@@ -713,13 +715,13 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
 
       if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
         taosMemoryFreeClear(pReq);
-        smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s",
-                 SMA_VID(pSma), suid, pItem->level, terrstr());
+        smaError("vgId:%d, process submit req for rsma suid:%"PRIu64", uid:%" PRIu64 " level %" PRIi8 " failed since %s",
+                 SMA_VID(pSma), suid, output->info.groupId, pItem->level, terrstr());
         goto _err;
       }
 
-      smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
-               SMA_VID(pSma), suid, pItem->level, output->info.version, htonl(pReq->header.contLen));
+      smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%"PRIu64", level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
+               SMA_VID(pSma), suid, output->info.groupId, pItem->level, output->info.version, htonl(pReq->header.contLen));
 
       taosMemoryFreeClear(pReq);
     }
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 975f8bb162..fc9524e828 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -371,6 +371,7 @@ typedef struct STableMergeScanInfo {
   SQueryTableDataCond    cond;
   int32_t                scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
   int32_t                dataBlockLoadFlag;
+  STsdbReader*           pReader;
 
   // if the upstream is an interval operator, the interval info is also kept here to get the time
   // window to check if current data block needs to be loaded.
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index e8a67cdc35..fec9217bdb 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -621,12 +621,14 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
     binfo.capacity = binfo.rows;
     blockDataEnsureCapacity(pBlock, binfo.rows);
     pBlock->info = binfo;
-    ASSERT(binfo.uid != 0);
 
-    uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
+    ASSERT(binfo.uid != 0);
+    pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid);
+    ASSERT(pBlock->info.groupId != 0);
+//    uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
+//    if (groupId) {
+//      pBlock->info.groupId = *groupId;
+//    }
 
     uint32_t status = 0;
     int32_t  code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
@@ -765,6 +767,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
     SSDataBlock* result = doTableScanGroup(pOperator);
     if (result != NULL) {
+      ASSERT(result->info.uid != 0);
       return result;
     }
 
@@ -4189,6 +4192,130 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
+                                   STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
+                                   STsdbReader** ppReader, const char* idstr) {
+  STsdbReader* pReader = NULL;
+  void* pStart = taosArrayGet(pTableListInfo->pTableList, tableStartIdx);
+  int32_t num = tableEndIdx - tableStartIdx + 1;
+
+  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr);
+  if (code != 0) {
+    return code;
+  }
+
+  *ppReader = pReader;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
+                                          SSDataBlock* pBlock, uint32_t* status) {
+  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
+  STableMergeScanInfo* pInfo = pOperator->info;
+
+  uint64_t uid = pBlock->info.uid;
+
+  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
+
+  pCost->totalBlocks += 1;
+  pCost->totalRows += pBlock->info.rows;
+
+  *status = pInfo->dataBlockLoadFlag;
+  if (pTableScanInfo->pFilterNode != NULL ||
+      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
+    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
+  }
+
+  SDataBlockInfo* pBlockInfo = &pBlock->info;
+  taosMemoryFreeClear(pBlock->pBlockAgg);
+
+  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->filterOutBlocks += 1;
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
+    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->skipBlocks += 1;
+
+    // clear all data in pBlock that are set when handing the previous block
+    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
+      pcol->pData = NULL;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
+    pCost->loadBlockStatis += 1;
+
+    bool             allColumnsHaveAgg = true;
+    SColumnDataAgg** pColAgg = NULL;
+    STsdbReader*     reader = pTableScanInfo->pReader;
+    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
+
+    if (allColumnsHaveAgg == true) {
+      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+
+      // todo create this buffer during creating operator
+      if (pBlock->pBlockAgg == NULL) {
+        pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
+      }
+
+      for (int32_t i = 0; i < numOfCols; ++i) {
+        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
+        if (!pColMatchInfo->needOutput) {
+          continue;
+        }
+        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
+      }
+
+      return TSDB_CODE_SUCCESS;
+    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
+      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
+    }
+  }
+
+  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
+
+  pCost->totalCheckedRows += pBlock->info.rows;
+  pCost->loadBlocks += 1;
+
+  STsdbReader* reader = pTableScanInfo->pReader;
+  SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
+  if (pCols == NULL) {
+    return terrno;
+  }
+
+  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
+
+  // currently only the tbname pseudo column
+  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
+                                          pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo));
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pTaskInfo->env, code);
+    }
+  }
+
+  if (pTableScanInfo->pFilterNode != NULL) {
+    int64_t st = taosGetTimestampMs();
+    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
+
+    double el = (taosGetTimestampUs() - st) / 1000.0;
+    pTableScanInfo->readRecorder.filterTime += el;
+
+    if (pBlock->info.rows == 0) {
+      pCost->filterOutBlocks += 1;
+      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
+             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
+    } else {
+      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -4314,6 +4441,138 @@ typedef struct STableMergeScanSortSourceParam {
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
+static SSDataBlock* getTableDataBlockTemp(void* param) {
+  STableMergeScanSortSourceParam* source = param;
+  SOperatorInfo*                  pOperator = source->pOperator;
+  STableMergeScanInfo*            pInfo = pOperator->info;
+  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  int32_t                         readIdx = source->readerIdx;
+  SSDataBlock*                    pBlock = source->inputBlock;
+  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
+
+  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
+
+  blockDataCleanup(pBlock);
+
+  int64_t st = taosGetTimestampUs();
+
+  void* pStart = taosArrayGet(pInfo->tableListInfo->pTableList, readIdx + pInfo->tableStartIndex);
+
+  SReadHandle* pHandle = &pInfo->readHandle;
+  tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
+
+  STsdbReader* reader = pInfo->pReader;
+  while (tsdbNextDataBlock(reader)) {
+    if (isTaskKilled(pOperator->pTaskInfo)) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+    }
+
+    // process this data block based on the probabilities
+    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    if (!processThisBlock) {
+      continue;
+    }
+
+    blockDataCleanup(pBlock);
+    SDataBlockInfo binfo = pBlock->info;
+    tsdbRetrieveDataBlockInfo(reader, &binfo);
+
+    blockDataEnsureCapacity(pBlock, binfo.rows);
+    pBlock->info.type = binfo.type;
+    pBlock->info.uid = binfo.uid;
+    pBlock->info.window = binfo.window;
+    pBlock->info.rows = binfo.rows;
+
+    if (tsdbIsAscendingOrder(pInfo->pReader)) {
+      pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
+    } else {
+      pQueryCond->twindows.ekey = pBlock->info.window.skey - 1;
+    }
+
+    uint32_t status = 0;
+    int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readIdx, pBlock, &status);
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, code);
+    }
+
+    // current block is filter out according to filter condition, continue load the next block
+    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
+      continue;
+    }
+
+    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
+    if (groupId) {
+      pBlock->info.groupId = *groupId;
+    }
+
+    pOperator->resultInfo.totalRows += pBlock->info.rows;  // pTableScanInfo->readRecorder.totalRows;
+    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+
+    tsdbReaderClose(pInfo->pReader);
+    pInfo->pReader = NULL;
+    return pBlock;
+  }
+  tsdbReaderClose(pInfo->pReader);
+  pInfo->pReader = NULL;
+  return NULL;
+}
+static SSDataBlock* getTableDataBlock2(void* param) {
+  STableMergeScanSortSourceParam* source = param;
+  SOperatorInfo*                  pOperator = source->pOperator;
+  int64_t                         uid = source->uid;
+  SSDataBlock*                    pBlock = source->inputBlock;
+  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
+
+  int64_t st = taosGetTimestampUs();
+
+  blockDataCleanup(pBlock);
+
+  STsdbReader* reader = pTableScanInfo->pReader;
+  while (tsdbTableNextDataBlock(reader, uid)) {
+    if (isTaskKilled(pOperator->pTaskInfo)) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+    }
+
+    // process this data block based on the probabilities
+    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    if (!processThisBlock) {
+      continue;
+    }
+
+    blockDataCleanup(pBlock);
+    SDataBlockInfo binfo = pBlock->info;
+    tsdbRetrieveDataBlockInfo(reader, &binfo);
+
+    blockDataEnsureCapacity(pBlock, binfo.rows);
+    pBlock->info.type = binfo.type;
+    pBlock->info.uid = binfo.uid;
+    pBlock->info.window = binfo.window;
+    pBlock->info.rows = binfo.rows;
+
+    uint32_t status = 0;
+    int32_t  code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, code);
+    }
+
+    // current block is filter out according to filter condition, continue load the next block
+    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
+      continue;
+    }
+
+    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
+    if (groupId) {
+      pBlock->info.groupId = *groupId;
+    }
+
+    pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
+    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+
+    return pBlock;
+  }
+  return NULL;
+}
+
 static SSDataBlock* getTableDataBlock(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;

From df786f1516cfb3692694e5cf4e2027bc49be06ff Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 28 Oct 2022 19:27:06 +0800
Subject: [PATCH 13/42] use sys mem

---
 source/libs/transport/src/transSvr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 025eab3f3e..b829251ccd 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -1037,7 +1037,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
     srv->pThreadObj[i] = thrd;
 
     uv_os_sock_t fds[2];
-    if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) {
+    if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) {
       goto End;
     }
 

From 335cb6513edc8044e31efaf6e19218b82885ba8c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:32:57 +0800
Subject: [PATCH 14/42] other:merge 3.0

---
 source/dnode/vnode/src/sma/smaRollup.c        |  1 +
 source/libs/executor/inc/executorimpl.h       | 33 ++++++++++---------
 source/libs/executor/src/executil.c           |  7 +++-
 source/libs/executor/src/scanoperator.c       |  9 +++--
 source/libs/executor/src/timewindowoperator.c |  2 ++
 5 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index af21ba83b7..0ed9ec210f 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -671,6 +671,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
   while (1) {
     uint64_t ts;
     bool     hasMore = false;
+//    ASSERT(0);
     int32_t  code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL);
     if (code < 0) {
       if (code == TSDB_CODE_QRY_IN_EXEC) {
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index fc9524e828..50c62b9017 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -340,21 +340,23 @@ typedef struct STableScanInfo {
 } STableScanInfo;
 
 typedef struct STableMergeScanInfo {
-  STableListInfo* tableListInfo;
-  int32_t         tableStartIndex;
-  int32_t         tableEndIndex;
-  bool            hasGroupId;
-  uint64_t        groupId;
-  SArray*         dataReaders;  // array of tsdbReaderT*
-  SReadHandle     readHandle;
-  int32_t         bufPageSize;
-  uint32_t        sortBufSize;  // max buffer size for in-memory sort
-  SArray*         pSortInfo;
-  SSortHandle*    pSortHandle;
-  SSDataBlock*    pSortInputBlock;
-  int64_t         startTs;  // sort start time
-  SArray*         sortSourceParams;
-  SLimitInfo      limitInfo;
+  STableListInfo*        tableListInfo;
+  int32_t                tableStartIndex;
+  int32_t                tableEndIndex;
+  bool                   hasGroupId;
+  uint64_t               groupId;
+  SArray*                dataReaders;  // array of tsdbReaderT*
+  SArray*                queryConds;   // array of queryTableDataCond
+  STsdbReader*           pReader;
+  SReadHandle            readHandle;
+  int32_t                bufPageSize;
+  uint32_t               sortBufSize;  // max buffer size for in-memory sort
+  SArray*                pSortInfo;
+  SSortHandle*           pSortHandle;
+  SSDataBlock*           pSortInputBlock;
+  int64_t                startTs;  // sort start time
+  SArray*                sortSourceParams;
+  SLimitInfo             limitInfo;
   SFileBlockLoadRecorder readRecorder;
   int64_t                numOfRows;
   SScanInfo              scanInfo;
@@ -371,7 +373,6 @@ typedef struct STableMergeScanInfo {
   SQueryTableDataCond    cond;
   int32_t                scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
   int32_t                dataBlockLoadFlag;
-  STsdbReader*           pReader;
 
   // if the upstream is an interval operator, the interval info is also kept here to get the time
   // window to check if current data block needs to be loaded.
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index dbef032041..56158e567e 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1681,9 +1681,14 @@ int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t
   STableKeyInfo keyInfo = {.uid = uid, .groupId = gid};
 
   taosArrayPush(pTableList->pTableList, &keyInfo);
-  if (pTableList->oneTableForEachGroup || pTableList->numOfOuputGroups > 1) {
+  if (!pTableList->oneTableForEachGroup) {
+    if (pTableList->map == NULL) {
+      pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+    }
+
     taosHashPut(pTableList->map, &uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
   }
+
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index fec9217bdb..32d2684a0a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -624,11 +624,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
 
     ASSERT(binfo.uid != 0);
     pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid);
-    ASSERT(pBlock->info.groupId != 0);
-//    uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-//    if (groupId) {
-//      pBlock->info.groupId = *groupId;
-//    }
 
     uint32_t status = 0;
     int32_t  code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
@@ -1570,6 +1565,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   if (groupIdPre) {
     pInfo->pRes->info.groupId = *groupIdPre;
   } else {
+    ASSERT(0);
     pInfo->pRes->info.groupId = 0;
   }
 
@@ -4438,6 +4434,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 typedef struct STableMergeScanSortSourceParam {
   SOperatorInfo* pOperator;
   int32_t        readerIdx;
+  int64_t        uid;
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
@@ -4514,8 +4511,10 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
   }
   tsdbReaderClose(pInfo->pReader);
   pInfo->pReader = NULL;
+
   return NULL;
 }
+
 static SSDataBlock* getTableDataBlock2(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 8e1b15f315..45e3c959ce 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -5384,6 +5384,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     maxTs = TMAX(maxTs, pBlock->info.window.ekey);
     minTs = TMIN(minTs, pBlock->info.window.skey);
+    qDebug("-------------------------groupId:%ld", pBlock->info.groupId);
+
     doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
   }
   pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

From 5b7855eb879c7cbc43344445d00c01d4d96a56ba Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:35:31 +0800
Subject: [PATCH 15/42] other:merge 3.0

---
 source/libs/executor/src/scanoperator.c | 135 ------------------------
 1 file changed, 135 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 32d2684a0a..e854d479fc 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4434,144 +4434,9 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 typedef struct STableMergeScanSortSourceParam {
   SOperatorInfo* pOperator;
   int32_t        readerIdx;
-  int64_t        uid;
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
-static SSDataBlock* getTableDataBlockTemp(void* param) {
-  STableMergeScanSortSourceParam* source = param;
-  SOperatorInfo*                  pOperator = source->pOperator;
-  STableMergeScanInfo*            pInfo = pOperator->info;
-  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
-  int32_t                         readIdx = source->readerIdx;
-  SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
-
-  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
-
-  blockDataCleanup(pBlock);
-
-  int64_t st = taosGetTimestampUs();
-
-  void* pStart = taosArrayGet(pInfo->tableListInfo->pTableList, readIdx + pInfo->tableStartIndex);
-
-  SReadHandle* pHandle = &pInfo->readHandle;
-  tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
-
-  STsdbReader* reader = pInfo->pReader;
-  while (tsdbNextDataBlock(reader)) {
-    if (isTaskKilled(pOperator->pTaskInfo)) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-    }
-
-    // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
-    if (!processThisBlock) {
-      continue;
-    }
-
-    blockDataCleanup(pBlock);
-    SDataBlockInfo binfo = pBlock->info;
-    tsdbRetrieveDataBlockInfo(reader, &binfo);
-
-    blockDataEnsureCapacity(pBlock, binfo.rows);
-    pBlock->info.type = binfo.type;
-    pBlock->info.uid = binfo.uid;
-    pBlock->info.window = binfo.window;
-    pBlock->info.rows = binfo.rows;
-
-    if (tsdbIsAscendingOrder(pInfo->pReader)) {
-      pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
-    } else {
-      pQueryCond->twindows.ekey = pBlock->info.window.skey - 1;
-    }
-
-    uint32_t status = 0;
-    int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readIdx, pBlock, &status);
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, code);
-    }
-
-    // current block is filter out according to filter condition, continue load the next block
-    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
-      continue;
-    }
-
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
-    pOperator->resultInfo.totalRows += pBlock->info.rows;  // pTableScanInfo->readRecorder.totalRows;
-    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
-
-    tsdbReaderClose(pInfo->pReader);
-    pInfo->pReader = NULL;
-    return pBlock;
-  }
-  tsdbReaderClose(pInfo->pReader);
-  pInfo->pReader = NULL;
-
-  return NULL;
-}
-
-static SSDataBlock* getTableDataBlock2(void* param) {
-  STableMergeScanSortSourceParam* source = param;
-  SOperatorInfo*                  pOperator = source->pOperator;
-  int64_t                         uid = source->uid;
-  SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
-
-  int64_t st = taosGetTimestampUs();
-
-  blockDataCleanup(pBlock);
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  while (tsdbTableNextDataBlock(reader, uid)) {
-    if (isTaskKilled(pOperator->pTaskInfo)) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-    }
-
-    // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
-    if (!processThisBlock) {
-      continue;
-    }
-
-    blockDataCleanup(pBlock);
-    SDataBlockInfo binfo = pBlock->info;
-    tsdbRetrieveDataBlockInfo(reader, &binfo);
-
-    blockDataEnsureCapacity(pBlock, binfo.rows);
-    pBlock->info.type = binfo.type;
-    pBlock->info.uid = binfo.uid;
-    pBlock->info.window = binfo.window;
-    pBlock->info.rows = binfo.rows;
-
-    uint32_t status = 0;
-    int32_t  code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, code);
-    }
-
-    // current block is filter out according to filter condition, continue load the next block
-    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
-      continue;
-    }
-
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
-    pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
-    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
-
-    return pBlock;
-  }
-  return NULL;
-}
-
 static SSDataBlock* getTableDataBlock(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;

From d1097b7c2d77e724f69f2e4118086e299f5bc408 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:37:43 +0800
Subject: [PATCH 16/42] other:merge 3.0

---
 source/libs/executor/inc/executorimpl.h |  34 ++++----
 source/libs/executor/src/scanoperator.c | 108 ------------------------
 2 files changed, 17 insertions(+), 125 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 50c62b9017..c7a6ae5774 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -340,23 +340,23 @@ typedef struct STableScanInfo {
 } STableScanInfo;
 
 typedef struct STableMergeScanInfo {
-  STableListInfo*        tableListInfo;
-  int32_t                tableStartIndex;
-  int32_t                tableEndIndex;
-  bool                   hasGroupId;
-  uint64_t               groupId;
-  SArray*                dataReaders;  // array of tsdbReaderT*
-  SArray*                queryConds;   // array of queryTableDataCond
-  STsdbReader*           pReader;
-  SReadHandle            readHandle;
-  int32_t                bufPageSize;
-  uint32_t               sortBufSize;  // max buffer size for in-memory sort
-  SArray*                pSortInfo;
-  SSortHandle*           pSortHandle;
-  SSDataBlock*           pSortInputBlock;
-  int64_t                startTs;  // sort start time
-  SArray*                sortSourceParams;
-  SLimitInfo             limitInfo;
+  STableListInfo* tableListInfo;
+  int32_t         tableStartIndex;
+  int32_t         tableEndIndex;
+  bool            hasGroupId;
+  uint64_t        groupId;
+  SArray*         dataReaders;  // array of tsdbReaderT*
+  SReadHandle     readHandle;
+  int32_t         bufPageSize;
+  uint32_t        sortBufSize;  // max buffer size for in-memory sort
+  SArray*         pSortInfo;
+  SSortHandle*    pSortHandle;
+  SSDataBlock*    pSortInputBlock;
+  int64_t         startTs;  // sort start time
+  SArray*         sortSourceParams;
+  SLimitInfo      limitInfo;
+
+
   SFileBlockLoadRecorder readRecorder;
   int64_t                numOfRows;
   SScanInfo              scanInfo;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index e854d479fc..7795eff410 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4204,114 +4204,6 @@ int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle*
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
-                                          SSDataBlock* pBlock, uint32_t* status) {
-  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
-  STableMergeScanInfo* pInfo = pOperator->info;
-
-  uint64_t uid = pBlock->info.uid;
-
-  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
-
-  pCost->totalBlocks += 1;
-  pCost->totalRows += pBlock->info.rows;
-
-  *status = pInfo->dataBlockLoadFlag;
-  if (pTableScanInfo->pFilterNode != NULL ||
-      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
-    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
-  }
-
-  SDataBlockInfo* pBlockInfo = &pBlock->info;
-  taosMemoryFreeClear(pBlock->pBlockAgg);
-
-  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
-    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->filterOutBlocks += 1;
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
-    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->skipBlocks += 1;
-
-    // clear all data in pBlock that are set when handing the previous block
-    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
-      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
-      pcol->pData = NULL;
-    }
-
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
-    pCost->loadBlockStatis += 1;
-
-    bool             allColumnsHaveAgg = true;
-    SColumnDataAgg** pColAgg = NULL;
-    STsdbReader*     reader = pTableScanInfo->pReader;
-    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
-
-    if (allColumnsHaveAgg == true) {
-      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
-      // todo create this buffer during creating operator
-      if (pBlock->pBlockAgg == NULL) {
-        pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-      }
-
-      for (int32_t i = 0; i < numOfCols; ++i) {
-        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
-        if (!pColMatchInfo->needOutput) {
-          continue;
-        }
-        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
-      }
-
-      return TSDB_CODE_SUCCESS;
-    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
-      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
-    }
-  }
-
-  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
-
-  pCost->totalCheckedRows += pBlock->info.rows;
-  pCost->loadBlocks += 1;
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
-  if (pCols == NULL) {
-    return terrno;
-  }
-
-  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
-
-  // currently only the tbname pseudo column
-  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
-                                          pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo));
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pTaskInfo->env, code);
-    }
-  }
-
-  if (pTableScanInfo->pFilterNode != NULL) {
-    int64_t st = taosGetTimestampMs();
-    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
-
-    double el = (taosGetTimestampUs() - st) / 1000.0;
-    pTableScanInfo->readRecorder.filterTime += el;
-
-    if (pBlock->info.rows == 0) {
-      pCost->filterOutBlocks += 1;
-      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
-             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
-    } else {
-      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {

From 6ff3668ecc8cab49b10b1c1aeca794e48a4c7b91 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:38:37 +0800
Subject: [PATCH 17/42] refactor: update the log.

---
 source/dnode/vnode/src/sma/smaRollup.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 0ed9ec210f..cd50450379 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -702,7 +702,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
 #endif
     for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
       SSDataBlock *output = taosArrayGetP(pResList, i);
-      smaError("uid:%ld, groupid:%ld", output->info.uid, output->info.groupId);
+      smaDebug("uid:%ld, groupid:%ld", output->info.uid, output->info.groupId);
 
       STsdb       *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
       SSubmitReq  *pReq = NULL;

From c52dc709f169696c8d154baf064d6c4d44af1032 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:39:25 +0800
Subject: [PATCH 18/42] refactor: remove some codes.

---
 source/libs/executor/src/scanoperator.c | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7795eff410..18add21881 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4188,22 +4188,6 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
-                                   STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
-                                   STsdbReader** ppReader, const char* idstr) {
-  STsdbReader* pReader = NULL;
-  void* pStart = taosArrayGet(pTableListInfo->pTableList, tableStartIdx);
-  int32_t num = tableEndIdx - tableStartIdx + 1;
-
-  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr);
-  if (code != 0) {
-    return code;
-  }
-
-  *ppReader = pReader;
-  return TSDB_CODE_SUCCESS;
-}
-
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {

From cb0adb0e9bd6e6256f14d68125f9a44a2c6c5adb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:40:55 +0800
Subject: [PATCH 19/42] refactor: remove some codes.

---
 source/libs/executor/src/scanoperator.c       | 1 -
 source/libs/executor/src/timewindowoperator.c | 1 -
 2 files changed, 2 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 18add21881..c2a0ccc8de 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1565,7 +1565,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   if (groupIdPre) {
     pInfo->pRes->info.groupId = *groupIdPre;
   } else {
-    ASSERT(0);
     pInfo->pRes->info.groupId = 0;
   }
 
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 45e3c959ce..49a7112eba 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -5384,7 +5384,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     maxTs = TMAX(maxTs, pBlock->info.window.ekey);
     minTs = TMIN(minTs, pBlock->info.window.skey);
-    qDebug("-------------------------groupId:%ld", pBlock->info.groupId);
 
     doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
   }

From 9b99c697852d12e566a7bba36f7722751e62f411 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 19:56:32 +0800
Subject: [PATCH 20/42] fix(query): support partition by + limit push down.

---
 include/common/tcommon.h                     |   8 -
 source/dnode/vnode/inc/vnode.h               |  12 +-
 source/dnode/vnode/src/inc/tsdb.h            |   4 +-
 source/dnode/vnode/src/inc/vnodeInt.h        |   3 -
 source/dnode/vnode/src/sma/smaRollup.c       |   1 -
 source/dnode/vnode/src/tsdb/tsdbCacheRead.c  |  21 +-
 source/dnode/vnode/src/tsdb/tsdbRead.c       | 107 ++--
 source/libs/executor/inc/executil.h          |  21 +-
 source/libs/executor/inc/executorimpl.h      |   2 +-
 source/libs/executor/src/cachescanoperator.c |  46 +-
 source/libs/executor/src/executil.c          | 109 +++-
 source/libs/executor/src/executor.c          |  46 +-
 source/libs/executor/src/executorimpl.c      | 206 +++---
 source/libs/executor/src/scanoperator.c      | 640 +++++--------------
 14 files changed, 516 insertions(+), 710 deletions(-)

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 2b45a5d206..3bfbb85958 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -164,14 +164,6 @@ typedef enum EStreamType {
   STREAM_FILL_OVER,
 } EStreamType;
 
-typedef struct {
-  SArray*   pGroupList;
-  SArray*   pTableList;
-  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
-  bool      needSortTableByGroupId;
-  uint64_t  suid;
-} STableListInfo;
-
 #pragma pack(push, 1)
 typedef struct SColumnDataAgg {
   int16_t colId;
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index bed6e93e5a..fcb9b4ab44 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -152,13 +152,13 @@ typedef struct STsdbReader STsdbReader;
 #define CACHESCAN_RETRIEVE_LAST_ROW    0x4
 #define CACHESCAN_RETRIEVE_LAST        0x8
 
-int32_t tsdbSetTableId(STsdbReader *pReader, int64_t uid);
-int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTableList, STsdbReader **ppReader,
-                       const char *idstr);
+int32_t  tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num);
+int32_t  tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
+                        STsdbReader **ppReader, const char *idstr);
 
 void     tsdbReaderClose(STsdbReader *pReader);
 bool     tsdbNextDataBlock(STsdbReader *pReader);
-bool     tsdbTableNextDataBlock(STsdbReader *pReader, int64_t uid);
+bool     tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
 void     tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
 int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
@@ -170,8 +170,8 @@ void    *tsdbGetIdx(SMeta *pMeta);
 void    *tsdbGetIvtIdx(SMeta *pMeta);
 uint64_t getReaderMaxVersion(STsdbReader *pReader);
 
-int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, uint64_t suid,
-                                void **pReader);
+int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables,
+                                int32_t numOfCols, uint64_t suid, void** pReader);
 int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
 void   *tsdbCacherowsReaderClose(void *pReader);
 int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index e5b8a1f327..c4e7ff92b7 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -716,7 +716,9 @@ typedef struct SCacheRowsReader {
   int32_t            numOfCols;
   int32_t            type;
   int32_t            tableIndex;  // currently returned result tables
-  SArray            *pTableList;  // table id list
+
+  STableKeyInfo     *pTableList;  // table id list
+  int32_t            numOfTables;
   SSttBlockLoadInfo *pLoadInfo;
   STsdbReadSnap     *pReadSnap;
   SDataFReader      *pDataFReader;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index a5dc4431ab..94178727be 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -162,10 +162,7 @@ int         tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub
 int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
                                 SSubmitBlkRsp* pRsp);
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
-STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
-                                void* pMemRef);
 int32_t     tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
-int32_t     tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list);
 
 // tq
 int     tqInit();
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index cd50450379..4b9b4f9242 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -671,7 +671,6 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
   while (1) {
     uint64_t ts;
     bool     hasMore = false;
-//    ASSERT(0);
     int32_t  code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL);
     if (code < 0) {
       if (code == TSDB_CODE_QRY_IN_EXEC) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index b8f49f38e4..dbc02363ea 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -97,10 +97,9 @@ static void saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea
   }
 }
 
-int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList, int32_t numOfCols, uint64_t suid,
-                                void** pReader) {
+int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
+                                uint64_t suid, void** pReader) {
   *pReader = NULL;
-
   SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader));
   if (p == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -111,14 +110,15 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList
   p->numOfCols = numOfCols;
   p->suid = suid;
 
-  if (taosArrayGetSize(pTableIdList) == 0) {
+  if (numOfTables == 0) {
     *pReader = p;
     return TSDB_CODE_SUCCESS;
   }
 
-  STableKeyInfo* pKeyInfo = taosArrayGet(pTableIdList, 0);
+  STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pTableIdList)[0];
   p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1, 1);
   p->pTableList = pTableIdList;
+  p->numOfTables = numOfTables;
 
   p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES);
   if (p->transferBuf == NULL) {
@@ -205,7 +205,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   SLRUCache* lruCache = pr->pVnode->pTsdb->lruCache;
   LRUHandle* h = NULL;
   SArray*    pRow = NULL;
-  size_t     numOfTables = taosArrayGetSize(pr->pTableList);
   bool       hasRes = false;
   SArray*    pLastCols = NULL;
 
@@ -243,8 +242,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
   // retrieve the only one last row of all tables in the uid list.
   if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
-    for (int32_t i = 0; i < numOfTables; ++i) {
-      STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i);
+    for (int32_t i = 0; i < pr->numOfTables; ++i) {
+      STableKeyInfo* pKeyInfo = &pr->pTableList[i];
 
       code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
       if (code != TSDB_CODE_SUCCESS) {
@@ -308,8 +307,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
     }
 
   } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
-    for (int32_t i = pr->tableIndex; i < numOfTables; ++i) {
-      STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pr->pTableList, i);
+    for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
+      STableKeyInfo* pKeyInfo = &pr->pTableList[i];
       code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
@@ -334,7 +333,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
     code = TSDB_CODE_INVALID_PARA;
   }
 
-_end:
+  _end:
   tsdbDataFReaderClose(&pr->pDataFReaderLast);
   tsdbDataFReaderClose(&pr->pDataFReader);
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 5a495f263e..8ebee92d15 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -270,24 +270,27 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
   }
 }
 
+static void clearBlockScanInfo(STableBlockScanInfo* p) {
+  p->iterInit = false;
+  p->iiter.hasVal = false;
+
+  if (p->iter.iter != NULL) {
+    p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);
+  }
+
+  if (p->iiter.iter != NULL) {
+    p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);
+  }
+
+  p->delSkyline = taosArrayDestroy(p->delSkyline);
+  p->pBlockList = taosArrayDestroy(p->pBlockList);
+  tMapDataClear(&p->mapData);
+}
+
 static void destroyBlockScanInfo(SHashObj* pTableMap) {
   STableBlockScanInfo* p = NULL;
-
   while ((p = taosHashIterate(pTableMap, p)) != NULL) {
-    p->iterInit = false;
-    p->iiter.hasVal = false;
-
-    if (p->iter.iter != NULL) {
-      p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);
-    }
-
-    if (p->iiter.iter != NULL) {
-      p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);
-    }
-
-    p->delSkyline = taosArrayDestroy(p->delSkyline);
-    p->pBlockList = taosArrayDestroy(p->pBlockList);
-    tMapDataClear(&p->mapData);
+    clearBlockScanInfo(p);
   }
 
   taosHashCleanup(pTableMap);
@@ -423,7 +426,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
     return true;
   }
 
-_err:
+  _err:
   return false;
 }
 
@@ -525,7 +528,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   *ppReader = pReader;
   return code;
 
-_end:
+  _end:
   tsdbReaderClose(pReader);
   *ppReader = NULL;
   return code;
@@ -577,7 +580,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader,
 
   pReader->cost.headFileLoadTime += (et1 - st) / 1000.0;
 
-_end:
+  _end:
   taosArrayDestroy(aBlockIdx);
   return code;
 }
@@ -951,7 +954,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1;
   tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
-            ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
+                ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
             pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain,
             unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr);
 
@@ -978,7 +981,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
-              ", rows:%d, code:%s %s",
+                  ", rows:%d, code:%s %s",
               pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow,
               tstrerror(code), pReader->idStr);
     return code;
@@ -987,7 +990,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
 
   tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64
-            ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
+                ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
             pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow,
             pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr);
 
@@ -1402,8 +1405,8 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock
   // log the reason why load the datablock for profile
   if (loadDataBlock) {
     tsdbDebug("%p uid:%" PRIu64
-              " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
-              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
+                  " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
+                  "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
               pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
               info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock,
               pReader->idStr);
@@ -1438,7 +1441,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
 
   double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
   tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64
-            " - %" PRId64 " %s",
+                " - %" PRId64 " %s",
             pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey,
             pReader->idStr);
 
@@ -2036,7 +2039,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
         pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL);
 
         tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-                  "-%" PRId64 " %s",
+                      "-%" PRId64 " %s",
                   pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr);
       } else {
         tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid,
@@ -2057,7 +2060,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
         pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL);
 
         tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-                  "-%" PRId64 " %s",
+                      "-%" PRId64 " %s",
                   pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr);
       } else {
         tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid,
@@ -2292,7 +2295,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
     }
   }
 
-_end:
+  _end:
   pResBlock->info.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
   blockDataUpdateTsWindow(pResBlock, 0);
 
@@ -2304,7 +2307,7 @@ _end:
 
   if (pResBlock->info.rows > 0) {
     tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
-              " rows:%d, elapsed time:%.2f ms %s",
+                  " rows:%d, elapsed time:%.2f ms %s",
               pReader, pResBlock->info.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
               pResBlock->info.rows, el, pReader->idStr);
   }
@@ -2385,13 +2388,13 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
 
   taosArrayDestroy(pDelData);
   pBlockScanInfo->iter.index =
-      ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
+          ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
   pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index;
   pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index;
   pBlockScanInfo->lastBlockDelIndex = pBlockScanInfo->iter.index;
   return code;
 
-_err:
+  _err:
   taosArrayDestroy(pDelData);
   return code;
 }
@@ -2722,7 +2725,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
   SDataBlockIter* pBlockIter = &pReader->status.blockIter;
 
   if (pBlockIter->numOfBlocks == 0) {
-  _begin:
+    _begin:
     code = doLoadLastBlockSequentially(pReader);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -2811,8 +2814,8 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
     int8_t  precision = pVnode->config.tsdbCfg.precision;
     int64_t now = taosGetTimestamp(precision);
     int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI)   ? 1L
-                                             : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
-                                                                                        : 1000000L);
+                                                                                        : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
+                                                                                                                                   : 1000000L);
 
     for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
       SRetention* pRetention = retentions + level;
@@ -3452,13 +3455,23 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
   return TSDB_CODE_SUCCESS;
 }
 
-// todo refactor, use arraylist instead
-int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) {
+// TODO refactor: with createDataBlockScanInfo
+int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num) {
   ASSERT(pReader != NULL);
+
+  STableBlockScanInfo* p = NULL;
+  while ((p = taosHashIterate(pReader->status.pTableMap, p)) != NULL) {
+    clearBlockScanInfo(p);
+  }
+
   taosHashClear(pReader->status.pTableMap);
 
-  STableBlockScanInfo info = {.lastKey = 0, .uid = uid};
-  taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
+  STableKeyInfo* pList = (STableKeyInfo*) pTableList;
+  for(int32_t i = 0; i < num; ++i) {
+    STableBlockScanInfo info = {.lastKey = 0, .uid = pList[i].uid};
+    taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
+  }
+
   return TDB_CODE_SUCCESS;
 }
 
@@ -3494,8 +3507,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
 }
 
 // ====================================== EXPOSED APIs ======================================
-int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader,
-                       const char* idstr) {
+int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
+                       STsdbReader** ppReader, const char* idstr) {
   STimeWindow window = pCond->twindows;
   if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
     pCond->twindows.skey += 1;
@@ -3554,8 +3567,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
     if (pReader->pSchema == NULL) {
       tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
     }
-  } else if (taosArrayGetSize(pTableList) > 0) {
-    STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
+  } else if (numOfTables > 0) {
+    STableKeyInfo* pKey = pTableList;
     pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
     if (pReader->pSchema == NULL) {
       tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
@@ -3564,8 +3577,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
 
   STsdbReader* p = pReader->innerReader[0] != NULL ? pReader->innerReader[0] : pReader;
 
-  int32_t numOfTables = taosArrayGetSize(pTableList);
-  pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList->pData, numOfTables);
+  pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList, numOfTables);
   if (pReader->status.pTableMap == NULL) {
     tsdbReaderClose(pReader);
     *ppReader = NULL;
@@ -3622,7 +3634,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
   tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
   return code;
 
-_err:
+  _err:
   tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
   return code;
 }
@@ -3775,7 +3787,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   return false;
 }
 
-bool tsdbTableNextDataBlock(STsdbReader* pReader, int64_t uid) {
+bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
   STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
   if (pBlockScanInfo == NULL) {  // no data block for the table of given uid
     return false;
@@ -3964,7 +3976,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
   }
 
   tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64
-            " in query %s",
+                " in query %s",
             pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey,
             pReader->idStr);
 
@@ -4155,7 +4167,7 @@ int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr
   }
 
   tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
-_exit:
+  _exit:
   return code;
 }
 
@@ -4174,4 +4186,3 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) {
   }
   tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
 }
-bool tsdbIsAscendingOrder(STsdbReader* pReader) { return ASCENDING_TRAVERSE(pReader->order); }
diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h
index 0cfef7dc24..1965a8cbf6 100644
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@@ -28,7 +28,7 @@
   do {                       \
     ASSERT((_c) != -1);      \
     longjmp((_obj), (_c));   \
-  } while (0);
+  } while (0)
 
 #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid)     \
   do {                                               \
@@ -95,6 +95,25 @@ typedef struct SColMatchInfo {
   int32_t matchType;  // determinate the source according to col id or slot id
 } SColMatchInfo;
 
+// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly
+// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups
+typedef struct STableListInfo {
+  bool      oneTableForEachGroup;
+  int32_t   numOfOuputGroups; // the data block will be generated one by one
+  int32_t*  groupOffset;      // keep the offset value for each group in the tableList
+  SArray*   pTableList;
+  SHashObj* map;              // speedup acquire the tableQueryInfo by table uid
+  uint64_t  suid;
+} STableListInfo;
+
+void     destroyTableList(STableListInfo* pTableList);
+int32_t  getNumOfOutputGroups(const STableListInfo* pTableList);
+bool     oneTableForEachGroup(const STableListInfo* pTableList);
+uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
+int32_t  addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
+int32_t  getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num);
+uint64_t getTotalTables(const STableListInfo* pTableList);
+
 struct SqlFunctionCtx;
 
 size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 8769e8ac2f..e4f6b42224 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -1077,7 +1077,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
 bool    groupbyTbname(SNodeList* pGroupList);
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
+int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort);
 void*   destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
 int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
                                    SGroupResInfo* pGroupResInfo);
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index f645e71c6e..d37efbe941 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -48,6 +48,10 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   int32_t numOfCols = 0;
   code =
       extractColMatchInfo(pScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   removeRedundantTsCol(pScanNode, &pInfo->matchInfo);
 
   code = extractCacheScanSlotId(pInfo->matchInfo.pList, pTaskInfo, &pInfo->pSlotIds);
@@ -61,11 +65,15 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
   pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
 
-  // partition by tbname
-  if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) {
+  // partition by tbname, todo opt perf
+  if (oneTableForEachGroup(pTableList) || (getTotalTables(pTableList) == 1)) {
     pInfo->retrieveType =
         CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
-    code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList,
+
+    STableKeyInfo* pList = taosArrayGet(pTableList->pTableList, 0);
+    size_t num = taosArrayGetSize(pTableList->pTableList);
+
+    code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
                                    taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
@@ -98,7 +106,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   pOperator->cost.openCost = 0;
   return pOperator;
 
-_error:
+  _error:
   pTaskInfo->code = code;
   destroyLastrowScanOperator(pInfo);
   taosMemoryFree(pOperator);
@@ -167,16 +175,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
         }
       }
 
-      if (pTableList->map != NULL) {
-        int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t));
-        if (groupId != NULL) {
-          pInfo->pRes->info.groupId = *groupId;
-        }
-      } else {
-        ASSERT(taosArrayGetSize(pTableList->pTableList) == 1);
-        STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0);
-        pInfo->pRes->info.groupId = pKeyInfo->groupId;
-      }
+      pInfo->pRes->info.groupId = getTableGroupId(pTableList, pInfo->pRes->info.uid);
 
       pInfo->indexOfBufferedRes += 1;
       return pInfo->pRes;
@@ -185,18 +184,25 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       return NULL;
     }
   } else {
-    size_t totalGroups = taosArrayGetSize(pTableList->pGroupList);
+    size_t totalGroups = getNumOfOutputGroups(pTableList);
 
     while (pInfo->currentGroupIndex < totalGroups) {
-      SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex);
 
-      tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList,
+      STableKeyInfo* pList = NULL;
+      int32_t num = 0;
+
+      int32_t code = getTablesOfGroup(pTableList, pInfo->currentGroupIndex, &pList, &num);
+      if (code != TSDB_CODE_SUCCESS) {
+        T_LONG_JMP(pTaskInfo->env, code);
+      }
+
+      tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
                               taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader);
       taosArrayClear(pInfo->pUidList);
 
-      int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList);
+      code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList);
       if (code != TSDB_CODE_SUCCESS) {
-        longjmp(pTaskInfo->env, code);
+        T_LONG_JMP(pTaskInfo->env, code);
       }
 
       pInfo->currentGroupIndex += 1;
@@ -206,7 +212,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
         if (pInfo->pseudoExprSup.numOfExprs > 0) {
           SExprSupp* pSup = &pInfo->pseudoExprSup;
 
-          STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0);
+          STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pTableList)[0];
           pInfo->pRes->info.groupId = pKeyInfo->groupId;
 
           if (taosArrayGetSize(pInfo->pUidList) > 0) {
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index a1eb3af391..6c5705650c 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -337,8 +337,8 @@ static EDealRes getColumn(SNode** pNode, void* pContext) {
     taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
     pSColumnNode->slotId = pData->index++;
     SColumnInfo cInfo = {.colId = pSColumnNode->colId,
-                         .type = pSColumnNode->node.resType.type,
-                         .bytes = pSColumnNode->node.resType.bytes};
+        .type = pSColumnNode->node.resType.type,
+        .bytes = pSColumnNode->node.resType.bytes};
 #if TAG_FILTER_DEBUG
     qDebug("tagfilter build column info, slotId:%d, colId:%d, type:%d", pSColumnNode->slotId, cInfo.colId, cInfo.type);
 #endif
@@ -507,7 +507,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
   //  int64_t st2 = taosGetTimestampUs();
   //  qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
 
-end:
+  end:
   taosHashCleanup(tags);
   taosHashCleanup(ctx.colHash);
   taosArrayDestroy(ctx.cInfoList);
@@ -544,6 +544,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
+
   ctx.index = 0;
   ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
   if (ctx.cInfoList == NULL) {
@@ -606,6 +607,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
       } else {
         void* tag = taosHashGet(tags, uid, sizeof(int64_t));
         ASSERT(tag);
+
         STagVal tagVal = {0};
         tagVal.cid = pColInfo->info.colId;
         const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
@@ -636,6 +638,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
       }
     }
   }
+
   pResBlock->info.rows = rows;
 
   //  int64_t st1 = taosGetTimestampUs();
@@ -661,10 +664,12 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
         }
         break;
       }
+
       default:
         code = TSDB_CODE_OPS_NOT_SUPPORT;
         goto end;
     }
+
     if (nodeType(pNode) == QUERY_NODE_COLUMN) {
       SColumnNode*     pSColumnNode = (SColumnNode*)pNode;
       SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, pSColumnNode->slotId);
@@ -674,10 +679,12 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
     } else {
       code = scalarCalculate(pNode, pBlockList, &output);
     }
+
     if (code != TSDB_CODE_SUCCESS) {
       releaseColInfoData(output.columnData);
       goto end;
     }
+
     taosArrayPush(groupData, &output.columnData);
   }
 
@@ -696,6 +703,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
+
   for (int i = 0; i < rows; i++) {
     STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
 
@@ -739,7 +747,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
   //  int64_t st2 = taosGetTimestampUs();
   //  qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
 
-end:
+  end:
   taosMemoryFreeClear(keyBuf);
   taosHashCleanup(tags);
   taosHashCleanup(ctx.colHash);
@@ -817,38 +825,86 @@ static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) {
   taosArrayDestroy(validUid);
   return 0;
 }
+
+static int32_t nameComparFn(const void* p1, const void* p2) {
+  const char* pName1 = *(const char**) p1;
+  const char* pName2 = *(const char**) p2;
+
+  int32_t ret = strcmp(pName1, pName2);
+  if (ret == 0) {
+    return 0;
+  } else {
+    return (ret > 0)? 1:-1;
+  }
+}
+
+static SArray* getTableNameList(const SNodeListNode* pList) {
+  int32_t len = LIST_LENGTH(pList->pNodeList);
+  SListCell* cell = pList->pNodeList->pHead;
+
+  SArray* pTbList = taosArrayInit(len, POINTER_BYTES);
+  for (int i = 0; i < pList->pNodeList->length; i++) {
+    SValueNode* valueNode = (SValueNode*) cell->pNode;
+    if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) {
+      terrno = TSDB_CODE_INVALID_PARA;
+      taosArrayDestroy(pTbList);
+      return NULL;
+    }
+
+    char* name = varDataVal(valueNode->datum.p);
+    taosArrayPush(pTbList, &name);
+    cell = cell->pNext;
+  }
+
+  size_t numOfTables = taosArrayGetSize(pTbList);
+
+  // order the name
+  taosArraySort(pTbList, nameComparFn);
+
+  // remove the duplicates
+  SArray* pNewList = taosArrayInit(taosArrayGetSize(pTbList), sizeof(void*));
+  taosArrayPush(pNewList, taosArrayGet(pTbList, 0));
+
+  for (int32_t i = 1; i < numOfTables; ++i) {
+    char** name = taosArrayGetLast(pNewList);
+    char** nameInOldList = taosArrayGet(pTbList, i);
+    if (strcmp(*name, *nameInOldList) == 0) {
+      continue;
+    }
+
+    taosArrayPush(pNewList, nameInOldList);
+  }
+
+  taosArrayDestroy(pTbList);
+  return pNewList;
+}
+
 static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) {
   if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
     return -1;
   }
+
   SOperatorNode* pNode = (SOperatorNode*)pTagCond;
   if (pNode->opType != OP_TYPE_IN) {
     return -1;
   }
+
   if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_COLUMN &&
        ((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME) &&
       (pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) {
     SNodeListNode* pList = (SNodeListNode*)pNode->pRight;
 
     int32_t len = LIST_LENGTH(pList->pNodeList);
-    if (len <= 0) return -1;
-
-    SListCell* cell = pList->pNodeList->pHead;
-
-    SArray* pTbList = taosArrayInit(len, sizeof(void*));
-    for (int i = 0; i < pList->pNodeList->length; i++) {
-      SValueNode* valueNode = (SValueNode*)cell->pNode;
-      if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) {
-        taosArrayDestroy(pTbList);
-        return -1;
-      }
-      char* name = varDataVal(valueNode->datum.p);
-      taosArrayPush(pTbList, &name);
-      cell = cell->pNext;
+    if (len <= 0) {
+      return -1;
     }
 
-    for (int i = 0; i < taosArrayGetSize(pTbList); i++) {
-      char*    name = taosArrayGetP(pTbList, i);
+    SArray* pTbList = getTableNameList(pList);
+    int32_t numOfTables = taosArrayGetSize(pTbList);
+
+    for (int i = 0; i < numOfTables; i++) {
+      char* name = taosArrayGetP(pTbList, i);
+
       uint64_t uid = 0;
       if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
         ETableType tbType = TSDB_TABLE_MAX;
@@ -863,11 +919,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
         terrno = 0;
       }
     }
+
     taosArrayDestroy(pTbList);
     return 0;
   }
+
   return -1;
 }
+
 int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
                      STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -946,14 +1005,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
   }
 
   taosArrayDestroy(res);
-
-  pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
-  if (pListInfo->pGroupList == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  // put into list as default group, remove it if grouping sorting is required later
-  taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
   return code;
 }
 
@@ -1686,4 +1737,4 @@ void destroyTableList(STableListInfo* pTableqinfoList) {
 
   pTableqinfoList->pTableList = NULL;
   pTableqinfoList->map = NULL;
-}
+}
\ No newline at end of file
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 9546c3895e..7f04623612 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -293,9 +293,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
     qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
   }
 
-  if (pListInfo->map == NULL) {
-    pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-  }
+
 
   // traverse to the stream scanner node to add this table id
   SOperatorInfo* pInfo = pTaskInfo->pRoot;
@@ -307,8 +305,10 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
   SStreamScanInfo* pScanInfo = pInfo->info;
   if (isAdd) {  // add new table id
     SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
+    int32_t numOfQualifiedTables = taosArrayGetSize(qa);
+
+    qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables);
 
-    qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
     if (code != TSDB_CODE_SUCCESS) {
       taosArrayDestroy(qa);
@@ -328,7 +328,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
       }
     }
 
-    for (int32_t i = 0; i < taosArrayGetSize(qa); ++i) {
+    STableListInfo* pTableListInfo = &pTaskInfo->tableqinfoList;
+
+    for (int32_t i = 0; i < numOfQualifiedTables; ++i) {
       uint64_t*     uid = taosArrayGet(qa, i);
       STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0};
 
@@ -358,8 +360,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
 
       if (!exists) {
 #endif
-      taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo);
-      taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
+      addTableIntoTableList(pTableListInfo, keyInfo.uid, keyInfo.groupId);
     }
 
     if (keyBuf != NULL) {
@@ -439,7 +440,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
 
   qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
 
-_error:
+  _error:
   // if failed to add ref for all tables in this query, abort current query
   return code;
 }
@@ -935,7 +936,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
       /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
       STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
-      int32_t         tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+      int32_t         numOfTables = getTotalTables(&pTaskInfo->tableqinfoList);
 
 #ifndef NDEBUG
       qDebug("switch to next table %" PRId64 " (cursor %d), %" PRId64 " rows returned", uid,
@@ -944,7 +945,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 #endif
 
       bool found = false;
-      for (int32_t i = 0; i < tableSz; i++) {
+      for (int32_t i = 0; i < numOfTables; i++) {
         STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
         if (pTableInfo->uid == uid) {
           found = true;
@@ -957,14 +958,17 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       ASSERT(found);
 
       if (pTableScanInfo->dataReader == NULL) {
-        if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond,
-                           pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 ||
-            pTableScanInfo->dataReader == NULL) {
+        STableKeyInfo* pList = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
+        int32_t num = getTotalTables(&pTaskInfo->tableqinfoList);
+
+        if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num,
+                           &pTableScanInfo->dataReader, NULL) < 0 || pTableScanInfo->dataReader == NULL) {
           ASSERT(0);
         }
       }
 
-      tsdbSetTableId(pTableScanInfo->dataReader, uid);
+      STableKeyInfo tki = {.uid = uid};
+      tsdbSetTableList(pTableScanInfo->dataReader, &tki, 1);
       int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
       pTableScanInfo->cond.twindows.skey = ts + 1;
       tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
@@ -972,7 +976,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       pTableScanInfo->scanTimes = 0;
 
       qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,
-             ts, pTableScanInfo->currentTable, tableSz);
+             ts, pTableScanInfo->currentTable, numOfTables);
       /*}*/
     } else {
       ASSERT(0);
@@ -994,9 +998,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
     initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
     pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
-    pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
-    taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList,
+
+    STableListInfo* pListInfo = &pTaskInfo->tableqinfoList;
+
+    pListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
+    taosArrayPush(pListInfo->pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
+
+    STableKeyInfo* pList = taosArrayGet(pListInfo->pTableList, 0);
+
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, taosArrayGetSize(pListInfo->pTableList),
                    &pInfo->dataReader, NULL);
 
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 75db42fccc..00fef26670 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -494,7 +494,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
         // todo: refactor this
         if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) {
           pInput->pPTS = pInput->pData[j];  // in case of merge function, this is not always the ts column data.
-                                            //          ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+          //          ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP);
         }
         ASSERT(pInput->pData[j] != NULL);
       } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
@@ -815,7 +815,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
   // abort current query execution.
   if (pTaskInfo->owner != 0 &&
       ((taosGetTimestampSec() - pTaskInfo->cost.start / 1000) > 10 * getMaximumIdleDurationSec())
-      /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
+    /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
     assert(pTaskInfo->cost.start != 0);
     //    qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
     //           ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
@@ -1739,8 +1739,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
-static void doDestroyTableList(STableListInfo* pTableqinfoList);
-
 typedef struct SFetchRspHandleWrapper {
   uint32_t exchangeId;
   int32_t  sourceIndex;
@@ -1965,7 +1963,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
       SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
       if (pRsp->numOfRows == 0) {
         qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
-               ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
+                   ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
                pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
@@ -1992,10 +1990,10 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
 
       if (pRsp->completed == 1) {
         qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
-               " execId:%d"
-               " index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
-               ", total:%.2f Kb,"
-               " completed:%d try next %d/%" PRIzu,
+                   " execId:%d"
+                   " index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
+                   ", total:%.2f Kb,"
+                   " completed:%d try next %d/%" PRIzu,
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks,
                pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0,
                completed + 1, i + 1, totalSources);
@@ -2003,7 +2001,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       } else {
         qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
-               " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb",
+                   " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb",
                GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks,
                pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0);
       }
@@ -2030,7 +2028,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
     sched_yield();
   }
 
-_error:
+  _error:
   pTaskInfo->code = code;
 }
 
@@ -2091,7 +2089,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
     if (pRsp->numOfRows == 0) {
       qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
-             ", totalRows:%" PRIu64 " try next",
+                 ", totalRows:%" PRIu64 " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 
@@ -2108,7 +2106,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
 
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
-             ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
+                 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
              pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
              totalSources);
@@ -2117,7 +2115,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
       pExchangeInfo->current += 1;
     } else {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
-             ", totalBytes:%" PRIu64,
+                 ", totalBytes:%" PRIu64,
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows,
              pLoadInfo->totalRows, pLoadInfo->totalSize);
     }
@@ -2305,7 +2303,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
       createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL);
   return pOperator;
 
-_error:
+  _error:
   if (pInfo != NULL) {
     doDestroyExchangeOperatorInfo(pInfo);
   }
@@ -3042,7 +3040,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
 
   return pOperator;
 
-_error:
+  _error:
   if (pInfo != NULL) {
     destroyAggOperatorInfo(pInfo);
   }
@@ -3212,8 +3210,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
 
   SInterval* pInterval =
       QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
-          ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
-          : &((SIntervalAggOperatorInfo*)downstream->info)->interval;
+      ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
+      : &((SIntervalAggOperatorInfo*)downstream->info)->interval;
 
   int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
   int32_t type = convertFillType(pPhyFillNode->mode);
@@ -3258,7 +3256,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 
-_error:
+  _error:
   if (pInfo != NULL) {
     destroyFillOperatorInfo(pInfo);
   }
@@ -3366,62 +3364,116 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) {
 
 static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); }
 
+static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) {
+  STableKeyInfo* pInfo1 = (STableKeyInfo*) p1;
+  STableKeyInfo* pInfo2 = (STableKeyInfo*) p2;
+
+  if (pInfo1->groupId == pInfo2->groupId) {
+    return 0;
+  } else {
+    return pInfo1->groupId < pInfo2->groupId? -1:1;
+  }
+}
+
 static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
-  taosArrayClear(pTableListInfo->pGroupList);
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn);
+  int32_t size = getTotalTables(pTableListInfo);
+
+  SArray* pList = taosArrayInit(4, sizeof(int32_t));
+
+  STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0);
+  uint64_t gid = pInfo->groupId;
+
+  int32_t start = 0;
+  taosArrayPush(pList, &start);
+
+  for(int32_t i = 1; i < size; ++i) {
+    pInfo = taosArrayGet(pTableListInfo->pTableList, i);
+    if (pInfo->groupId != gid) {
+      taosArrayPush(pList, &i);
+      gid = pInfo->groupId;
+    }
+  }
+
+  pTableListInfo->numOfOuputGroups = taosArrayGetSize(pList);
+  pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
+  memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
+  taosArrayDestroy(pList);
+
+# if 0
+
   SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t));
-  if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
-  for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
+  if (sortSupport == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  size_t num = taosArrayGetSize(pTableListInfo->pTableList);
+  for (int32_t i = 0; i < num; i++) {
     STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
     uint64_t*      groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
 
     int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
     if (index == -1) {
       void*   p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
+
       SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
       if (tGroup == NULL) {
-        taosArrayDestroy(sortSupport);
-        return TSDB_CODE_OUT_OF_MEMORY;
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _error;
       }
+
       if (taosArrayPush(tGroup, info) == NULL) {
         qError("taos push info array error");
-        taosArrayDestroy(sortSupport);
-        return TSDB_CODE_QRY_APP_ERROR;
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _error;
       }
+
       if (p == NULL) {
         if (taosArrayPush(sortSupport, groupId) == NULL) {
           qError("taos push support array error");
-          taosArrayDestroy(sortSupport);
-          return TSDB_CODE_QRY_APP_ERROR;
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _error;
         }
+
         if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) == NULL) {
           qError("taos push group array error");
-          taosArrayDestroy(sortSupport);
-          return TSDB_CODE_QRY_APP_ERROR;
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _error;
         }
       } else {
         int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
         if (taosArrayInsert(sortSupport, pos, groupId) == NULL) {
           qError("taos insert support array error");
-          taosArrayDestroy(sortSupport);
-          return TSDB_CODE_QRY_APP_ERROR;
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _error;
         }
+
         if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) {
           qError("taos insert group array error");
-          taosArrayDestroy(sortSupport);
-          return TSDB_CODE_QRY_APP_ERROR;
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _error;
         }
       }
     } else {
       SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
       if (taosArrayPush(tGroup, info) == NULL) {
         qError("taos push uid array error");
-        taosArrayDestroy(sortSupport);
-        return TSDB_CODE_QRY_APP_ERROR;
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _error;
       }
     }
   }
+
   taosArrayDestroy(sortSupport);
+#endif
+
   return TDB_CODE_SUCCESS;
+
+  _error:
+//  taosArrayDestroy(sortSupport);
+  return code;
 }
 
 bool groupbyTbname(SNodeList* pGroupList) {
@@ -3437,38 +3489,44 @@ bool groupbyTbname(SNodeList* pGroupList) {
   return bytbname;
 }
 
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) {
+int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) {
+  int32_t code = TSDB_CODE_SUCCESS;
   if (group == NULL) {
-    return TDB_CODE_SUCCESS;
+    return code;
   }
 
   pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   if (pTableListInfo->map == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    return code;
   }
 
-  bool assignUid = groupbyTbname(group);
-
+  bool   assignUid = groupbyTbname(group);
   size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
 
-  if (assignUid) {
+  if (assignUid) { // in case of group/partition by tbname, the group id is equalled to the uid of table
     for (int32_t i = 0; i < numOfTables; i++) {
       STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
       info->groupId = info->uid;
       taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t));
     }
+
+    pTableListInfo->oneTableForEachGroup = true;
+    if (groupSort) {
+      pTableListInfo->numOfOuputGroups = numOfTables;
+    }
   } else {
-    int32_t code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo);
+    code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
+
+    if (groupSort) {
+      code = sortTableGroup(pTableListInfo);
+    }
   }
 
-  if (pTableListInfo->needSortTableByGroupId) {
-    return sortTableGroup(pTableListInfo);
-  }
-
-  return TDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) {
@@ -3505,6 +3563,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
+      // NOTE: this is an patch to fix the physical plan
+      // TODO remove it later
+      if (pTableScanNode->scan.node.pLimit != NULL) {
+        pTableScanNode->groupSort = true;
+      }
+
       int32_t code =
           createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
                                   pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
@@ -3563,8 +3627,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
         int32_t sz = taosArrayGetSize(pTableListInfo->pTableList);
         for (int32_t i = 0; i < sz; i++) {
           STableKeyInfo* pKeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
-          qDebug("creating stream task: add table %" PRId64, pKeyInfo->uid);
+          qDebug("creating stream task: add table uid:%" PRIu64, pKeyInfo->uid);
         }
+
+        qDebug("table in hashmap, %d", (int32_t) getTotalTables(pTableListInfo));
 #endif
       }
 
@@ -3599,13 +3665,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       }
 
       SQueryTableDataCond cond = {0};
-      int32_t             code = initTableblockDistQueryCond(pBlockNode->suid, &cond);
+
+      int32_t code = initTableblockDistQueryCond(pBlockNode->suid, &cond);
       if (code != TSDB_CODE_SUCCESS) {
         return NULL;
       }
 
+      size_t num = getTotalTables(pTableListInfo);
+      void*  pList = NULL;
+      if (num > 0) {
+        pList = taosArrayGet(pTableListInfo->pTableList, 0);
+      }
+
       STsdbReader* pReader = NULL;
-      tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, "");
+      tsdbReaderOpen(pHandle->vnode, &cond, pList, num, &pReader, "");
       cleanupQueryTableDataCond(&cond);
 
       pOperator = createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo);
@@ -3639,7 +3712,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     return pOperator;
   }
 
-  size_t          size = LIST_LENGTH(pPhyNode->pChildren);
+  size_t  size = LIST_LENGTH(pPhyNode->pChildren);
+
   SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
   for (int32_t i = 0; i < size; ++i) {
     SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
@@ -3848,7 +3922,7 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32
     *length = *(int32_t*)(*result);
   }
 
-_downstream:
+  _downstream:
   for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
     code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
     if (code != TDB_CODE_SUCCESS) {
@@ -3968,35 +4042,17 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
 
   return code;
 
-_complete:
+  _complete:
   taosMemoryFree(sql);
   doDestroyTask(*pTaskInfo);
   terrno = code;
   return code;
 }
 
-void doDestroyTableList(STableListInfo* pTableqinfoList) {
-  taosArrayDestroy(pTableqinfoList->pTableList);
-  taosHashCleanup(pTableqinfoList->map);
-  if (pTableqinfoList->needSortTableByGroupId) {
-    for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) {
-      SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
-      if (tmp == pTableqinfoList->pTableList) {
-        continue;
-      }
-      taosArrayDestroy(tmp);
-    }
-  }
-  taosArrayDestroy(pTableqinfoList->pGroupList);
-
-  pTableqinfoList->pTableList = NULL;
-  pTableqinfoList->map = NULL;
-}
-
 void doDestroyTask(SExecTaskInfo* pTaskInfo) {
   qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
 
-  doDestroyTableList(&pTaskInfo->tableqinfoList);
+  destroyTableList(&pTaskInfo->tableqinfoList);
   destroyOperatorInfo(pTaskInfo->pRoot);
   cleanupTableSchemaInfo(&pTaskInfo->schemaInfo);
   cleanupStreamInfo(&pTaskInfo->streamInfo);
@@ -4125,8 +4181,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
     int32_t     size = 0;
     void*       pVal = NULL;
     SWinKey     key = {
-            .ts = *(TSKEY*)pPos->key,
-            .groupId = pPos->groupId,
+        .ts = *(TSKEY*)pPos->key,
+        .groupId = pPos->groupId,
     };
     int32_t code = streamStateGet(pState, &key, &pVal, &size);
     ASSERT(code == 0);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9a61e555f0..a139809d8f 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -42,7 +42,7 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S
 static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name",     "create_time",      "columns",
                                       "ttl",        "stable_name", "vgroup_id', 'uid", "type"};
 
-static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"};
+static char* SYSTABLE_IDX_EXCEPT[] = {"db_name", "vgroup_id"};
 
 typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
 typedef int32_t (*__sys_check)(SNode* cond);
@@ -363,8 +363,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
     if (pLimitInfo->remainOffset >= pBlock->info.rows) {
       pLimitInfo->remainOffset -= pBlock->info.rows;
       pBlock->info.rows = 0;
-      qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset,
-             GET_TASKID(pTaskInfo));
+      qDebug("current block ignore due to offset, current:%"PRId64", %s", pLimitInfo->remainOffset, GET_TASKID(pTaskInfo));
     } else {
       blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
       pLimitInfo->remainOffset = 0;
@@ -377,9 +376,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
     int32_t keep = pBlock->info.rows - overflowRows;
 
     blockDataKeepFirstNRows(pBlock, keep);
-    qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo));
-
-    //    setTaskStatus(pTaskInfo, TASK_COMPLETED);
+    qDebug("output limit %"PRId64" has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo));
     pOperator->status = OP_EXEC_DONE;
   }
 }
@@ -680,10 +677,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
     if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
       setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
       pTableScanInfo->scanFlag = REPEAT_SCAN;
-      qDebug(
-          "%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks "
-          "due to query func required",
-          GET_TASKID(pTaskInfo));
+      qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo));
 
       // do prepare for the next round table scan operation
       tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
@@ -710,8 +704,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
         setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
         pTableScanInfo->scanFlag = REPEAT_SCAN;
 
-        qDebug("%s start to repeat descending order scan data blocks due to query func required",
-               GET_TASKID(pTaskInfo));
+        qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo));
         tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
       }
     }
@@ -724,7 +717,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
   STableScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
-  // if scan table by table
+  // scan table one by one sequentially
   if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
     int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
 
@@ -741,57 +734,64 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
       }
 
       STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
-      tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
+      tsdbSetTableList(pInfo->dataReader, pTableInfo, 1);
       qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables,
              pInfo->currentTable, pTaskInfo->id.str);
 
       tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
       pInfo->scanTimes = 0;
     }
-  }
+  } else {  // scan table group by group sequentially
+    if (pInfo->currentGroupId == -1) {
+      if ((++pInfo->currentGroupId) >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)) {
+        doSetOperatorCompleted(pOperator);
+        return NULL;
+      }
 
-  if (pInfo->currentGroupId == -1) {
-    pInfo->currentGroupId++;
-    if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
-      setTaskStatus(pTaskInfo, TASK_COMPLETED);
-      return NULL;
+      int32_t num = 0;
+      STableKeyInfo* pList = NULL;
+      getTablesOfGroup(&pTaskInfo->tableqinfoList, pInfo->currentGroupId, &pList, &num);
+      ASSERT(pInfo->dataReader == NULL);
+
+      int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, (STsdbReader**)&pInfo->dataReader,
+                                    GET_TASKID(pTaskInfo));
+      if (code != TSDB_CODE_SUCCESS) {
+        T_LONG_JMP(pTaskInfo->env, code);
+      }
     }
+
     SSDataBlock* result = doTableScanGroup(pOperator);
     if (result != NULL) {
       ASSERT(result->info.uid != 0);
       return result;
     }
 
-
-    int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
-                                  GET_TASKID(pTaskInfo));
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pTaskInfo->env, code);
+    if ((++pInfo->currentGroupId) >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)) {
+      doSetOperatorCompleted(pOperator);
       return NULL;
     }
-  }
 
-  SSDataBlock* result = doTableScanGroup(pOperator);
-  if (result) {
-    return result;
-  }
+    // reset value for the next group data output
+    pOperator->status = OP_OPENED;
+    pInfo->limitInfo.numOfOutputRows = 0;
+    pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset;
 
-  pInfo->currentGroupId++;
-  if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
-    setTaskStatus(pTaskInfo, TASK_COMPLETED);
+    int32_t num = 0;
+    STableKeyInfo* pList = NULL;
+    getTablesOfGroup(&pTaskInfo->tableqinfoList, pInfo->currentGroupId, &pList, &num);
+
+    tsdbSetTableList(pInfo->dataReader, pList, num);
+    tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
+    pInfo->scanTimes = 0;
+
+    result = doTableScanGroup(pOperator);
+    if (result != NULL) {
+      return result;
+    }
+
+    doSetOperatorCompleted(pOperator);
     return NULL;
   }
-
-  tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
-  pInfo->scanTimes = 0;
-
-  result = doTableScanGroup(pOperator);
-  if (result) {
-    return result;
-  }
-
-  setTaskStatus(pTaskInfo, TASK_COMPLETED);
-  return NULL;
 }
 
 static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
@@ -819,8 +819,8 @@ static void destroyTableScanOperatorInfo(void* param) {
   taosMemoryFreeClear(param);
 }
 
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
-                                           SExecTaskInfo* pTaskInfo) {
+SOperatorInfo*  createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
+                                            SExecTaskInfo* pTaskInfo) {
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
   SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -837,7 +837,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   }
 
   initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
-
   code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -885,7 +884,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pOperator->cost.openCost = 0;
   return pOperator;
 
-_error:
+  _error:
   if (pInfo != NULL) {
     destroyTableScanOperatorInfo(pInfo);
   }
@@ -1033,7 +1032,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re
       createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL);
   return pOperator;
 
-_error:
+  _error:
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
   return NULL;
@@ -1077,39 +1076,55 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
   pTableScanInfo->cond.twindows = *pWin;
   pTableScanInfo->scanTimes = 0;
   pTableScanInfo->currentGroupId = -1;
-}
-
-static void freeArray(void* array) { taosArrayDestroy(array); }
-
-static void resetTableScanOperator(SOperatorInfo* pTableScanOp) {
-  STableScanInfo* pTableScanInfo = pTableScanOp->info;
-  pTableScanInfo->cond.startVersion = -1;
-  pTableScanInfo->cond.endVersion = -1;
-  SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
-  SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList;
-  taosArrayClearP(gpTbls, freeArray);
-  taosArrayPush(gpTbls, &allTbls);
-  STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
-  resetTableScanInfo(pTableScanOp->info, &win);
+  tsdbReaderClose(pTableScanInfo->dataReader);
+  pTableScanInfo->dataReader = NULL;
 }
 
 static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
                                        int64_t maxVersion) {
-  SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
-  taosArrayClear(gpTbls);
   STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
-  SArray*       tbls = taosArrayInit(1, sizeof(STableKeyInfo));
-  taosArrayPush(tbls, &tblInfo);
-  taosArrayPush(gpTbls, &tbls);
 
-  STimeWindow     win = {.skey = startTs, .ekey = endTs};
-  STableScanInfo* pTableScanInfo = pTableScanOp->info;
-  pTableScanInfo->cond.startVersion = -1;
-  pTableScanInfo->cond.endVersion = maxVersion;
-  resetTableScanInfo(pTableScanOp->info, &win);
-  SSDataBlock* pRes = doTableScan(pTableScanOp);
-  resetTableScanOperator(pTableScanOp);
-  return pRes;
+  STableScanInfo*     pTableScanInfo = pTableScanOp->info;
+  SQueryTableDataCond cond = pTableScanInfo->cond;
+
+  cond.startVersion = -1;
+  cond.endVersion = maxVersion;
+  cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs};
+
+  SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
+
+  SSDataBlock* pBlock = pTableScanInfo->pResBlock;
+  blockDataCleanup(pBlock);
+
+  STsdbReader* pReader = NULL;
+  int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
+                                GET_TASKID(pTaskInfo));
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    return NULL;
+  }
+
+  bool hasBlock = tsdbNextDataBlock(pReader);
+  if (hasBlock) {
+    SDataBlockInfo binfo = {0};
+    tsdbRetrieveDataBlockInfo(pReader, &binfo);
+
+    SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
+    blockDataEnsureCapacity(pBlock, binfo.rows);
+
+    pBlock->info.window = binfo.window;
+    pBlock->info.uid = binfo.uid;
+    pBlock->info.rows = binfo.rows;
+
+    relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
+    doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
+  }
+
+  tsdbReaderClose(pReader);
+  qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
+             ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
+
+  return pBlock->info.rows > 0 ? pBlock : NULL;
 }
 
 static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
@@ -1300,9 +1315,6 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
     SSessionKey startWin = {0};
     getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin);
     if (IS_INVALID_SESSION_WIN_KEY(startWin)) {
-      // char* tmp = streamStateSessionDump(pInfo->windowSup.pStreamAggSup->pState);
-      // qInfo("%s", tmp);
-      // taosMemoryFree(tmp);
       // window has been closed.
       continue;
     }
@@ -1699,7 +1711,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
       }
     }
 #if 0
-  } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
     SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
     if (pResult && pResult->info.rows > 0) {
       qDebug("stream scan tsdb return %d rows", pResult->info.rows);
@@ -1847,7 +1859,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
 
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
   // TODO: refactor
-FETCH_NEXT_BLOCK:
+  FETCH_NEXT_BLOCK:
   if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
     if (pInfo->validBlockIndex >= total) {
       doClearBufferedBlocks(pInfo);
@@ -1974,7 +1986,7 @@ FETCH_NEXT_BLOCK:
 
     int32_t totBlockNum = taosArrayGetSize(pInfo->pBlockLists);
 
-  NEXT_SUBMIT_BLK:
+    NEXT_SUBMIT_BLK:
     while (1) {
       if (pInfo->tqReader->pMsg == NULL) {
         if (pInfo->validBlockIndex >= totBlockNum) {
@@ -2229,7 +2241,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
   pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL);
   return pOperator;
 
-_end:
+  _end:
   taosMemoryFree(pInfo);
   taosMemoryFree(pOperator);
   pTaskInfo->code = code;
@@ -2335,11 +2347,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
       pTSInfo->cond.endVersion = pHandle->version;
     }
 
-    SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
+    STableKeyInfo* pList = NULL;
+    int32_t num = 0;
+    getTablesOfGroup(&pTaskInfo->tableqinfoList, 0, &pList, &num);
+
     if (pHandle->initTableReader) {
       pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
       pTSInfo->dataReader = NULL;
-      if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, tableList, &pTSInfo->dataReader, NULL) < 0) {
+      if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         goto _error;
       }
@@ -2367,7 +2382,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     // set the extract column id to streamHandle
     tqReaderSetColIdList(pInfo->tqReader, pColIds);
     SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
-    int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
+    code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
     if (code != 0) {
       taosArrayDestroy(tableIdList);
       goto _error;
@@ -2411,7 +2426,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
 
   return pOperator;
 
-_error:
+  _error:
   if (pColIds != NULL) {
     taosArrayDestroy(pColIds);
   }
@@ -3035,11 +3050,11 @@ static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
   if (func == NULL) return -1;
 
   SMetaFltParam param = {.suid = 0,
-                         .cid = 0,
-                         .type = TSDB_DATA_TYPE_VARCHAR,
-                         .val = pVal->datum.p,
-                         .reverse = reverse,
-                         .filterFunc = func};
+      .cid = 0,
+      .type = TSDB_DATA_TYPE_VARCHAR,
+      .val = pVal->datum.p,
+      .reverse = reverse,
+      .filterFunc = func};
   return -1;
 }
 
@@ -3054,11 +3069,11 @@ static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
   if (func == NULL) return -1;
 
   SMetaFltParam param = {.suid = 0,
-                         .cid = 0,
-                         .type = TSDB_DATA_TYPE_BIGINT,
-                         .val = &pVal->datum.i,
-                         .reverse = reverse,
-                         .filterFunc = func};
+      .cid = 0,
+      .type = TSDB_DATA_TYPE_BIGINT,
+      .val = &pVal->datum.i,
+      .reverse = reverse,
+      .filterFunc = func};
 
   int32_t ret = metaFilterCreateTime(pMeta, &param, result);
   return ret;
@@ -3252,80 +3267,18 @@ static int tableUidCompare(const void* a, const void* b) {
   }
   return u1 < u2 ? -1 : 1;
 }
-
-typedef struct MergeIndex {
-  int idx;
-  int len;
-} MergeIndex;
-
-static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) {
-  uint64_t v;
-  int32_t  m;
-  while (s <= e) {
-    m = s + (e - s) / 2;
-    v = *(uint64_t*)taosArrayGet(arr, m);
-    if (v >= k) {
-      e = m - 1;
-    } else {
-      s = m + 1;
-    }
-  }
-  return s;
-}
-
-void optSysIntersection(SArray* in, SArray* out) {
-  int32_t sz = (int32_t)taosArrayGetSize(in);
-  if (sz <= 0) {
-    return;
-  }
-  MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
-  for (int i = 0; i < sz; i++) {
-    SArray* t = taosArrayGetP(in, i);
-    mi[i].len = (int32_t)taosArrayGetSize(t);
-    mi[i].idx = 0;
-  }
-
-  SArray* base = taosArrayGetP(in, 0);
-  for (int i = 0; i < taosArrayGetSize(base); i++) {
-    uint64_t tgt = *(uint64_t*)taosArrayGet(base, i);
-    bool     has = true;
-    for (int j = 1; j < taosArrayGetSize(in); j++) {
-      SArray* oth = taosArrayGetP(in, j);
-      int     mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
-      if (mid >= 0 && mid < mi[j].len) {
-        uint64_t val = *(uint64_t*)taosArrayGet(oth, mid);
-        has = (val == tgt ? true : false);
-        mi[j].idx = mid;
-      } else {
-        has = false;
-      }
-    }
-    if (has == true) {
-      taosArrayPush(out, &tgt);
-    }
-  }
-  taosMemoryFreeClear(mi);
-}
-
 static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) {
   // TODO, find comm mem from mRslt
   for (int i = 0; i < taosArrayGetSize(mRslt); i++) {
-    SArray* arslt = taosArrayGetP(mRslt, i);
-    taosArraySort(arslt, tableUidCompare);
-  }
-  optSysIntersection(mRslt, rslt);
-  return 0;
-}
-static int32_t optSysSpecialColumn(SNode* cond) {
-  SOperatorNode* pOper = (SOperatorNode*)cond;
-  SColumnNode*   pCol = (SColumnNode*)pOper->pLeft;
-  for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) {
-    if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) {
-      return 1;
-    }
+    SArray* aRslt = taosArrayGetP(mRslt, i);
+    taosArrayAddAll(rslt, aRslt);
   }
+  taosArraySort(rslt, tableUidCompare);
+  taosArrayRemoveDuplicate(rslt, tableUidCompare, NULL);
+
   return 0;
 }
+
 static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
   int ret = -1;
   if (nodeType(cond) == QUERY_NODE_OPERATOR) {
@@ -3349,6 +3302,7 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
   SNodeList*           pList = (SNodeList*)pNode->pParameterList;
 
   int32_t len = LIST_LENGTH(pList);
+  if (len <= 0) return ret;
 
   bool    hasIdx = false;
   bool    hasRslt = true;
@@ -3359,16 +3313,12 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
     if (cell == NULL) break;
 
     SArray* aRslt = taosArrayInit(16, sizeof(int64_t));
+
     ret = optSysTabFilteImpl(arg, cell->pNode, aRslt);
     if (ret == 0) {
       // has index
       hasIdx = true;
-      if (optSysSpecialColumn(cell->pNode) == 0) {
-        taosArrayPush(mRslt, &aRslt);
-      } else {
-        // db_name/vgroup not result
-        taosArrayDestroy(aRslt);
-      }
+      taosArrayPush(mRslt, &aRslt);
     } else if (ret == -2) {
       // current vg
       hasIdx = true;
@@ -4043,7 +3993,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
 
   return pOperator;
 
-_error:
+  _error:
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
   terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -4153,6 +4103,9 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
   int32_t    code =
       extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
 
   code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
   if (code != TSDB_CODE_SUCCESS) {
@@ -4179,7 +4132,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
 
   return pOperator;
 
-_error:
+  _error:
   taosMemoryFree(pInfo);
   taosMemoryFree(pOperator);
   terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -4210,8 +4163,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return TSDB_CODE_SUCCESS;
   }
 
-  pTableListInfo->needSortTableByGroupId = groupSort;
-  code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags);
+  pTableListInfo->numOfOuputGroups = 1;
+  code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags, groupSort);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -4225,144 +4178,15 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
 int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
                                   int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, const char* idstr) {
   for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
-    SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
-    taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
-
-    STsdbReader* pReader = NULL;
-    tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, &pReader, idstr);
+    STableKeyInfo* pList = taosArrayGet(pTableListInfo->pTableList, i);
+    STsdbReader*   pReader = NULL;
+    tsdbReaderOpen(pHandle->vnode, pQueryCond, pList, 1, &pReader, idstr);
     taosArrayPush(arrayReader, &pReader);
-
-    taosArrayDestroy(subTableList);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
-                                   STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
-                                   STsdbReader** ppReader, const char* idstr) {
-  STsdbReader* pReader = NULL;
-  SArray*      subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
-  for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
-    taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
-  }
-  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, &pReader, idstr);
-  if (code != 0) {
-    taosArrayDestroy(subTableList);
-    return code;
-  }
-  *ppReader = pReader;
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
-                                          SSDataBlock* pBlock, uint32_t* status) {
-  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
-  STableMergeScanInfo* pInfo = pOperator->info;
-
-  uint64_t uid = pBlock->info.uid;
-
-  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
-
-  pCost->totalBlocks += 1;
-  pCost->totalRows += pBlock->info.rows;
-
-  *status = pInfo->dataBlockLoadFlag;
-  if (pTableScanInfo->pFilterNode != NULL ||
-      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
-    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
-  }
-
-  SDataBlockInfo* pBlockInfo = &pBlock->info;
-  taosMemoryFreeClear(pBlock->pBlockAgg);
-
-  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
-    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->filterOutBlocks += 1;
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
-    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->skipBlocks += 1;
-
-    // clear all data in pBlock that are set when handing the previous block
-    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
-      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
-      pcol->pData = NULL;
-    }
-
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
-    pCost->loadBlockStatis += 1;
-
-    bool             allColumnsHaveAgg = true;
-    SColumnDataAgg** pColAgg = NULL;
-    STsdbReader*     reader = pTableScanInfo->pReader;
-    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
-
-    if (allColumnsHaveAgg == true) {
-      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
-      // todo create this buffer during creating operator
-      if (pBlock->pBlockAgg == NULL) {
-        pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-      }
-
-      for (int32_t i = 0; i < numOfCols; ++i) {
-        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
-        if (!pColMatchInfo->needOutput) {
-          continue;
-        }
-        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
-      }
-
-      return TSDB_CODE_SUCCESS;
-    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
-      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
-    }
-  }
-
-  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
-
-  pCost->totalCheckedRows += pBlock->info.rows;
-  pCost->loadBlocks += 1;
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
-  if (pCols == NULL) {
-    return terrno;
-  }
-
-  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
-
-  // currently only the tbname pseudo column
-  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
-                                          pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo));
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pTaskInfo->env, code);
-    }
-  }
-
-  if (pTableScanInfo->pFilterNode != NULL) {
-    int64_t st = taosGetTimestampMs();
-    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
-
-    double el = (taosGetTimestampUs() - st) / 1000.0;
-    pTableScanInfo->readRecorder.filterTime += el;
-
-    if (pBlock->info.rows == 0) {
-      pCost->filterOutBlocks += 1;
-      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
-             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
-    } else {
-      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -4446,7 +4270,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
   pCost->totalCheckedRows += pBlock->info.rows;
   pCost->loadBlocks += 1;
 
-  STsdbReader* reader = pTableScanInfo->pReader;  // taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
+  STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
   SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
   if (pCols == NULL) {
     return terrno;
@@ -4485,143 +4309,9 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 typedef struct STableMergeScanSortSourceParam {
   SOperatorInfo* pOperator;
   int32_t        readerIdx;
-  int64_t        uid;
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
-static SSDataBlock* getTableDataBlockTemp(void* param) {
-  STableMergeScanSortSourceParam* source = param;
-  SOperatorInfo*                  pOperator = source->pOperator;
-  STableMergeScanInfo*            pInfo = pOperator->info;
-  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
-  int32_t                         readIdx = source->readerIdx;
-  SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
-
-  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
-
-  blockDataCleanup(pBlock);
-
-  int64_t st = taosGetTimestampUs();
-
-  SArray* subTable = taosArrayInit(1, sizeof(STableKeyInfo));
-  taosArrayPush(subTable, taosArrayGet(pInfo->tableListInfo->pTableList, readIdx + pInfo->tableStartIndex));
-  SReadHandle* pHandle = &pInfo->readHandle;
-  tsdbReaderOpen(pHandle->vnode, pQueryCond, subTable, &pInfo->pReader, GET_TASKID(pTaskInfo));
-  taosArrayDestroy(subTable);
-
-  STsdbReader* reader = pInfo->pReader;
-  while (tsdbNextDataBlock(reader)) {
-    if (isTaskKilled(pOperator->pTaskInfo)) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-    }
-
-    // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
-    if (!processThisBlock) {
-      continue;
-    }
-
-    blockDataCleanup(pBlock);
-    SDataBlockInfo binfo = pBlock->info;
-    tsdbRetrieveDataBlockInfo(reader, &binfo);
-
-    blockDataEnsureCapacity(pBlock, binfo.rows);
-    pBlock->info.type = binfo.type;
-    pBlock->info.uid = binfo.uid;
-    pBlock->info.window = binfo.window;
-    pBlock->info.rows = binfo.rows;
-
-    if (tsdbIsAscendingOrder(pInfo->pReader)) {
-      pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
-    } else {
-      pQueryCond->twindows.ekey = pBlock->info.window.skey - 1;
-    }
-
-    uint32_t status = 0;
-    int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readIdx, pBlock, &status);
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, code);
-    }
-
-    // current block is filter out according to filter condition, continue load the next block
-    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
-      continue;
-    }
-
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
-    pOperator->resultInfo.totalRows += pBlock->info.rows;  // pTableScanInfo->readRecorder.totalRows;
-    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
-
-    tsdbReaderClose(pInfo->pReader);
-    pInfo->pReader = NULL;
-    return pBlock;
-  }
-  tsdbReaderClose(pInfo->pReader);
-  pInfo->pReader = NULL;
-  return NULL;
-}
-static SSDataBlock* getTableDataBlock2(void* param) {
-  STableMergeScanSortSourceParam* source = param;
-  SOperatorInfo*                  pOperator = source->pOperator;
-  int64_t                         uid = source->uid;
-  SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
-
-  int64_t st = taosGetTimestampUs();
-
-  blockDataCleanup(pBlock);
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  while (tsdbTableNextDataBlock(reader, uid)) {
-    if (isTaskKilled(pOperator->pTaskInfo)) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-    }
-
-    // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
-    if (!processThisBlock) {
-      continue;
-    }
-
-    blockDataCleanup(pBlock);
-    SDataBlockInfo binfo = pBlock->info;
-    tsdbRetrieveDataBlockInfo(reader, &binfo);
-
-    blockDataEnsureCapacity(pBlock, binfo.rows);
-    pBlock->info.type = binfo.type;
-    pBlock->info.uid = binfo.uid;
-    pBlock->info.window = binfo.window;
-    pBlock->info.rows = binfo.rows;
-
-    uint32_t status = 0;
-    int32_t  code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, code);
-    }
-
-    // current block is filter out according to filter condition, continue load the next block
-    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
-      continue;
-    }
-
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
-    pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
-    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
-
-    return pBlock;
-  }
-  return NULL;
-}
-
 static SSDataBlock* getTableDataBlock(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;
@@ -4657,6 +4347,7 @@ static SSDataBlock* getTableDataBlock(void* param) {
 
     uint32_t status = 0;
     int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status);
+    //    int32_t  code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
     if (code != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pOperator->pTaskInfo->env, code);
     }
@@ -4699,14 +4390,6 @@ SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
   return pList;
 }
 
-int32_t dumpSQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
-  memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
-  dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
-  for (int i = 0; i < src->numOfCols; i++) {
-    dst->colList[i] = src->colList[i];
-  }
-  return 0;
-}
 int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -4727,9 +4410,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   int32_t tableEndIdx = pInfo->tableEndIndex;
 
   STableListInfo* tableListInfo = pInfo->tableListInfo;
+  pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
+  createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
+                            pInfo->dataReaders, GET_TASKID(pTaskInfo));
 
-  // pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
-  pInfo->pReader = NULL;
   // todo the total available buffer should be determined by total capacity of buffer of this task.
   // the additional one is reserved for merge result
   pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
@@ -4737,27 +4421,18 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
                                              pInfo->pSortInputBlock, pTaskInfo->id.str);
 
-  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL);
-
-  // one table has one data block
-  int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
-  pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond));
-
-  for (int32_t i = 0; i < numOfTable; ++i) {
-    STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i + tableStartIdx);
+  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL);
 
+  size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
+  for (int32_t i = 0; i < numReaders; ++i) {
     STableMergeScanSortSourceParam param = {0};
     param.readerIdx = i;
     param.pOperator = pOperator;
     param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
     taosArrayPush(pInfo->sortSourceParams, &param);
-
-    SQueryTableDataCond cond;
-    dumpSQueryTableCond(&pInfo->cond, &cond);
-    taosArrayPush(pInfo->queryConds, &cond);
   }
 
-  for (int32_t i = 0; i < numOfTable; ++i) {
+  for (int32_t i = 0; i < numReaders; ++i) {
     SSortSource*                    ps = taosMemoryCalloc(1, sizeof(SSortSource));
     STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
     ps->param = param;
@@ -4777,7 +4452,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
 
-  int32_t numOfTable = taosArrayGetSize(pInfo->queryConds);
+  size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
 
   SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
   pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
@@ -4786,7 +4461,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
   pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
 
-  for (int32_t i = 0; i < numOfTable; ++i) {
+  for (int32_t i = 0; i < numReaders; ++i) {
     STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
     blockDataDestroy(param->inputBlock);
   }
@@ -4794,13 +4469,12 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
 
   tsortDestroySortHandle(pInfo->pSortHandle);
 
-  for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) {
-    SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i);
-    taosMemoryFree(cond->colList);
+  for (int32_t i = 0; i < numReaders; ++i) {
+    STsdbReader* reader = taosArrayGetP(pInfo->dataReaders, i);
+    tsdbReaderClose(reader);
   }
-  taosArrayDestroy(pInfo->queryConds);
-  pInfo->queryConds = NULL;
-
+  taosArrayDestroy(pInfo->dataReaders);
+  pInfo->dataReaders = NULL;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -4884,23 +4558,13 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
 void destroyTableMergeScanOperatorInfo(void* param) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->cond);
-
-  int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds);
-
-  for (int32_t i = 0; i < numOfTable; i++) {
-    STableMergeScanSortSourceParam* param = taosArrayGet(pTableScanInfo->sortSourceParams, i);
-    blockDataDestroy(param->inputBlock);
-  }
   taosArrayDestroy(pTableScanInfo->sortSourceParams);
 
-  tsdbReaderClose(pTableScanInfo->pReader);
-  pTableScanInfo->pReader = NULL;
-
-  for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) {
-    SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
-    taosMemoryFree(pCond->colList);
+  for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) {
+    STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, i);
+    tsdbReaderClose(reader);
   }
-  taosArrayDestroy(pTableScanInfo->queryConds);
+  taosArrayDestroy(pTableScanInfo->dataReaders);
 
   if (pTableScanInfo->matchInfo.pList != NULL) {
     taosArrayDestroy(pTableScanInfo->matchInfo.pList);
@@ -5012,7 +4676,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pOperator->cost.openCost = 0;
   return pOperator;
 
-_error:
+  _error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   taosMemoryFree(pInfo);
   taosMemoryFree(pOperator);

From 029a4a85b318ca4f4643bf0ececeab2f81caefe1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 22:21:02 +0800
Subject: [PATCH 21/42] fix(query): support partition by + limit push down.

---
 source/dnode/vnode/src/sma/smaRollup.c | 12 ++++++------
 source/libs/executor/src/executil.c    |  8 ++++++--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 19a0fbd629..0ac5f0d8ff 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -706,20 +706,20 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
 
       // TODO: the schema update should be handled later(TD-17965)
       if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) {
-        smaError("vgId:%d, build submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
-                 suid, pItem->level, terrstr());
+        smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%"PRIu64", level %" PRIi8 " failed since %s", SMA_VID(pSma),
+                 suid, output->info.groupId, pItem->level, terrstr());
         goto _err;
       }
 
       if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
         taosMemoryFreeClear(pReq);
-        smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s",
-                 SMA_VID(pSma), suid, pItem->level, terrstr());
+        smaError("vgId:%d, process submit req for rsma table suid:%" PRIu64 ", uid:%"PRIu64", level %" PRIi8 " failed since %s",
+                 SMA_VID(pSma), suid, output->info.groupId, pItem->level, terrstr());
         goto _err;
       }
 
-      smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
-               SMA_VID(pSma), suid, pItem->level, output->info.version, htonl(pReq->header.contLen));
+      smaDebug("vgId:%d, process submit req for rsma table suid:%" PRIu64 ", uid:%"PRIu64", level %" PRIi8 " ver %" PRIi64 " len %" PRIu32,
+               SMA_VID(pSma), suid, output->info.groupId, pItem->level, output->info.version, htonl(pReq->header.contLen));
 
       taosMemoryFreeClear(pReq);
     }
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index dbef032041..6bce88fa23 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1679,11 +1679,15 @@ uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
 
 int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid) {
   STableKeyInfo keyInfo = {.uid = uid, .groupId = gid};
-
   taosArrayPush(pTableList->pTableList, &keyInfo);
-  if (pTableList->oneTableForEachGroup || pTableList->numOfOuputGroups > 1) {
+  if (!pTableList->oneTableForEachGroup) {
+    if (pTableList->map == NULL) {
+      pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+    }
+
     taosHashPut(pTableList->map, &uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
   }
+
   return TSDB_CODE_SUCCESS;
 }
 

From 61634070ec9ce92c18839456374ea822e131edba Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 23:23:53 +0800
Subject: [PATCH 22/42] other: merge 3.0

---
 source/dnode/vnode/inc/vnode.h          |   1 -
 source/libs/executor/src/scanoperator.c | 409 ++++++++++++++++++++++--
 2 files changed, 380 insertions(+), 30 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index fcb9b4ab44..d7e4935edd 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -165,7 +165,6 @@ SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdLi
 int32_t  tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
 int32_t  tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
 int64_t  tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
-bool     tsdbIsAscendingOrder(STsdbReader *pReader);
 void    *tsdbGetIdx(SMeta *pMeta);
 void    *tsdbGetIvtIdx(SMeta *pMeta);
 uint64_t getReaderMaxVersion(STsdbReader *pReader);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index a139809d8f..5a860c3b95 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -42,7 +42,7 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S
 static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name",     "create_time",      "columns",
                                       "ttl",        "stable_name", "vgroup_id', 'uid", "type"};
 
-static char* SYSTABLE_IDX_EXCEPT[] = {"db_name", "vgroup_id"};
+static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"};
 
 typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
 typedef int32_t (*__sys_check)(SNode* cond);
@@ -3267,15 +3267,79 @@ static int tableUidCompare(const void* a, const void* b) {
   }
   return u1 < u2 ? -1 : 1;
 }
+
+typedef struct MergeIndex {
+  int idx;
+  int len;
+} MergeIndex;
+
+static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) {
+  uint64_t v;
+  int32_t  m;
+  while (s <= e) {
+    m = s + (e - s) / 2;
+    v = *(uint64_t*)taosArrayGet(arr, m);
+    if (v >= k) {
+      e = m - 1;
+    } else {
+      s = m + 1;
+    }
+  }
+  return s;
+}
+
+void optSysIntersection(SArray* in, SArray* out) {
+  int32_t sz = (int32_t)taosArrayGetSize(in);
+  if (sz <= 0) {
+    return;
+  }
+  MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
+  for (int i = 0; i < sz; i++) {
+    SArray* t = taosArrayGetP(in, i);
+    mi[i].len = (int32_t)taosArrayGetSize(t);
+    mi[i].idx = 0;
+  }
+
+  SArray* base = taosArrayGetP(in, 0);
+  for (int i = 0; i < taosArrayGetSize(base); i++) {
+    uint64_t tgt = *(uint64_t*)taosArrayGet(base, i);
+    bool     has = true;
+    for (int j = 1; j < taosArrayGetSize(in); j++) {
+      SArray* oth = taosArrayGetP(in, j);
+      int     mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
+      if (mid >= 0 && mid < mi[j].len) {
+        uint64_t val = *(uint64_t*)taosArrayGet(oth, mid);
+        has = (val == tgt ? true : false);
+        mi[j].idx = mid;
+      } else {
+        has = false;
+      }
+    }
+    if (has == true) {
+      taosArrayPush(out, &tgt);
+    }
+  }
+  taosMemoryFreeClear(mi);
+}
+
 static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) {
   // TODO, find comm mem from mRslt
   for (int i = 0; i < taosArrayGetSize(mRslt); i++) {
-    SArray* aRslt = taosArrayGetP(mRslt, i);
-    taosArrayAddAll(rslt, aRslt);
+    SArray* arslt = taosArrayGetP(mRslt, i);
+    taosArraySort(arslt, tableUidCompare);
   }
-  taosArraySort(rslt, tableUidCompare);
-  taosArrayRemoveDuplicate(rslt, tableUidCompare, NULL);
+  optSysIntersection(mRslt, rslt);
+  return 0;
+}
 
+static int32_t optSysSpecialColumn(SNode* cond) {
+  SOperatorNode* pOper = (SOperatorNode*)cond;
+  SColumnNode*   pCol = (SColumnNode*)pOper->pLeft;
+  for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) {
+    if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) {
+      return 1;
+    }
+  }
   return 0;
 }
 
@@ -3302,7 +3366,6 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
   SNodeList*           pList = (SNodeList*)pNode->pParameterList;
 
   int32_t len = LIST_LENGTH(pList);
-  if (len <= 0) return ret;
 
   bool    hasIdx = false;
   bool    hasRslt = true;
@@ -3318,7 +3381,12 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
     if (ret == 0) {
       // has index
       hasIdx = true;
-      taosArrayPush(mRslt, &aRslt);
+      if (optSysSpecialColumn(cell->pNode) == 0) {
+        taosArrayPush(mRslt, &aRslt);
+      } else {
+        // db_name/vgroup not result
+        taosArrayDestroy(aRslt);
+      }
     } else if (ret == -2) {
       // current vg
       hasIdx = true;
@@ -3389,7 +3457,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
 
     SMetaReader mr = {0};
     metaReaderInit(&mr, pInfo->readHandle.meta, 0);
-    int32_t ret = metaGetTableEntryByUid(&mr, *uid);
+    ret = metaGetTableEntryByUid(&mr, *uid);
     if (ret < 0) {
       metaReaderClear(&mr);
       continue;
@@ -4187,6 +4255,130 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
+                                   STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
+                                   STsdbReader** ppReader, const char* idstr) {
+  STsdbReader* pReader = NULL;
+  void*   pStart = taosArrayGet(pTableListInfo->pTableList, tableStartIdx);
+  int32_t num = tableEndIdx - tableStartIdx + 1;
+
+  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr);
+  if (code != 0) {
+    return code;
+  }
+
+  *ppReader = pReader;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
+                                          SSDataBlock* pBlock, uint32_t* status) {
+  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
+  STableMergeScanInfo* pInfo = pOperator->info;
+
+  uint64_t uid = pBlock->info.uid;
+
+  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
+
+  pCost->totalBlocks += 1;
+  pCost->totalRows += pBlock->info.rows;
+
+  *status = pInfo->dataBlockLoadFlag;
+  if (pTableScanInfo->pFilterNode != NULL ||
+      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
+    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
+  }
+
+  SDataBlockInfo* pBlockInfo = &pBlock->info;
+  taosMemoryFreeClear(pBlock->pBlockAgg);
+
+  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->filterOutBlocks += 1;
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
+    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->skipBlocks += 1;
+
+    // clear all data in pBlock that are set when handing the previous block
+    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
+      pcol->pData = NULL;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
+    pCost->loadBlockStatis += 1;
+
+    bool             allColumnsHaveAgg = true;
+    SColumnDataAgg** pColAgg = NULL;
+    STsdbReader*     reader = pTableScanInfo->pReader;
+    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
+
+    if (allColumnsHaveAgg == true) {
+      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+
+      // todo create this buffer during creating operator
+      if (pBlock->pBlockAgg == NULL) {
+        pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
+      }
+
+      for (int32_t i = 0; i < numOfCols; ++i) {
+        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
+        if (!pColMatchInfo->needOutput) {
+          continue;
+        }
+        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
+      }
+
+      return TSDB_CODE_SUCCESS;
+    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
+      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
+    }
+  }
+
+  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
+
+  pCost->totalCheckedRows += pBlock->info.rows;
+  pCost->loadBlocks += 1;
+
+  STsdbReader* reader = pTableScanInfo->pReader;
+  SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
+  if (pCols == NULL) {
+    return terrno;
+  }
+
+  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
+
+  // currently only the tbname pseudo column
+  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
+                                          pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo));
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pTaskInfo->env, code);
+    }
+  }
+
+  if (pTableScanInfo->pFilterNode != NULL) {
+    int64_t st = taosGetTimestampMs();
+    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
+
+    double el = (taosGetTimestampUs() - st) / 1000.0;
+    pTableScanInfo->readRecorder.filterTime += el;
+
+    if (pBlock->info.rows == 0) {
+      pCost->filterOutBlocks += 1;
+      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
+             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
+    } else {
+      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -4229,8 +4421,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 
     bool             allColumnsHaveAgg = true;
     SColumnDataAgg** pColAgg = NULL;
-    STsdbReader*     reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
-    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
+//    STsdbReader* reader = pTableScanInfo->pReader; // taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
 
     if (allColumnsHaveAgg == true) {
       int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@@ -4309,9 +4500,141 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 typedef struct STableMergeScanSortSourceParam {
   SOperatorInfo* pOperator;
   int32_t        readerIdx;
+  uint64_t       uid;
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
+static SSDataBlock* getTableDataBlockTemp(void* param) {
+  STableMergeScanSortSourceParam* source = param;
+  SOperatorInfo*                  pOperator = source->pOperator;
+  STableMergeScanInfo*            pInfo = pOperator->info;
+  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  int32_t                         readIdx = source->readerIdx;
+  SSDataBlock*                    pBlock = source->inputBlock;
+  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
+
+  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
+
+  blockDataCleanup(pBlock);
+
+  int64_t st = taosGetTimestampUs();
+
+  void* p =taosArrayGet(pInfo->tableListInfo->pTableList, readIdx + pInfo->tableStartIndex);
+  SReadHandle* pHandle = &pInfo->readHandle;
+  tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
+
+  STsdbReader* reader = pInfo->pReader;
+  while (tsdbNextDataBlock(reader)) {
+    if (isTaskKilled(pOperator->pTaskInfo)) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+    }
+
+    // process this data block based on the probabilities
+    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    if (!processThisBlock) {
+      continue;
+    }
+
+    blockDataCleanup(pBlock);
+    SDataBlockInfo binfo = pBlock->info;
+    tsdbRetrieveDataBlockInfo(reader, &binfo);
+
+    blockDataEnsureCapacity(pBlock, binfo.rows);
+    pBlock->info.type = binfo.type;
+    pBlock->info.uid = binfo.uid;
+    pBlock->info.window = binfo.window;
+    pBlock->info.rows = binfo.rows;
+
+    if (pQueryCond->order == TSDB_ORDER_ASC) {
+      pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
+    } else {
+      pQueryCond->twindows.ekey = pBlock->info.window.skey - 1;
+    }
+
+    uint32_t status = 0;
+    int32_t  code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readIdx, pBlock, &status);
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, code);
+    }
+
+    // current block is filter out according to filter condition, continue load the next block
+    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
+      continue;
+    }
+
+    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
+    if (groupId) {
+      pBlock->info.groupId = *groupId;
+    }
+
+    pOperator->resultInfo.totalRows += pBlock->info.rows;  // pTableScanInfo->readRecorder.totalRows;
+    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+
+    tsdbReaderClose(pInfo->pReader);
+    pInfo->pReader = NULL;
+    return pBlock;
+  }
+  tsdbReaderClose(pInfo->pReader);
+  pInfo->pReader = NULL;
+  return NULL;
+}
+static SSDataBlock* getTableDataBlock2(void* param) {
+  STableMergeScanSortSourceParam* source = param;
+  SOperatorInfo*                  pOperator = source->pOperator;
+  int64_t                         uid = source->uid;
+  SSDataBlock*                    pBlock = source->inputBlock;
+  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
+
+  int64_t st = taosGetTimestampUs();
+
+  blockDataCleanup(pBlock);
+
+  STsdbReader* reader = pTableScanInfo->pReader;
+  while (tsdbTableNextDataBlock(reader, uid)) {
+    if (isTaskKilled(pOperator->pTaskInfo)) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+    }
+
+    // process this data block based on the probabilities
+    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    if (!processThisBlock) {
+      continue;
+    }
+
+    blockDataCleanup(pBlock);
+    SDataBlockInfo binfo = pBlock->info;
+    tsdbRetrieveDataBlockInfo(reader, &binfo);
+
+    blockDataEnsureCapacity(pBlock, binfo.rows);
+    pBlock->info.type = binfo.type;
+    pBlock->info.uid = binfo.uid;
+    pBlock->info.window = binfo.window;
+    pBlock->info.rows = binfo.rows;
+
+    uint32_t status = 0;
+    int32_t  code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
+    if (code != TSDB_CODE_SUCCESS) {
+      T_LONG_JMP(pOperator->pTaskInfo->env, code);
+    }
+
+    // current block is filter out according to filter condition, continue load the next block
+    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
+      continue;
+    }
+
+    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
+    if (groupId) {
+      pBlock->info.groupId = *groupId;
+    }
+
+    pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
+    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+
+    return pBlock;
+  }
+  return NULL;
+}
+
 static SSDataBlock* getTableDataBlock(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;
@@ -4390,6 +4713,15 @@ SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
   return pList;
 }
 
+int32_t dumpSQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
+  memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
+  dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
+  for (int i = 0; i < src->numOfCols; i++) {
+    dst->colList[i] = src->colList[i];
+  }
+  return 0;
+}
+
 int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -4409,10 +4741,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   int32_t tableStartIdx = pInfo->tableStartIndex;
   int32_t tableEndIdx = pInfo->tableEndIndex;
 
-  STableListInfo* tableListInfo = pInfo->tableListInfo;
-  pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
-  createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
-                            pInfo->dataReaders, GET_TASKID(pTaskInfo));
+  pInfo->pReader = NULL;
 
   // todo the total available buffer should be determined by total capacity of buffer of this task.
   // the additional one is reserved for merge result
@@ -4421,18 +4750,28 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
                                              pInfo->pSortInputBlock, pTaskInfo->id.str);
 
-  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL);
+  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL);
+
+  // one table has one data block
+  int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
+  pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond));
+
+  for (int32_t i = 0; i < numOfTable; ++i) {
+    STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i + tableStartIdx);
+    size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
 
-  size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
-  for (int32_t i = 0; i < numReaders; ++i) {
     STableMergeScanSortSourceParam param = {0};
     param.readerIdx = i;
     param.pOperator = pOperator;
     param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
     taosArrayPush(pInfo->sortSourceParams, &param);
+
+    SQueryTableDataCond cond;
+    dumpSQueryTableCond(&pInfo->cond, &cond);
+    taosArrayPush(pInfo->queryConds, &cond);
   }
 
-  for (int32_t i = 0; i < numReaders; ++i) {
+  for (int32_t i = 0; i < numOfTable; ++i) {
     SSortSource*                    ps = taosMemoryCalloc(1, sizeof(SSortSource));
     STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
     ps->param = param;
@@ -4452,7 +4791,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
 
-  size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
+  int32_t numOfTable = taosArrayGetSize(pInfo->queryConds);
 
   SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
   pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
@@ -4461,7 +4800,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
   pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
 
-  for (int32_t i = 0; i < numReaders; ++i) {
+  for (int32_t i = 0; i < numOfTable; ++i) {
     STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
     blockDataDestroy(param->inputBlock);
   }
@@ -4469,12 +4808,13 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
 
   tsortDestroySortHandle(pInfo->pSortHandle);
 
-  for (int32_t i = 0; i < numReaders; ++i) {
-    STsdbReader* reader = taosArrayGetP(pInfo->dataReaders, i);
-    tsdbReaderClose(reader);
+  for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) {
+    SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i);
+    taosMemoryFree(cond->colList);
   }
-  taosArrayDestroy(pInfo->dataReaders);
-  pInfo->dataReaders = NULL;
+  taosArrayDestroy(pInfo->queryConds);
+  pInfo->queryConds = NULL;
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -4558,13 +4898,24 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
 void destroyTableMergeScanOperatorInfo(void* param) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->cond);
+
+  int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds);
+
+  for (int32_t i = 0; i < numOfTable; i++) {
+    STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i);
+    blockDataDestroy(p->inputBlock);
+  }
+
   taosArrayDestroy(pTableScanInfo->sortSourceParams);
 
-  for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) {
-    STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, i);
-    tsdbReaderClose(reader);
+  tsdbReaderClose(pTableScanInfo->pReader);
+  pTableScanInfo->pReader = NULL;
+
+  for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) {
+    SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
+    taosMemoryFree(pCond->colList);
   }
-  taosArrayDestroy(pTableScanInfo->dataReaders);
+  taosArrayDestroy(pTableScanInfo->queryConds);
 
   if (pTableScanInfo->matchInfo.pList != NULL) {
     taosArrayDestroy(pTableScanInfo->matchInfo.pList);

From d87e8f0250778c88614c428a441465c54821d766 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 23:28:53 +0800
Subject: [PATCH 23/42] other: update the command.

---
 contrib/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 969724cc8d..21eac42c79 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED})
         PREFIX        "jemalloc"
         SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
         BUILD_IN_SOURCE     1
-        CONFIGURE_COMMAND   ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto'
+        CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
         BUILD_COMMAND       ${MAKE}
     )
     INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)

From 317ee1daea5f807f40eb66ef0c4cb93b4cbc5df7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 28 Oct 2022 23:29:38 +0800
Subject: [PATCH 24/42] other: update the command.

---
 contrib/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 21eac42c79..8f94313382 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED})
         PREFIX        "jemalloc"
         SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
         BUILD_IN_SOURCE     1
-        CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
+        CONFIGURE_COMMAND  ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls
         BUILD_COMMAND       ${MAKE}
     )
     INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)

From 86a75bc844b6f4ecef769ee26ca1e0d3428954a3 Mon Sep 17 00:00:00 2001
From: Chait Diwadkar <94201190+cdiwadkar16@users.noreply.github.com>
Date: Fri, 28 Oct 2022 10:14:20 -0700
Subject: [PATCH 25/42] Need quotes around the location in TAGS
 ('California.SanFrancicso'

The insert fails without quotes around the location in the TAGS part
---
 docs/examples/c/insert_example.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/examples/c/insert_example.c b/docs/examples/c/insert_example.c
index ce8fdc5b93..a921f794fd 100644
--- a/docs/examples/c/insert_example.c
+++ b/docs/examples/c/insert_example.c
@@ -36,10 +36,10 @@ int main() {
   executeSQL(taos, "CREATE DATABASE power");
   executeSQL(taos, "USE power");
   executeSQL(taos, "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
-  executeSQL(taos, "INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)"
-                "d1002 USING meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)"
-                "d1003 USING meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000)"
-                "d1004 USING meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)");
+  executeSQL(taos, "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)"
+                "d1002 USING meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)"
+                "d1003 USING meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000)"
+                "d1004 USING meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)");
   taos_close(taos);
   taos_cleanup();
 }
@@ -48,4 +48,4 @@ int main() {
 // affected rows 0
 // affected rows 0
 // affected rows 0
-// affected rows 8
\ No newline at end of file
+// affected rows 8

From d7f3ddf2dd13b8f4c7377591c9361bae776649c8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 10:17:41 +0800
Subject: [PATCH 26/42] fix(query): set correct reader handle.

---
 source/libs/executor/src/scanoperator.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 5a860c3b95..c343b9f589 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4461,7 +4461,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
   pCost->totalCheckedRows += pBlock->info.rows;
   pCost->loadBlocks += 1;
 
-  STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx);
+  STsdbReader* reader = pTableScanInfo->pReader;
   SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
   if (pCols == NULL) {
     return terrno;

From f9aa286dc1c1da2068f2290c58cd9bd0d91277d2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 10:22:02 +0800
Subject: [PATCH 27/42] fix(query): fix syntax error on aarch64

---
 source/dnode/vnode/src/sma/smaRollup.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index bb12f853f1..03532eb6d4 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -701,7 +701,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
 #endif
     for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
       SSDataBlock *output = taosArrayGetP(pResList, i);
-      smaDebug("uid:%ld, groupid:%ld", output->info.uid, output->info.groupId);
+      smaDebug("result block, uid:%"PRIu64", groupid:%"PRIu64", rows:%d", output->info.uid, output->info.groupId,
+              output->info.rows);
 
       STsdb       *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
       SSubmitReq  *pReq = NULL;

From 4cd14c4163287b8a34012bfddaf96da5c50a2437 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 10:25:14 +0800
Subject: [PATCH 28/42] fix(query): fix memory leak.

---
 source/libs/executor/src/scanoperator.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index c343b9f589..6935a9ab93 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2295,6 +2295,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   int32_t numOfCols = 0;
   int32_t code =
       extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
 
   int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList);
   SArray* pColIds = taosArrayInit(numOfOutput, sizeof(int16_t));
@@ -2356,6 +2359,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
       pTSInfo->dataReader = NULL;
       if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
+        destroyTableScanOperatorInfo(pTableScanOp);
         goto _error;
       }
     }

From 783a65f827b8296eea20d41ec7dc6904f060ad45 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sat, 29 Oct 2022 10:57:15 +0800
Subject: [PATCH 29/42] refactor(sync) delete ASSERT

---
 source/libs/sync/src/syncMain.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index b77652bc81..481733cd17 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -3375,8 +3375,12 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde
           pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
         } else {
           code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
-          ASSERT(code == 0);
-          ASSERT(pEntry != NULL);
+          // ASSERT(code == 0);
+          // ASSERT(pEntry != NULL);
+          if (code != 0 || pEntry == NULL) {
+			syncNodeErrorLog(ths, "get log entry error");
+            continue;
+          }
         }
 
         SRpcMsg rpcMsg;
@@ -3737,4 +3741,4 @@ void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p
   snprintf(logBuf, sizeof(logBuf), "recv sync-heartbeat-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 "}, %s",
            host, port, pMsg->term, pMsg->privateTerm, s);
   syncNodeEventLog(pSyncNode, logBuf);
-}
\ No newline at end of file
+}

From bae6d824c2576e9fcc84ebade3992b6f93adb8f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 12:54:39 +0800
Subject: [PATCH 30/42] fix(query): init blockscanInfo correctly.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 17 ++++++++++-------
 tests/system-test/2-query/interp.py    |  2 +-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 5e5e287cf9..9add18a4dc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3478,6 +3478,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
 
   initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader);
   resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
+//  resetDataBlockScanInfo(pReader->status.pTableMap, pReader->window.skey);
 
   // no data in files, let's try buffer in memory
   if (pReader->status.fileIter.numOfFiles == 0) {
@@ -3731,13 +3732,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
 
   if (pReader->innerReader[0] != NULL && pReader->step == 0) {
     bool ret = doTsdbNextDataBlock(pReader->innerReader[0]);
-
-    // prepare for the main scan
-    int32_t code = doOpenReaderImpl(pReader);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-
     pReader->step = EXTERNAL_ROWS_PREV;
     if (ret) {
       return ret;
@@ -3745,6 +3739,14 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   }
 
   if (pReader->step == EXTERNAL_ROWS_PREV) {
+    // prepare for the main scan
+    int32_t code = doOpenReaderImpl(pReader);
+    resetDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey);
+
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
     pReader->step = EXTERNAL_ROWS_MAIN;
   }
 
@@ -3756,6 +3758,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   if (pReader->innerReader[1] != NULL && pReader->step == EXTERNAL_ROWS_MAIN) {
     // prepare for the next row scan
     int32_t code = doOpenReaderImpl(pReader->innerReader[1]);
+    resetDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py
index bee20710b5..71501d5967 100644
--- a/tests/system-test/2-query/interp.py
+++ b/tests/system-test/2-query/interp.py
@@ -11,7 +11,7 @@ class TDTestCase:
 
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
-        tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), True)
 
     def run(self):
         dbname = "db"

From e4de732285d0e1835c157ff993bfe20227de2b77 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 18:15:00 +0800
Subject: [PATCH 31/42] fix(query): always build hash map from uid to keyinfo

---
 source/libs/executor/inc/executorimpl.h |  2 +-
 source/libs/executor/src/executil.c     | 38 ++++++++++++------------
 source/libs/executor/src/executor.c     |  1 +
 source/libs/executor/src/executorimpl.c | 22 +++++++-------
 source/libs/executor/src/scanoperator.c | 39 ++++++++-----------------
 5 files changed, 45 insertions(+), 57 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index e4f6b42224..17ab756933 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -1077,7 +1077,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
 bool    groupbyTbname(SNodeList* pGroupList);
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort);
+int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort);
 void*   destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
 int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
                                    SGroupResInfo* pGroupResInfo);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 877dc5ec27..6b598d7bd3 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -741,7 +741,8 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
 
     int32_t len = (int32_t)(pStart - (char*)keyBuf);
     info->groupId = calcGroupId(keyBuf, len);
-    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t));
+
+    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &i, sizeof(int32_t));
   }
 
   //  int64_t st2 = taosGetTimestampUs();
@@ -1665,29 +1666,29 @@ uint64_t getTotalTables(const STableListInfo* pTableList) {
 }
 
 uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
-  if (pTableList->oneTableForEachGroup) {
-    return tableUid;
-  }
+  ASSERT(pTableList->map != NULL);
+  int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
+  ASSERT(slot != NULL);
 
-  uint64_t* groupId = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
-  if (groupId != NULL) {
-    return *groupId;
-  } else {
-    return 0;
-  }
+  STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, *slot);
+  ASSERT(pKeyInfo->uid == tableUid);
+
+  return pKeyInfo->groupId;
 }
 
 int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid) {
-  STableKeyInfo keyInfo = {.uid = uid, .groupId = gid};
-  taosArrayPush(pTableList->pTableList, &keyInfo);
-  if (!pTableList->oneTableForEachGroup) {
-    if (pTableList->map == NULL) {
-      pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-    }
-
-    taosHashPut(pTableList->map, &uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
+  if (pTableList->map == NULL) {
+    ASSERT(taosArrayGetSize(pTableList->pTableList) == 0);
+    pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   }
 
+  STableKeyInfo keyInfo = {.uid = uid, .groupId = gid};
+  taosArrayPush(pTableList->pTableList, &keyInfo);
+
+  int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1;
+  taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot));
+
+  qDebug("uid:%"PRIu64", groupId:%"PRIu64" added into table list, slot:%d, %d", uid, gid, slot, slot + 1);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1724,6 +1725,7 @@ int32_t  getNumOfOutputGroups(const STableListInfo* pTableList) {
   return pTableList->numOfOuputGroups;
 }
 
+// todo remove it
 bool     oneTableForEachGroup(const STableListInfo* pTableList) {
   return pTableList->oneTableForEachGroup;
 }
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 7f04623612..765968999a 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -360,6 +360,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
 
       if (!exists) {
 #endif
+
       addTableIntoTableList(pTableListInfo, keyInfo.uid, keyInfo.groupId);
     }
 
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 00fef26670..b5c6c5cf39 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3489,11 +3489,8 @@ bool groupbyTbname(SNodeList* pGroupList) {
   return bytbname;
 }
 
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) {
+int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) {
   int32_t code = TSDB_CODE_SUCCESS;
-  if (group == NULL) {
-    return code;
-  }
 
   pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   if (pTableListInfo->map == NULL) {
@@ -3501,19 +3498,22 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
     return code;
   }
 
-  bool   assignUid = groupbyTbname(group);
+  bool groupByTbname = groupbyTbname(group);
   size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
-
-  if (assignUid) { // in case of group/partition by tbname, the group id is equalled to the uid of table
+  if (group == NULL || groupByTbname) {
     for (int32_t i = 0; i < numOfTables; i++) {
       STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
-      info->groupId = info->uid;
-      taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t));
+      info->groupId = groupByTbname? info->uid:0;
+
+      taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &i, sizeof(int32_t));
     }
 
-    pTableListInfo->oneTableForEachGroup = true;
-    if (groupSort) {
+    pTableListInfo->oneTableForEachGroup = groupByTbname;
+
+    if (groupSort && groupByTbname) {
       pTableListInfo->numOfOuputGroups = numOfTables;
+    } else {
+      pTableListInfo->numOfOuputGroups = 1;
     }
   } else {
     code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 6935a9ab93..7524a4ea1a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1137,12 +1137,13 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
 }
 
 static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) {
-  SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
-  uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
-  if (groupId) {
-    return *groupId;
-  }
-  return 0;
+  return getTableGroupId(&pInfo->pTableScanOp->pTaskInfo->tableqinfoList, uid);
+//  SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
+//  uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
+//  if (groupId) {
+//    return *groupId;
+//  }
+//  return 0;
 }
 
 static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
@@ -1561,12 +1562,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   pInfo->pRes->info.type = STREAM_NORMAL;
   pInfo->pRes->info.version = pBlock->info.version;
 
-  uint64_t* groupIdPre = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-  if (groupIdPre) {
-    pInfo->pRes->info.groupId = *groupIdPre;
-  } else {
-    pInfo->pRes->info.groupId = 0;
-  }
+  pInfo->pRes->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid);
 
   // todo extract method
   for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
@@ -4236,7 +4232,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
   }
 
   pTableListInfo->numOfOuputGroups = 1;
-  code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags, groupSort);
+  code = setGroupIdMapForAllTables(pTableListInfo, pHandle, pGroupTags, groupSort);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -4566,10 +4562,7 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
       continue;
     }
 
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
+    pBlock->info.groupId = getTableGroupId(&pOperator->pTaskInfo->tableqinfoList, pBlock->info.uid);
 
     pOperator->resultInfo.totalRows += pBlock->info.rows;  // pTableScanInfo->readRecorder.totalRows;
     pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
@@ -4626,11 +4619,7 @@ static SSDataBlock* getTableDataBlock2(void* param) {
       continue;
     }
 
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
+    pBlock->info.groupId = getTableGroupId(&pOperator->pTaskInfo->tableqinfoList, pBlock->info.uid);
     pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
     pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
 
@@ -4684,11 +4673,7 @@ static SSDataBlock* getTableDataBlock(void* param) {
       continue;
     }
 
-    uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
-    if (groupId) {
-      pBlock->info.groupId = *groupId;
-    }
-
+    pBlock->info.groupId = getTableGroupId(&pOperator->pTaskInfo->tableqinfoList, pBlock->info.uid);
     pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
     pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
 

From 1886087e7b7ce9dee5012dd6a5ab86a32ece0da8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 18:17:43 +0800
Subject: [PATCH 32/42] refactor: do some internal refactor.

---
 source/libs/executor/src/executil.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 6b598d7bd3..fe6fb7c2d6 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1658,17 +1658,13 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimit
 }
 
 uint64_t getTotalTables(const STableListInfo* pTableList) {
-  if (pTableList->map != NULL) {
-    ASSERT(taosArrayGetSize(pTableList->pTableList) == taosHashGetSize(pTableList->map));
-  }
-
+  ASSERT(taosArrayGetSize(pTableList->pTableList) == taosHashGetSize(pTableList->map));
   return taosArrayGetSize(pTableList->pTableList);
 }
 
 uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
-  ASSERT(pTableList->map != NULL);
   int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
-  ASSERT(slot != NULL);
+  ASSERT(pTableList->map != NULL && slot != NULL);
 
   STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, *slot);
   ASSERT(pKeyInfo->uid == tableUid);
@@ -1688,7 +1684,7 @@ int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t
   int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1;
   taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot));
 
-  qDebug("uid:%"PRIu64", groupId:%"PRIu64" added into table list, slot:%d, %d", uid, gid, slot, slot + 1);
+  qDebug("uid:%"PRIu64", groupId:%"PRIu64" added into table list, slot:%d, total:%d", uid, gid, slot, slot + 1);
   return TSDB_CODE_SUCCESS;
 }
 

From afdb1467b7dfef70c3039f423b7463a18b0f6654 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 18:27:22 +0800
Subject: [PATCH 33/42] refactor: do some internal refactor.

---
 source/dnode/vnode/inc/vnode.h         | 10 +++++-----
 source/dnode/vnode/src/tsdb/tsdbRead.c | 22 +++++++++++-----------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index d7e4935edd..6200c88d75 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -152,9 +152,9 @@ typedef struct STsdbReader STsdbReader;
 #define CACHESCAN_RETRIEVE_LAST_ROW    0x4
 #define CACHESCAN_RETRIEVE_LAST        0x8
 
-int32_t  tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num);
-int32_t  tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                        STsdbReader **ppReader, const char *idstr);
+int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
+int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
+                       STsdbReader **ppReader, const char *idstr);
 
 void     tsdbReaderClose(STsdbReader *pReader);
 bool     tsdbNextDataBlock(STsdbReader *pReader);
@@ -169,8 +169,8 @@ void    *tsdbGetIdx(SMeta *pMeta);
 void    *tsdbGetIvtIdx(SMeta *pMeta);
 uint64_t getReaderMaxVersion(STsdbReader *pReader);
 
-int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables,
-                                int32_t numOfCols, uint64_t suid, void** pReader);
+int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+                                uint64_t suid, void **pReader);
 int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
 void   *tsdbCacherowsReaderClose(void *pReader);
 int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 8ebee92d15..808763b97c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -426,7 +426,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
     return true;
   }
 
-  _err:
+_err:
   return false;
 }
 
@@ -528,7 +528,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   *ppReader = pReader;
   return code;
 
-  _end:
+_end:
   tsdbReaderClose(pReader);
   *ppReader = NULL;
   return code;
@@ -580,7 +580,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader,
 
   pReader->cost.headFileLoadTime += (et1 - st) / 1000.0;
 
-  _end:
+_end:
   taosArrayDestroy(aBlockIdx);
   return code;
 }
@@ -1405,8 +1405,8 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock
   // log the reason why load the datablock for profile
   if (loadDataBlock) {
     tsdbDebug("%p uid:%" PRIu64
-                  " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
-                  "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
+              " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, "
+              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
               pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
               info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock,
               pReader->idStr);
@@ -2039,7 +2039,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
         pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL);
 
         tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-                      "-%" PRId64 " %s",
+                  "-%" PRId64 " %s",
                   pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr);
       } else {
         tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid,
@@ -2060,7 +2060,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
         pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL);
 
         tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-                      "-%" PRId64 " %s",
+                  "-%" PRId64 " %s",
                   pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr);
       } else {
         tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid,
@@ -2295,7 +2295,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
     }
   }
 
-  _end:
+_end:
   pResBlock->info.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
   blockDataUpdateTsWindow(pResBlock, 0);
 
@@ -2388,13 +2388,13 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
 
   taosArrayDestroy(pDelData);
   pBlockScanInfo->iter.index =
-          ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
+      ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
   pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index;
   pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index;
   pBlockScanInfo->lastBlockDelIndex = pBlockScanInfo->iter.index;
   return code;
 
-  _err:
+_err:
   taosArrayDestroy(pDelData);
   return code;
 }
@@ -2725,7 +2725,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
   SDataBlockIter* pBlockIter = &pReader->status.blockIter;
 
   if (pBlockIter->numOfBlocks == 0) {
-    _begin:
+  _begin:
     code = doLoadLastBlockSequentially(pReader);
     if (code != TSDB_CODE_SUCCESS) {
       return code;

From 6fc0e1b60d48374500fc09a0520286ce5dfc01e8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 18:32:17 +0800
Subject: [PATCH 34/42] refactor: do some internal refactor.

---
 source/libs/executor/src/executil.c | 41 +++++++++++++----------------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index fe6fb7c2d6..57a6645215 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -337,8 +337,8 @@ static EDealRes getColumn(SNode** pNode, void* pContext) {
     taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
     pSColumnNode->slotId = pData->index++;
     SColumnInfo cInfo = {.colId = pSColumnNode->colId,
-        .type = pSColumnNode->node.resType.type,
-        .bytes = pSColumnNode->node.resType.bytes};
+                         .type = pSColumnNode->node.resType.type,
+                         .bytes = pSColumnNode->node.resType.bytes};
 #if TAG_FILTER_DEBUG
     qDebug("tagfilter build column info, slotId:%d, colId:%d, type:%d", pSColumnNode->slotId, cInfo.colId, cInfo.type);
 #endif
@@ -507,7 +507,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
   //  int64_t st2 = taosGetTimestampUs();
   //  qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
 
-  end:
+end:
   taosHashCleanup(tags);
   taosHashCleanup(ctx.colHash);
   taosArrayDestroy(ctx.cInfoList);
@@ -748,7 +748,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
   //  int64_t st2 = taosGetTimestampUs();
   //  qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
 
-  end:
+end:
   taosMemoryFreeClear(keyBuf);
   taosHashCleanup(tags);
   taosHashCleanup(ctx.colHash);
@@ -828,24 +828,24 @@ static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) {
 }
 
 static int32_t nameComparFn(const void* p1, const void* p2) {
-  const char* pName1 = *(const char**) p1;
-  const char* pName2 = *(const char**) p2;
+  const char* pName1 = *(const char**)p1;
+  const char* pName2 = *(const char**)p2;
 
   int32_t ret = strcmp(pName1, pName2);
   if (ret == 0) {
     return 0;
   } else {
-    return (ret > 0)? 1:-1;
+    return (ret > 0) ? 1 : -1;
   }
 }
 
 static SArray* getTableNameList(const SNodeListNode* pList) {
-  int32_t len = LIST_LENGTH(pList->pNodeList);
+  int32_t    len = LIST_LENGTH(pList->pNodeList);
   SListCell* cell = pList->pNodeList->pHead;
 
   SArray* pTbList = taosArrayInit(len, POINTER_BYTES);
   for (int i = 0; i < pList->pNodeList->length; i++) {
-    SValueNode* valueNode = (SValueNode*) cell->pNode;
+    SValueNode* valueNode = (SValueNode*)cell->pNode;
     if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) {
       terrno = TSDB_CODE_INVALID_PARA;
       taosArrayDestroy(pTbList);
@@ -1120,7 +1120,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
 
 int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
                             int32_t type, SColMatchInfo* pMatchInfo) {
-  size_t numOfCols = LIST_LENGTH(pNodeList);
+  size_t  numOfCols = LIST_LENGTH(pNodeList);
   int32_t code = 0;
 
   pMatchInfo->matchType = type;
@@ -1463,10 +1463,10 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
   while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
     SColumnInfoData* p = taosArrayGet(pCols, i);
     SColMatchItem*   pmInfo = taosArrayGet(pColMatchInfo, j);
-/*    if (!outputEveryColumn && pmInfo->reserved) {
-      j++;
-      continue;
-    }*/
+    /*    if (!outputEveryColumn && pmInfo->reserved) {
+          j++;
+          continue;
+        }*/
 
     if (p->info.colId == pmInfo->colId) {
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId);
@@ -1684,11 +1684,12 @@ int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t
   int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1;
   taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot));
 
-  qDebug("uid:%"PRIu64", groupId:%"PRIu64" added into table list, slot:%d, total:%d", uid, gid, slot, slot + 1);
+  qDebug("uid:%" PRIu64 ", groupId:%" PRIu64 " added into table list, slot:%d, total:%d", uid, gid, slot, slot + 1);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, int32_t* size) {
+int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo,
+                         int32_t* size) {
   int32_t total = getNumOfOutputGroups(pTableList);
   if (ordinalGroupIndex < 0 || ordinalGroupIndex >= total) {
     return TSDB_CODE_INVALID_PARA;
@@ -1717,14 +1718,10 @@ int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupI
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t  getNumOfOutputGroups(const STableListInfo* pTableList) {
-  return pTableList->numOfOuputGroups;
-}
+int32_t getNumOfOutputGroups(const STableListInfo* pTableList) { return pTableList->numOfOuputGroups; }
 
 // todo remove it
-bool     oneTableForEachGroup(const STableListInfo* pTableList) {
-  return pTableList->oneTableForEachGroup;
-}
+bool oneTableForEachGroup(const STableListInfo* pTableList) { return pTableList->oneTableForEachGroup; }
 
 void destroyTableList(STableListInfo* pTableqinfoList) {
   pTableqinfoList->pTableList = taosArrayDestroy(pTableqinfoList->pTableList);

From 21364c4041ca2b02246fa71d8f0babf4d01a7dac Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Sat, 29 Oct 2022 18:34:42 +0800
Subject: [PATCH 35/42] fix: add api ut cases

---
 source/libs/catalog/src/catalog.c         |  2 +
 source/libs/catalog/test/catalogTests.cpp | 94 +++++++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c
index 8f621545c9..b6a22b5fa7 100644
--- a/source/libs/catalog/src/catalog.c
+++ b/source/libs/catalog/src/catalog.c
@@ -1114,6 +1114,7 @@ int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn,
   CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, exists));
 }
 
+#if 0
 int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, SMetaData* pRsp) {
   CTG_API_ENTER();
 
@@ -1176,6 +1177,7 @@ _return:
 
   CTG_API_LEAVE(code);
 }
+#endif
 
 int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, catalogCallback fp,
                                void* param, int64_t* jobId) {
diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp
index de34868c55..0bdd9841ab 100644
--- a/source/libs/catalog/test/catalogTests.cpp
+++ b/source/libs/catalog/test/catalogTests.cpp
@@ -63,6 +63,7 @@ enum {
   CTGT_RSP_QNODELIST,
   CTGT_RSP_UDF,
   CTGT_RSP_SVRVER,
+  CTGT_RSP_DNODElIST,
   CTGT_RSP_TBMETA_NOT_EXIST,
 };
 
@@ -702,6 +703,30 @@ void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRs
   pRsp->pCont = pReq;
 }
 
+void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
+  rpcFreeCont(pMsg->pCont);
+
+  SDnodeListRsp dRsp = {0};
+  dRsp.dnodeList = taosArrayInit(1, sizeof(SEpSet));
+  SEpSet epSet = {0};
+  epSet.numOfEps = 1;
+  tstrncpy(epSet.eps[0].fqdn, "localhost", TSDB_FQDN_LEN);
+  epSet.eps[0].port = 6030;
+  
+  (void)taosArrayPush(dRsp.dnodeList, &epSet);
+  
+  int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp);
+  void   *pReq = rpcMallocCont(rspLen);
+  tSerializeSDnodeListRsp(pReq, rspLen, &dRsp);
+
+  pRsp->code = 0;
+  pRsp->contLen = rspLen;
+  pRsp->pCont = pReq;
+
+  tFreeSDnodeListRsp(&dRsp);
+}
+
+
 
 void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   switch (pMsg->msgType) {
@@ -727,6 +752,9 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
     case TDMT_MND_SERVER_VERSION:
       ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
       break;
+    case TDMT_MND_DNODE_LIST:
+      ctgTestRspDndeList(shandle, pEpSet, pMsg, pRsp);
+      break;
     default:
       break;
   }
@@ -779,6 +807,9 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp
     case CTGT_RSP_SVRVER:
       ctgTestRspSvrVer(shandle, pEpSet, pMsg, pRsp);
       break;
+    case CTGT_RSP_DNODElIST:
+      ctgTestRspDndeList(shandle, pEpSet, pMsg, pRsp);
+      break;
     default:
       ctgTestRspAuto(shandle, pEpSet, pMsg, pRsp);
       break;
@@ -2948,6 +2979,69 @@ TEST(apiTest, catalogGetServerVersion_test) {
   catalogDestroy();
 }
 
+TEST(apiTest, catalogUpdateTableIndex_test) {
+  struct SCatalog  *pCtg = NULL;
+  SRequestConnInfo connInfo = {0};  
+  SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
+
+  ctgTestInitLogFile();
+
+  memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
+  ctgTestRspIdx = 0;
+  ctgTestRspFunc[0] = CTGT_RSP_SVRVER;
+
+  ctgTestSetRspByIdx();
+
+  initQueryModuleMsgHandle();
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  STableIndexRsp rsp = {0};
+  strcpy(rsp.dbFName, ctgTestDbname);
+  strcpy(rsp.tbName, ctgTestSTablename);
+  rsp.suid = ctgTestSuid;
+  rsp.version = 1;
+  code = catalogUpdateTableIndex(pCtg, &rsp);
+  ASSERT_EQ(code, 0);
+
+  catalogDestroy();
+}
+
+
+TEST(apiTest, catalogGetDnodeList_test) {
+  struct SCatalog  *pCtg = NULL;
+  SRequestConnInfo connInfo = {0};  
+  SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo;
+
+  ctgTestInitLogFile();
+
+  memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc));
+  ctgTestRspIdx = 0;
+  ctgTestRspFunc[0] = CTGT_RSP_DNODElIST;
+
+  ctgTestSetRspByIdx();
+
+  initQueryModuleMsgHandle();
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SArray* pList = NULL;
+  code = catalogGetDnodeList(pCtg, mockPointer, &pList);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(taosArrayGetSize(pList), 1);
+
+  taosArrayDestroy(pList);
+
+  catalogDestroy();
+}
 
 
 int main(int argc, char **argv) {

From e182637af59e29336ef2770f8d52eae9a0224f69 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Sat, 29 Oct 2022 19:58:44 +0800
Subject: [PATCH 36/42] fix: make test pass when running tfs_test standalone in
 mac

---
 source/libs/tfs/test/tfsTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp
index f4b0adf1f7..df37630fd7 100644
--- a/source/libs/tfs/test/tfsTest.cpp
+++ b/source/libs/tfs/test/tfsTest.cpp
@@ -42,7 +42,7 @@ TEST_F(TfsTest, 01_Open_Close) {
   STfs *pTfs = tfsOpen(&dCfg, 1);
   ASSERT_EQ(pTfs, nullptr);
 
-  taosMkDir(root);
+  taosMulMkDir(root);
   pTfs = tfsOpen(&dCfg, 1);
   ASSERT_NE(pTfs, nullptr);
 

From 4e7478279956c5f0776f9c7459ab60effe3d1871 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 29 Oct 2022 20:47:14 +0800
Subject: [PATCH 37/42] refactor: do some internal refactor.

---
 source/libs/executor/src/executil.c     |  2 --
 source/libs/executor/src/executorimpl.c | 28 +++++++++++++++-------
 source/libs/executor/src/scanoperator.c | 31 +++++++++++--------------
 3 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 57a6645215..5b417999b4 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -741,8 +741,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
 
     int32_t len = (int32_t)(pStart - (char*)keyBuf);
     info->groupId = calcGroupId(keyBuf, len);
-
-    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &i, sizeof(int32_t));
   }
 
   //  int64_t st2 = taosGetTimestampUs();
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index b5c6c5cf39..e31a34e5ad 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3379,7 +3379,7 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn);
-  int32_t size = getTotalTables(pTableListInfo);
+  int32_t size = taosArrayGetSize(pTableListInfo->pTableList);
 
   SArray* pList = taosArrayInit(4, sizeof(int32_t));
 
@@ -3403,7 +3403,6 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
   taosArrayDestroy(pList);
 
 # if 0
-
   SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t));
   if (sortSupport == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -3504,13 +3503,12 @@ int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* p
     for (int32_t i = 0; i < numOfTables; i++) {
       STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
       info->groupId = groupByTbname? info->uid:0;
-
-      taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &i, sizeof(int32_t));
     }
 
     pTableListInfo->oneTableForEachGroup = groupByTbname;
 
     if (groupSort && groupByTbname) {
+      taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn);
       pTableListInfo->numOfOuputGroups = numOfTables;
     } else {
       pTableListInfo->numOfOuputGroups = 1;
@@ -3526,6 +3524,13 @@ int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* p
     }
   }
 
+  // add all table entry in the hash map
+  size_t size = taosArrayGetSize(pTableListInfo->pTableList);
+  for(int32_t i = 0; i < size; ++i) {
+    STableKeyInfo* p = taosArrayGet(pTableListInfo->pTableList, i);
+    taosHashPut(pTableListInfo->map, &p->uid, sizeof(uint64_t), &i, sizeof(int32_t));
+  }
+
   return code;
 }
 
@@ -3590,7 +3595,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t                   code =
-          createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle,
+          createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, /*pTableScanNode->groupSort*/true, pHandle,
                                   pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
       if (code) {
         pTaskInfo->code = code;
@@ -3652,16 +3657,23 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
       SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
       pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo));
+      pTableListInfo->numOfOuputGroups = 1;
 
       if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
-        int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pTableListInfo->pTableList);
+        SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo));
+        int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList);
         if (code != TSDB_CODE_SUCCESS) {
           pTaskInfo->code = terrno;
           return NULL;
         }
+
+        for(int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) {
+          STableKeyInfo* p = taosArrayGet(pList, i);
+          addTableIntoTableList(pTableListInfo, p->uid, 0);
+        }
+        taosArrayDestroy(pList);
       } else {  // Create one table group.
-        STableKeyInfo info = {.uid = pBlockNode->uid, .groupId = 0};
-        taosArrayPush(pTableListInfo->pTableList, &info);
+        addTableIntoTableList(pTableListInfo, pBlockNode->uid, 0);
       }
 
       SQueryTableDataCond cond = {0};
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7524a4ea1a..766d7ec18c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1118,6 +1118,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
 
     relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
     doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
+
+    pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, binfo.uid);
   }
 
   tsdbReaderClose(pReader);
@@ -3050,11 +3052,11 @@ static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
   if (func == NULL) return -1;
 
   SMetaFltParam param = {.suid = 0,
-      .cid = 0,
-      .type = TSDB_DATA_TYPE_VARCHAR,
-      .val = pVal->datum.p,
-      .reverse = reverse,
-      .filterFunc = func};
+                         .cid = 0,
+                         .type = TSDB_DATA_TYPE_VARCHAR,
+                         .val = pVal->datum.p,
+                         .reverse = reverse,
+                         .filterFunc = func};
   return -1;
 }
 
@@ -3069,11 +3071,11 @@ static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
   if (func == NULL) return -1;
 
   SMetaFltParam param = {.suid = 0,
-      .cid = 0,
-      .type = TSDB_DATA_TYPE_BIGINT,
-      .val = &pVal->datum.i,
-      .reverse = reverse,
-      .filterFunc = func};
+                         .cid = 0,
+                         .type = TSDB_DATA_TYPE_BIGINT,
+                         .val = &pVal->datum.i,
+                         .reverse = reverse,
+                         .filterFunc = func};
 
   int32_t ret = metaFilterCreateTime(pMeta, &param, result);
   return ret;
@@ -4746,9 +4748,6 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond));
 
   for (int32_t i = 0; i < numOfTable; ++i) {
-    STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i + tableStartIdx);
-    size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
-
     STableMergeScanSortSourceParam param = {0};
     param.readerIdx = i;
     param.pOperator = pOperator;
@@ -4959,10 +4958,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     goto _error;
   }
 
-  if (pTableScanNode->pGroupTags) {
-    taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
-  }
-
   SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
 
   int32_t numOfCols = 0;
@@ -5016,7 +5011,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pOperator->cost.openCost = 0;
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   taosMemoryFree(pInfo);
   taosMemoryFree(pOperator);

From 2c524cad4a7003afd4508a62f98779421ccad5e5 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 30 Oct 2022 03:08:33 +0800
Subject: [PATCH 38/42] refactor: do some internal refactor.

---
 source/libs/executor/src/executil.c             | 7 ++++++-
 source/libs/executor/src/executorimpl.c         | 6 +++---
 source/libs/executor/src/scanoperator.c         | 3 ++-
 tests/system-test/1-insert/create_retentions.py | 2 +-
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 5b417999b4..bb65a9f7f2 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1697,7 +1697,12 @@ int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupI
   // 1. only one group exists, and 2. one table exists for each group.
   if (total == 1) {
     *size = getTotalTables(pTableList);
-    *pKeyInfo = taosArrayGet(pTableList->pTableList, 0);
+    if (*size > 0) {
+      *pKeyInfo = taosArrayGet(pTableList->pTableList, 0);
+    } else {
+      *pKeyInfo = NULL;
+    }
+
     return TSDB_CODE_SUCCESS;
   } else if (total == getTotalTables(pTableList)) {
     *size = 1;
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index e31a34e5ad..54c92c10bd 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3630,12 +3630,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
 
 #ifndef NDEBUG
         int32_t sz = taosArrayGetSize(pTableListInfo->pTableList);
+        qDebug("create stream task, total:%d", sz);
+
         for (int32_t i = 0; i < sz; i++) {
           STableKeyInfo* pKeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
-          qDebug("creating stream task: add table uid:%" PRIu64, pKeyInfo->uid);
+          qDebug("add table uid:%" PRIu64", gid:%"PRIu64, pKeyInfo->uid, pKeyInfo->groupId);
         }
-
-        qDebug("table in hashmap, %d", (int32_t) getTotalTables(pTableListInfo));
 #endif
       }
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 766d7ec18c..415fa60287 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4225,6 +4225,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return code;
   }
 
+  pTableListInfo->numOfOuputGroups = 1;
+
   int64_t st1 = taosGetTimestampUs();
   qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1 - st) / 1000.0, idStr);
 
@@ -4233,7 +4235,6 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return TSDB_CODE_SUCCESS;
   }
 
-  pTableListInfo->numOfOuputGroups = 1;
   code = setGroupIdMapForAllTables(pTableListInfo, pHandle, pGroupTags, groupSort);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py
index 54abcec391..47d53d5b34 100644
--- a/tests/system-test/1-insert/create_retentions.py
+++ b/tests/system-test/1-insert/create_retentions.py
@@ -46,7 +46,7 @@ class TDTestCase:
 
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
-        tdSql.init(conn.cursor(), False)
+        tdSql.init(conn.cursor(), True)
 
     @property
     def create_databases_sql_err(self):

From 365e6eec808ee4c177545427a01eafa3732a3cab Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 30 Oct 2022 11:29:55 +0800
Subject: [PATCH 39/42] refactor: do some internal refactor.

---
 source/libs/executor/src/executil.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index bb65a9f7f2..0f2e5194c1 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1697,12 +1697,7 @@ int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupI
   // 1. only one group exists, and 2. one table exists for each group.
   if (total == 1) {
     *size = getTotalTables(pTableList);
-    if (*size > 0) {
-      *pKeyInfo = taosArrayGet(pTableList->pTableList, 0);
-    } else {
-      *pKeyInfo = NULL;
-    }
-
+    *pKeyInfo = (*size == 0)? NULL:taosArrayGet(pTableList->pTableList, 0);
     return TSDB_CODE_SUCCESS;
   } else if (total == getTotalTables(pTableList)) {
     *size = 1;
@@ -1723,7 +1718,6 @@ int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupI
 
 int32_t getNumOfOutputGroups(const STableListInfo* pTableList) { return pTableList->numOfOuputGroups; }
 
-// todo remove it
 bool oneTableForEachGroup(const STableListInfo* pTableList) { return pTableList->oneTableForEachGroup; }
 
 void destroyTableList(STableListInfo* pTableqinfoList) {

From 8e22c89136ea15cfc2e876380435183bcaa8059d Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Sun, 30 Oct 2022 12:45:55 +0800
Subject: [PATCH 40/42] fix: coverity scan problem for trow

---
 source/common/src/trow.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/source/common/src/trow.c b/source/common/src/trow.c
index b007075efe..e4818aaa87 100644
--- a/source/common/src/trow.c
+++ b/source/common/src/trow.c
@@ -73,7 +73,13 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
   } else if (tdValTypeIsNone(pVal->valType)) {
     printf("NONE ");
     return;
+  } 
+  if(!pVal->val) {
+    ASSERT(0);
+    printf("BadVal ");
+    return;
   }
+
   switch (colType) {
     case TSDB_DATA_TYPE_BOOL:
       printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true");
@@ -678,6 +684,10 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp
   }
   // TS KEY is stored in STSRow.ts and not included in STSRow.data field.
   if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    if (!val) {
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+    }
     TD_ROW_KEY(pRow) = *(TSKEY *)val;
     // The primary TS key is Norm all the time, thus its valType is not stored in bitmap.
     return TSDB_CODE_SUCCESS;

From 93e75c73caf83f18f7401e545fa96627719f1f6c Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Sun, 30 Oct 2022 14:01:43 +0800
Subject: [PATCH 41/42] fix: taosadapter build on windows (#17763)

* fix: taosadapter build on windows

* fix: disable taosadapter build on windows temporarily
---
 tools/CMakeLists.txt | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index e01796617d..79be05e9fc 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -86,9 +86,34 @@ ELSE ()
     )
     MESSAGE("CURRENT SOURCE DIR ${CMAKE_CURRENT_SOURCE_DIR}")
 
-    include(ExternalProject)
+    IF (TD_WINDOWS)
+        INCLUDE(ExternalProject)
+        ExternalProject_Add(taosadapter
+            PREFIX        "taosadapter"
+            SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter
+            BUILD_ALWAYS         off
+            DEPENDS             taos
+            BUILD_IN_SOURCE     1
+            CONFIGURE_COMMAND cmake -E echo "taosadapter no need cmake to config"
+            PATCH_COMMAND
+            COMMAND     git clean -f -d
+            BUILD_COMMAND
+            COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client
+            COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib
+            COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}"
+            COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}"
 
-    ExternalProject_Add(taosadapter
+            INSTALL_COMMAND
+            COMMAND cmake -E time upx taosadapter ||:
+            COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin
+            COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/
+            COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/
+            COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/
+            COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin
+            )
+    ELSE (TD_WINDOWS)
+        INCLUDE(ExternalProject)
+        ExternalProject_Add(taosadapter
             PREFIX        "taosadapter"
             SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter
             BUILD_ALWAYS         off
@@ -108,4 +133,5 @@ ELSE ()
             COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/
             COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin
             )
+    ENDIF (TD_WINDOWS)
 ENDIF ()

From acb663421665d665b729e5c5a02c622d3d5c64bc Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 30 Oct 2022 20:07:41 +0800
Subject: [PATCH 42/42] enh: set  sst trigger default value from 8 to 1

---
 include/util/tdef.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/util/tdef.h b/include/util/tdef.h
index e298509ad6..6e1fa87854 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -360,7 +360,7 @@ typedef enum ELogicConditionType {
 #define TSDB_DEFAULT_DB_SCHEMALESS      TSDB_DB_SCHEMALESS_OFF
 #define TSDB_MIN_STT_TRIGGER            1
 #define TSDB_MAX_STT_TRIGGER            16
-#define TSDB_DEFAULT_SST_TRIGGER        8
+#define TSDB_DEFAULT_SST_TRIGGER        1
 #define TSDB_MIN_HASH_PREFIX            0
 #define TSDB_MAX_HASH_PREFIX            128
 #define TSDB_DEFAULT_HASH_PREFIX        0