From 927464113bcb6e54b3bdf2ee15d53212ee83dcc6 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Sat, 18 Jun 2022 20:15:47 +0800
Subject: [PATCH 01/24] feat: tag condition split

---
 include/libs/nodes/plannodes.h                | 32 +++++-----
 include/libs/nodes/querynodes.h               | 13 ++--
 source/libs/nodes/src/nodesCloneFuncs.c       |  1 +
 source/libs/nodes/src/nodesCodeFuncs.c        | 11 +++-
 source/libs/nodes/src/nodesUtilFuncs.c        | 60 ++++++++++++++++---
 source/libs/parser/src/parTranslater.c        |  4 +-
 source/libs/planner/src/planLogicCreater.c    |  4 +-
 source/libs/planner/src/planOptimizer.c       | 34 +----------
 source/libs/planner/src/planPhysiCreater.c    |  9 +++
 source/libs/planner/test/planOptimizeTest.cpp |  2 +
 10 files changed, 104 insertions(+), 66 deletions(-)

diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index a9002b5d19..ac37b0c44f 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -64,6 +64,7 @@ typedef struct SScanLogicNode {
   int8_t        intervalUnit;
   int8_t        slidingUnit;
   SNode*        pTagCond;
+  SNode*        pTagIndexCond;
   int8_t        triggerType;
   int64_t       watermark;
   int16_t       tsColId;
@@ -143,21 +144,21 @@ typedef enum EWindowAlgorithm {
 } EWindowAlgorithm;
 
 typedef struct SWindowLogicNode {
-  SLogicNode         node;
-  EWindowType        winType;
-  SNodeList*         pFuncs;
-  int64_t            interval;
-  int64_t            offset;
-  int64_t            sliding;
-  int8_t             intervalUnit;
-  int8_t             slidingUnit;
-  int64_t            sessionGap;
-  SNode*             pTspk;
-  SNode*             pStateExpr;
-  int8_t             triggerType;
-  int64_t            watermark;
-  double             filesFactor;
-  EWindowAlgorithm   windowAlgo;
+  SLogicNode       node;
+  EWindowType      winType;
+  SNodeList*       pFuncs;
+  int64_t          interval;
+  int64_t          offset;
+  int64_t          sliding;
+  int8_t           intervalUnit;
+  int8_t           slidingUnit;
+  int64_t          sessionGap;
+  SNode*           pTspk;
+  SNode*           pStateExpr;
+  int8_t           triggerType;
+  int64_t          watermark;
+  double           filesFactor;
+  EWindowAlgorithm windowAlgo;
 } SWindowLogicNode;
 
 typedef struct SFillLogicNode {
@@ -438,6 +439,7 @@ typedef struct SSubplan {
   SPhysiNode*    pNode;         // physical plan of current subplan
   SDataSinkNode* pDataSink;     // data of the subplan flow into the datasink
   SNode*         pTagCond;
+  SNode*         pTagIndexCond;
 } SSubplan;
 
 typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index 3a0d3178a3..78c6f3467e 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -283,11 +283,11 @@ typedef enum ESqlClause {
 } ESqlClause;
 
 typedef struct SDeleteStmt {
-  ENodeType   type;           // QUERY_NODE_DELETE_STMT
-  SNode*      pFromTable;     // FROM clause
-  SNode*      pWhere;         // WHERE clause
-  SNode*      pCountFunc;     // count the number of rows affected
-  SNode*      pTagIndexCond;  // pWhere divided into pTagIndexCond and timeRange
+  ENodeType   type;        // QUERY_NODE_DELETE_STMT
+  SNode*      pFromTable;  // FROM clause
+  SNode*      pWhere;      // WHERE clause
+  SNode*      pCountFunc;  // count the number of rows affected
+  SNode*      pTagCond;    // pWhere divided into pTagCond and timeRange
   STimeWindow timeRange;
   uint8_t     precision;
   bool        deleteZeroRows;
@@ -391,7 +391,8 @@ void    nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
 
 char*   nodesGetFillModeString(EFillMode mode);
 int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);
-int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond);
+int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond,
+                           SNode** pOtherCond);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index d0eb1548c1..02c6c582e3 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -351,6 +351,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   COPY_SCALAR_FIELD(intervalUnit);
   COPY_SCALAR_FIELD(slidingUnit);
   CLONE_NODE_FIELD(pTagCond);
+  CLONE_NODE_FIELD(pTagIndexCond);
   COPY_SCALAR_FIELD(triggerType);
   COPY_SCALAR_FIELD(watermark);
   COPY_SCALAR_FIELD(tsColId);
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 1b2eadd8e8..3d5e018dfa 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -2193,6 +2193,7 @@ static const char* jkSubplanNodeAddr = "NodeAddr";
 static const char* jkSubplanRootNode = "RootNode";
 static const char* jkSubplanDataSink = "DataSink";
 static const char* jkSubplanTagCond = "TagCond";
+static const char* jkSubplanTagIndexCond = "TagIndexCond";
 
 static int32_t subplanToJson(const void* pObj, SJson* pJson) {
   const SSubplan* pNode = (const SSubplan*)pObj;
@@ -2222,6 +2223,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkSubplanTagCond, nodeToJson, pNode->pTagCond);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond);
+  }
 
   return code;
 }
@@ -2255,6 +2259,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeObject(pJson, jkSubplanTagCond, (SNode**)&pNode->pTagCond);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond);
+  }
 
   return code;
 }
@@ -3821,7 +3828,7 @@ static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond);
+    code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagCond);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey);
@@ -3850,7 +3857,7 @@ static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) {
     code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond);
+    code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagCond);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey);
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 6ebd0f2d39..b0c2aa7407 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -646,7 +646,7 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyNode(pStmt->pFromTable);
       nodesDestroyNode(pStmt->pWhere);
       nodesDestroyNode(pStmt->pCountFunc);
-      nodesDestroyNode(pStmt->pTagIndexCond);
+      nodesDestroyNode(pStmt->pTagCond);
       break;
     }
     case QUERY_NODE_QUERY: {
@@ -665,7 +665,13 @@ void nodesDestroyNode(SNode* pNode) {
       SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
       destroyLogicNode((SLogicNode*)pLogicNode);
       nodesDestroyList(pLogicNode->pScanCols);
+      nodesDestroyList(pLogicNode->pScanPseudoCols);
       taosMemoryFreeClear(pLogicNode->pVgroupList);
+      nodesDestroyList(pLogicNode->pDynamicScanFuncs);
+      nodesDestroyNode(pLogicNode->pTagCond);
+      nodesDestroyNode(pLogicNode->pTagIndexCond);
+      taosArrayDestroy(pLogicNode->pSmaIndexes);
+      nodesDestroyList(pLogicNode->pPartTags);
       break;
     }
     case QUERY_NODE_LOGIC_PLAN_JOIN: {
@@ -858,6 +864,8 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyList(pSubplan->pChildren);
       nodesDestroyNode((SNode*)pSubplan->pNode);
       nodesDestroyNode((SNode*)pSubplan->pDataSink);
+      nodesDestroyNode((SNode*)pSubplan->pTagCond);
+      nodesDestroyNode((SNode*)pSubplan->pTagIndexCond);
       nodesClearList(pSubplan->pParents);
       break;
     }
@@ -1609,6 +1617,7 @@ int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc) {
 typedef struct SClassifyConditionCxt {
   bool hasPrimaryKey;
   bool hasTagIndexCol;
+  bool hasTagCol;
   bool hasOtherCol;
 } SClassifyConditionCxt;
 
@@ -1620,6 +1629,9 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) {
       pCxt->hasPrimaryKey = true;
     } else if (pCol->hasIndex) {
       pCxt->hasTagIndexCol = true;
+      pCxt->hasTagCol = true;
+    } else if (COLUMN_TYPE_TAG == pCol->colType) {
+      pCxt->hasTagCol = true;
     } else {
       pCxt->hasOtherCol = true;
     }
@@ -1628,23 +1640,31 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-typedef enum EConditionType { COND_TYPE_PRIMARY_KEY = 1, COND_TYPE_TAG_INDEX, COND_TYPE_NORMAL } EConditionType;
+typedef enum EConditionType {
+  COND_TYPE_PRIMARY_KEY = 1,
+  COND_TYPE_TAG_INDEX,
+  COND_TYPE_TAG,
+  COND_TYPE_NORMAL
+} EConditionType;
 
 static EConditionType classifyCondition(SNode* pNode) {
   SClassifyConditionCxt cxt = {.hasPrimaryKey = false, .hasTagIndexCol = false, .hasOtherCol = false};
   nodesWalkExpr(pNode, classifyConditionImpl, &cxt);
   return cxt.hasOtherCol ? COND_TYPE_NORMAL
-                         : (cxt.hasPrimaryKey && cxt.hasTagIndexCol
+                         : (cxt.hasPrimaryKey && cxt.hasTagCol
                                 ? COND_TYPE_NORMAL
-                                : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY : COND_TYPE_TAG_INDEX));
+                                : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY
+                                                     : (cxt.hasTagIndexCol ? COND_TYPE_TAG_INDEX : COND_TYPE_TAG)));
 }
 
-static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) {
+static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond,
+                                  SNode** pOtherCond) {
   SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(*pCondition);
 
   int32_t code = TSDB_CODE_SUCCESS;
 
   SNodeList* pPrimaryKeyConds = NULL;
+  SNodeList* pTagIndexConds = NULL;
   SNodeList* pTagConds = NULL;
   SNodeList* pOtherConds = NULL;
   SNode*     pCond = NULL;
@@ -1656,6 +1676,14 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S
         }
         break;
       case COND_TYPE_TAG_INDEX:
+        if (NULL != pTagIndexCond) {
+          code = nodesListMakeAppend(&pTagIndexConds, nodesCloneNode(pCond));
+        }
+        if (NULL != pTagCond) {
+          code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond));
+        }
+        break;
+      case COND_TYPE_TAG:
         if (NULL != pTagCond) {
           code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond));
         }
@@ -1673,11 +1701,15 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S
   }
 
   SNode* pTempPrimaryKeyCond = NULL;
+  SNode* pTempTagIndexCond = NULL;
   SNode* pTempTagCond = NULL;
   SNode* pTempOtherCond = NULL;
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesMergeConds(&pTempTagIndexCond, &pTagIndexConds);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesMergeConds(&pTempTagCond, &pTagConds);
   }
@@ -1689,6 +1721,9 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S
     if (NULL != pPrimaryKeyCond) {
       *pPrimaryKeyCond = pTempPrimaryKeyCond;
     }
+    if (NULL != pTagIndexCond) {
+      *pTagIndexCond = pTempTagIndexCond;
+    }
     if (NULL != pTagCond) {
       *pTagCond = pTempTagCond;
     }
@@ -1699,9 +1734,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S
     *pCondition = NULL;
   } else {
     nodesDestroyList(pPrimaryKeyConds);
+    nodesDestroyList(pTagIndexConds);
     nodesDestroyList(pTagConds);
     nodesDestroyList(pOtherConds);
     nodesDestroyNode(pTempPrimaryKeyCond);
+    nodesDestroyNode(pTempTagIndexCond);
     nodesDestroyNode(pTempTagCond);
     nodesDestroyNode(pTempOtherCond);
   }
@@ -1709,10 +1746,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S
   return code;
 }
 
-int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) {
+int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond,
+                           SNode** pOtherCond) {
   if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCondition) &&
       LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCondition)->condType) {
-    return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagCond, pOtherCond);
+    return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond);
   }
 
   switch (classifyCondition(*pCondition)) {
@@ -1722,6 +1760,14 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode**
       }
       break;
     case COND_TYPE_TAG_INDEX:
+      if (NULL != pTagIndexCond) {
+        *pTagIndexCond = *pCondition;
+      }
+      if (NULL != pTagCond) {
+        *pTagCond = *pCondition;
+      }
+      break;
+    case COND_TYPE_TAG:
       if (NULL != pTagCond) {
         *pTagCond = *pCondition;
       }
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 3cb885c8e1..66990a8e41 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1917,7 +1917,7 @@ static int32_t getFillTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWin
   }
 
   SNode* pPrimaryKeyCond = NULL;
-  nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL);
+  nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL, NULL);
 
   int32_t code = TSDB_CODE_SUCCESS;
   if (NULL != pPrimaryKeyCond) {
@@ -2503,7 +2503,7 @@ static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet
 
   SNode*  pPrimaryKeyCond = NULL;
   SNode*  pOtherCond = NULL;
-  int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, &pDelete->pTagIndexCond, &pOtherCond);
+  int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, NULL, &pDelete->pTagCond, &pOtherCond);
   if (TSDB_CODE_SUCCESS == code && NULL != pOtherCond) {
     code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE);
   }
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 3cd5eeb655..7fad7416cd 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -1033,8 +1033,8 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p
     }
   }
 
-  if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) {
-    pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond);
+  if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagCond) {
+    pScan->pTagCond = nodesCloneNode(pDelete->pTagCond);
     if (NULL == pScan->pTagCond) {
       code = TSDB_CODE_OUT_OF_MEMORY;
     }
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index a7c25162b7..3a2cbd090c 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -15,7 +15,6 @@
 
 #include "filter.h"
 #include "functionMgt.h"
-#include "index.h"
 #include "planInt.h"
 #include "ttime.h"
 
@@ -307,32 +306,6 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
   return code;
 }
 
-static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** pOtherCond) {
-  int32_t       code = TSDB_CODE_SUCCESS;
-  SIdxFltStatus idxStatus = idxGetFltStatus(*pTagCond);
-  switch (idxStatus) {
-    case SFLT_NOT_INDEX:
-      code = cpdCondAppend(pOtherCond, pTagCond);
-      break;
-    case SFLT_COARSE_INDEX:
-      pScan->pTagCond = nodesCloneNode(*pTagCond);
-      if (NULL == pScan->pTagCond) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        break;
-      }
-      code = cpdCondAppend(pOtherCond, pTagCond);
-      break;
-    case SFLT_ACCURATE_INDEX:
-      pScan->pTagCond = *pTagCond;
-      *pTagCond = NULL;
-      break;
-    default:
-      code = TSDB_CODE_FAILED;
-      break;
-  }
-  return code;
-}
-
 static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
   if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) ||
       TSDB_SYSTEM_TABLE == pScan->tableType) {
@@ -340,15 +313,12 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
   }
 
   SNode*  pPrimaryKeyCond = NULL;
-  SNode*  pTagCond = NULL;
   SNode*  pOtherCond = NULL;
-  int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pTagCond, &pOtherCond);
+  int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond,
+                                    &pOtherCond);
   if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) {
     code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond);
   }
-  if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) {
-    code = cpdApplyTagIndex(pScan, &pTagCond, &pOtherCond);
-  }
   if (TSDB_CODE_SUCCESS == code) {
     pScan->node.pConditions = pOtherCond;
   }
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index 656976b3ca..567df6c4d1 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -436,6 +436,15 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS
     }
   }
 
+  if (TSDB_CODE_SUCCESS == code) {
+    if (NULL != pScanLogicNode->pTagIndexCond) {
+      pSubplan->pTagIndexCond = nodesCloneNode(pScanLogicNode->pTagIndexCond);
+      if (NULL == pSubplan->pTagIndexCond) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+      }
+    }
+  }
+
   if (TSDB_CODE_SUCCESS == code) {
     *pPhyNode = (SPhysiNode*)pScanPhysiNode;
   } else {
diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp
index 84ccea668d..ead3956450 100644
--- a/source/libs/planner/test/planOptimizeTest.cpp
+++ b/source/libs/planner/test/planOptimizeTest.cpp
@@ -40,6 +40,8 @@ TEST_F(PlanOptimizeTest, ConditionPushDown) {
   run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 or tag1 < 2");
 
   run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello'");
+
+  run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello' AND c1 > 10");
 }
 
 TEST_F(PlanOptimizeTest, orderByPrimaryKey) {

From 804e6e95f331ac437c77de32988d68cd15b0bebd Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Mon, 20 Jun 2022 13:26:19 +0800
Subject: [PATCH 02/24] fix: a problem of tag condition double free

---
 source/libs/nodes/src/nodesUtilFuncs.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index b0c2aa7407..3561a7f8ea 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -1764,7 +1764,14 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode**
         *pTagIndexCond = *pCondition;
       }
       if (NULL != pTagCond) {
-        *pTagCond = *pCondition;
+        SNode* pTempCond = *pCondition;
+        if (NULL != pTagIndexCond) {
+          pTempCond = nodesCloneNode(*pCondition);
+          if (NULL == pTempCond) {
+            return TSDB_CODE_OUT_OF_MEMORY;
+          }
+        }
+        *pTagCond = pTempCond;
       }
       break;
     case COND_TYPE_TAG:

From 934d0f9784b3cab31f669aecc64bc54a2393f1e6 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 20 Jun 2022 15:23:48 +0800
Subject: [PATCH 03/24] fix:error in windows

---
 include/common/tmsgdef.h               |  1 +
 source/dnode/mnode/impl/src/mndMain.c  | 57 +++++++++-----------------
 source/dnode/mnode/impl/src/mndTrans.c | 31 ++++++++++++++
 3 files changed, 51 insertions(+), 38 deletions(-)

diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 1b640642d7..d35c5475de 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -144,6 +144,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TTL_TIMER, "ttl-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "kill-trans", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "kill-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "kill-conn", NULL, NULL)
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 95b721b4dd..7a103438dc 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -63,6 +63,21 @@ static void mndPullupTrans(SMnode *pMnode) {
   tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
 }
 
+static void mndTtlTimer(SMnode *pMnode) {
+  int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
+  SMsgHead   *pHead = rpcMallocCont(contLen);
+  if (pHead == NULL) {
+    mError("ttl time malloc err. contLen:%d", contLen);
+    return;
+  }
+
+  int32_t t = taosGetTimestampSec();
+  *(int32_t*)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
+
+  SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pHead, .contLen = contLen};
+  tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+}
+
 static void mndCalMqRebalance(SMnode *pMnode) {
   int32_t contLen = 0;
   void *  pReq = mndBuildTimerMsg(&contLen);
@@ -77,54 +92,20 @@ static void mndPullupTelem(SMnode *pMnode) {
   tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
 }
 
-static void mndPushTtlTime(SMnode *pMnode) {
-  SSdb   *pSdb = pMnode->pSdb;
-  SVgObj *pVgroup = NULL;
-  void   *pIter = NULL;
-
-  while (1) {
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
-    if (pIter == NULL) break;
-
-    int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
-    SMsgHead   *pHead = rpcMallocCont(contLen);
-    if (pHead == NULL) {
-      mError("ttl time malloc err. contLen:%d", contLen);
-      sdbRelease(pSdb, pVgroup);
-      continue;
-    }
-    pHead->contLen = htonl(contLen);
-    pHead->vgId = htonl(pVgroup->vgId);
-
-    int32_t t = taosGetTimestampSec();
-    *(int32_t*)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
-
-    SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
-
-    SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
-    int32_t code = tmsgSendReq(&epSet, &rpcMsg);
-    if(code != 0){
-      mError("ttl time seed err. code:%d", code);
-    }
-    mError("ttl time seed succ. time:%d", t);
-    sdbRelease(pSdb, pVgroup);
-  }
-}
-
 static void *mndThreadFp(void *param) {
   SMnode *pMnode = param;
   int64_t lastTime = 0;
   setThreadName("mnode-timer");
 
   while (1) {
-    if (lastTime % (864000) == 0) {   // sleep 1 day for ttl
-      mndPushTtlTime(pMnode);
-    }
-
     lastTime++;
     taosMsleep(100);
     if (mndGetStop(pMnode)) break;
 
+    if (lastTime % (864000) == 1) {   // sleep 1 day for ttl
+      mndTtlTimer(pMnode);
+    }
+
     if (lastTime % (tsTransPullupInterval * 10) == 0) {
       mndPullupTrans(pMnode);
     }
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 31a955b030..07450613c6 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -21,6 +21,7 @@
 #include "mndShow.h"
 #include "mndSync.h"
 #include "mndUser.h"
+#include "mndVgroup.h"
 
 #define TRANS_VER_NUMBER   1
 #define TRANS_ARRAY_SIZE   8
@@ -56,6 +57,7 @@ static bool    mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->dep
 
 static void    mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
 static int32_t mndProcessTransReq(SRpcMsg *pReq);
+static int32_t mndProcessTtl(SRpcMsg *pReq);
 static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
 
 static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@@ -72,6 +74,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
       .deleteFp = (SdbDeleteFp)mndTransActionDelete,
   };
 
+  mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtl);
   mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
   mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
 
@@ -1346,6 +1349,34 @@ static int32_t mndProcessTransReq(SRpcMsg *pReq) {
   return 0;
 }
 
+static int32_t mndProcessTtl(SRpcMsg *pReq) {
+  SMnode *pMnode = pReq->info.node;
+  SSdb   *pSdb = pMnode->pSdb;
+  SVgObj *pVgroup = NULL;
+  void   *pIter = NULL;
+
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    if (pIter == NULL) break;
+
+    SMsgHead   *pHead = (SMsgHead *)(pReq->pCont);
+
+    pHead->contLen = htonl(pReq->contLen);
+    pHead->vgId = htonl(pVgroup->vgId);
+
+    SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pReq->pCont, .contLen = pReq->contLen};
+
+    SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
+    int32_t code = tmsgSendReq(&epSet, &rpcMsg);
+    if(code != 0){
+      mError("ttl time seed err. code:%d", code);
+    }
+    mError("ttl time seed succ");
+    sdbRelease(pSdb, pVgroup);
+  }
+  return 0;
+}
+
 int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
   SArray *pArray = NULL;
   if (pTrans->stage == TRN_STAGE_REDO_ACTION) {

From f2ad546666dc1e4b1ba8672f879828efbf467f0c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 21 Jun 2022 18:37:47 +0800
Subject: [PATCH 04/24] fix: add configure for ttl unit

---
 include/common/tglobal.h                | 3 +++
 source/common/src/tglobal.c             | 7 +++++++
 source/dnode/mnode/impl/src/mndMain.c   | 2 +-
 source/dnode/mnode/impl/src/mndStb.c    | 2 +-
 source/dnode/vnode/src/meta/metaTable.c | 3 +--
 source/dnode/vnode/src/vnd/vnodeSvr.c   | 2 +-
 6 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 8c03d3ff42..3f68bb4147 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -138,6 +138,9 @@ extern bool tsSmlDataFormat;
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
 
+// ttl unit
+extern int32_t tsTtlUnit;
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 269c92a670..cd7cae8471 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -188,6 +188,8 @@ bool tsStartUdfd = true;
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
 
+int32_t tsTtlUnit = 86400;
+
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
   tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
   tsDiskCfg[index].level = level;
@@ -468,6 +470,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
 
+  if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
+
+
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
   return 0;
 }
@@ -620,6 +625,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
 
+  tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
+
   tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
 
   if (tsQueryBufferSize >= 0) {
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 7a5220d0b2..e0c3b8d2a1 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -95,7 +95,7 @@ static void *mndThreadFp(void *param) {
     taosMsleep(100);
     if (mndGetStop(pMnode)) break;
 
-    if (lastTime % (864000) == 1) {   // sleep 1 day for ttl
+    if (lastTime % (600) == 1) {
       mndTtlTimer(pMnode);
     }
 
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index b5777183db..27c1e558e8 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -805,7 +805,7 @@ static int32_t mndProcessTtl(SRpcMsg *pReq) {
     if(code != 0){
       mError("ttl time seed err. code:%d", code);
     }
-    mError("ttl time seed succ. time:%d", t);
+    mDebug("ttl time seed succ. time:%d", t);
     sdbRelease(pSdb, pVgroup);
   }
   return 0;
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index bf5d5912f9..2bbea593fa 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -400,8 +400,7 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
 
   if (ttlDays <= 0) return;
 
-  ttlKey->dtime = ctime / 1000 + ttlDays * 24 * 60 * 60;
-//  ttlKey->dtime = ctime / 1000 + ttlDays;
+  ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit;
   ttlKey->uid = pME->uid;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index b5a2da091f..c5023185c8 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -309,7 +309,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
   if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
   int32_t t = ntohl(*(int32_t *)pReq);
-  vError("rec ttl time:%d", t);
+  vDebug("rec ttl time:%d", t);
   int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
   if (ret != 0) {
     goto end;

From ab37b6e546edf177d787bbe1faa58f9b96556680 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 22 Jun 2022 11:18:10 +0800
Subject: [PATCH 05/24] refactor: adjust code styles

---
 include/common/tglobal.h              |  2 --
 source/common/src/tglobal.c           |  4 ----
 source/dnode/mnode/impl/src/mndMain.c |  3 +--
 source/dnode/mnode/impl/src/mndStb.c  | 26 +++++++++++++-------------
 source/dnode/vnode/src/vnd/vnodeSvr.c |  2 +-
 5 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 3f68bb4147..c20eff7773 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -137,8 +137,6 @@ extern bool tsSmlDataFormat;
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
-
-// ttl unit
 extern int32_t tsTtlUnit;
 
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index cd7cae8471..877b5ed26f 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -187,7 +187,6 @@ bool tsStartUdfd = true;
 // internal
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
-
 int32_t tsTtlUnit = 86400;
 
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
@@ -469,10 +468,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
-
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
 
-
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
   return 0;
 }
@@ -624,7 +621,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
-
   tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
 
   tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index e0c3b8d2a1..40e435d2c1 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -67,7 +67,6 @@ static void mndTtlTimer(SMnode *pMnode) {
   int32_t contLen = 0;
   void   *pReq = mndBuildTimerMsg(&contLen);
   SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen};
-
   tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
 }
 
@@ -525,7 +524,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
   if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
 
   if (pMsg->msgType != TDMT_MND_MQ_TIMER && pMsg->msgType != TDMT_MND_TELEM_TIMER &&
-      pMsg->msgType != TDMT_MND_TRANS_TIMER) {
+      pMsg->msgType != TDMT_MND_TRANS_TIMER && pMsg->msgType != TDMT_MND_TTL_TIMER) {
     mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
 
     SEpSet epSet = {0};
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 27c1e558e8..e0ac4f8ea8 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -37,7 +37,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndStbActionInsert(SSdb *pSdb, SStbObj *pStb);
 static int32_t  mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
 static int32_t  mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
-static int32_t  mndProcessTtl(SRpcMsg *pReq);
+static int32_t  mndProcessTtlTimer(SRpcMsg *pReq);
 static int32_t  mndProcessCreateStbReq(SRpcMsg *pReq);
 static int32_t  mndProcessAlterStbReq(SRpcMsg *pReq);
 static int32_t  mndProcessDropStbReq(SRpcMsg *pReq);
@@ -63,8 +63,7 @@ int32_t mndInitStb(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtl);
-
+  mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb);
@@ -775,7 +774,7 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
   return 0;
 }
 
-static int32_t mndProcessTtl(SRpcMsg *pReq) {
+static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
   SMnode *pMnode = pReq->info.node;
   SSdb   *pSdb = pMnode->pSdb;
   SVgObj *pVgroup = NULL;
@@ -785,10 +784,10 @@ static int32_t mndProcessTtl(SRpcMsg *pReq) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
 
-    int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
-    SMsgHead   *pHead = rpcMallocCont(contLen);
+    int32_t   contLen = sizeof(SMsgHead) + sizeof(int32_t);
+    SMsgHead *pHead = rpcMallocCont(contLen);
     if (pHead == NULL) {
-      mError("ttl time malloc err. contLen:%d", contLen);
+      sdbCancelFetch(pSdb, pVgroup);
       sdbRelease(pSdb, pVgroup);
       continue;
     }
@@ -796,18 +795,19 @@ static int32_t mndProcessTtl(SRpcMsg *pReq) {
     pHead->vgId = htonl(pVgroup->vgId);
 
     int32_t t = taosGetTimestampSec();
-    *(int32_t*)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
+    *(int32_t *)((char *)pHead + sizeof(SMsgHead)) = htonl(t);
 
     SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
-
-    SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
+    SEpSet  epSet = mndGetVgroupEpset(pMnode, pVgroup);
     int32_t code = tmsgSendReq(&epSet, &rpcMsg);
-    if(code != 0){
-      mError("ttl time seed err. code:%d", code);
+    if (code != 0) {
+      mError("failed to send ttl time seed, code:0x%x", code);
+    } else {
+      mDebug("send ttl time seed success, time:%d", t);
     }
-    mDebug("ttl time seed succ. time:%d", t);
     sdbRelease(pSdb, pVgroup);
   }
+
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index c5023185c8..61f6dd1193 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -309,7 +309,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
   if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
   int32_t t = ntohl(*(int32_t *)pReq);
-  vDebug("rec ttl time:%d", t);
+  vDebug("vgId:%d, recv ttl msg, time:%d", pVnode->config.vgId, t);
   int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
   if (ret != 0) {
     goto end;

From db34294beda2760bfbdad329d3e65cb97bfd85e0 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 22 Jun 2022 16:32:10 +0800
Subject: [PATCH 06/24] opt: filter by tag

---
 include/common/tdatablock.h             |   3 +-
 include/common/tdataformat.h            |   2 +
 source/client/src/clientImpl.c          |   4 +-
 source/common/src/tdatablock.c          |   2 +-
 source/common/src/tdataformat.c         |  12 ++
 source/dnode/vnode/src/meta/metaQuery.c |   3 -
 source/libs/executor/src/scanoperator.c |   4 +-
 source/libs/nodes/src/nodesUtilFuncs.c  |   1 +
 source/libs/scalar/src/sclvector.c      |  40 +++-
 tests/system-test/2-query/json_tag.py   | 266 ++++++++++++------------
 10 files changed, 183 insertions(+), 154 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 709462a744..92af7208a2 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -71,7 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 #define colDataGetData(p1_, r_) \
   ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
 
-#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
+#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \
+                                  (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
 
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
   if (!pColumnInfoData->hasNull) {
diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2e69640a06..f9ede63f7f 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -70,6 +70,8 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
 // STag
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
 void    tTagFree(STag *pTag);
+bool    tTagIsJson(const void *pTag);
+bool    tTagIsJsonNull(void *tagVal);
 bool    tTagGet(const STag *pTag, STagVal *pTagVal);
 char   *tTagValToData(const STagVal *pTagVal, bool isJson);
 int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 8920922006..6a75613fa4 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1581,7 +1581,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
         char*   jsonInnerData = data + CHAR_BYTES;
         if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
           len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L));
-        } else if (jsonInnerType & TD_TAG_JSON) {
+        } else if (tTagIsJson(data)) {
           len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len);
         } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
           len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2;
@@ -1666,7 +1666,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
         if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
           sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
           varDataSetLen(dst, strlen(varDataVal(dst)));
-        } else if (jsonInnerType & TD_TAG_JSON) {
+        } else if (tTagIsJson(data)) {
           char* jsonString = parseTagDatatoJson(data);
           STR_TO_VARSTR(dst, jsonString);
           taosMemoryFree(jsonString);
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 3c3d3e953d..bc46da14e5 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -110,7 +110,7 @@ int32_t getJsonValueLen(const char* data) {
     dataLen = DOUBLE_BYTES + CHAR_BYTES;
   } else if (*data == TSDB_DATA_TYPE_BOOL) {
     dataLen = CHAR_BYTES + CHAR_BYTES;
-  } else if (*data & TD_TAG_JSON) {  // json string
+  } else if (tTagIsJson(data)) {  // json string
     dataLen = ((STag*)(data))->len;
   } else {
     ASSERT(0);
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 1ddb606ccf..8460a27a0e 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -924,6 +924,18 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
 
   return n;
 }
+
+bool tTagIsJson(const void *pTag){
+  return (((const STag *)pTag)->flags & TD_TAG_JSON);
+}
+
+bool tTagIsJsonNull(void *data){
+  STag *pTag = (STag*)data;
+  int8_t   isJson = tTagIsJson(pTag);
+  if(!isJson) return false;
+  return ((STag*)data)->nTag == 0;
+}
+
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
   int32_t  code = 0;
   uint8_t *p = NULL;
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index a5ca90e55f..d655883a76 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -613,9 +613,6 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) {
   ASSERT(pEntry->type == TSDB_CHILD_TABLE);
   STag *tag = (STag *)pEntry->ctbEntry.pTags;
   if (type == TSDB_DATA_TYPE_JSON) {
-    if (tag->nTag == 0) {
-      return NULL;
-    }
     return tag;
   }
   bool find = tTagGet(tag, val);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 8a3f6128fe..4f4fa33830 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -337,7 +337,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_
       }
 
       for (int32_t i = 0; i < pBlock->info.rows; ++i) {
-        colDataAppend(pColInfoData, i, data, (data == NULL));
+        colDataAppend(pColInfoData, i, data, (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
       }
 
       if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL &&
@@ -1824,7 +1824,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
         } else {
           data = (char*)p;
         }
-        colDataAppend(pDst, count, data, (data == NULL));
+        colDataAppend(pDst, count, data, (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
 
         if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) &&
             data != NULL) {
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index e49e4952d3..0a7a203e23 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -1135,6 +1135,7 @@ void* nodesGetValueFromNode(SValueNode* pNode) {
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
+    case TSDB_DATA_TYPE_JSON:
       return (void*)pNode->datum.p;
     default:
       break;
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index bff49c2ca9..50f2fd59d8 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -23,6 +23,7 @@
 #include "sclvector.h"
 #include "tcompare.h"
 #include "tdatablock.h"
+#include "tdataformat.h"
 #include "ttypes.h"
 #include "ttime.h"
 
@@ -506,6 +507,16 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
     }
   }
 
+  // if types can not comparable
+  if((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) ||
+     (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) ||
+     (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) ||
+     (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) ||
+     ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) ||
+     ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL)))
+    return false;
+
+
   if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){
     *isNull = true;
     return true;
@@ -519,24 +530,28 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
 
   *fp = filterGetCompFunc(type, optr);
 
-  if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){
+  if(IS_NUMERIC_TYPE(type)){
     if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
-      convertNcharToDouble(*pLeftData, pLeftOut);
-      *pLeftData = pLeftOut;
+      ASSERT(0);
+//      convertNcharToDouble(*pLeftData, pLeftOut);
+//      *pLeftData = pLeftOut;
     } else if(typeLeft == TSDB_DATA_TYPE_BINARY) {
-      convertBinaryToDouble(*pLeftData, pLeftOut);
-      *pLeftData = pLeftOut;
+      ASSERT(0);
+//      convertBinaryToDouble(*pLeftData, pLeftOut);
+//      *pLeftData = pLeftOut;
     } else if(typeLeft != type) {
       convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
       *pLeftData = pLeftOut;
     }
 
     if(typeRight == TSDB_DATA_TYPE_NCHAR) {
-      convertNcharToDouble(*pRightData, pRightOut);
-      *pRightData = pRightOut;
+      ASSERT(0);
+//      convertNcharToDouble(*pRightData, pRightOut);
+//      *pRightData = pRightOut;
     } else if(typeRight == TSDB_DATA_TYPE_BINARY) {
-      convertBinaryToDouble(*pRightData, pRightOut);
-      *pRightData = pRightOut;
+      ASSERT(0);
+//      convertBinaryToDouble(*pRightData, pRightOut);
+//      *pRightData = pRightOut;
     } else if(typeRight != type) {
       convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
       *pRightData = pRightOut;
@@ -1693,6 +1708,13 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 
 STagVal getJsonValue(char *json, char *key, bool *isExist) {
   STagVal val = {.pKey = key};
+  if (tTagIsJson((const STag *)json) == false){
+    if(isExist){
+      *isExist = false;
+    }
+    return val;
+  }
+
   bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
   if(isExist){
     *isExist = find;
diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py
index 957e916e34..f06104c149 100644
--- a/tests/system-test/2-query/json_tag.py
+++ b/tests/system-test/2-query/json_tag.py
@@ -68,7 +68,7 @@ class TDTestCase:
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')")
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')")
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')")
-        #
+
         # test invalidate json key, key must can be printed assic char
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')")
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')")
@@ -79,7 +79,7 @@ class TDTestCase:
         # test invalidate json value, value number can not be inf,nan TD-12166
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')")
         tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')")
-        #
+
         #test length limit
         char1= ''.join(['abcd']*64)
         char3= ''.join(['abcd']*1021)
@@ -87,15 +87,15 @@ class TDTestCase:
         tdSql.error("CREATE TABLE if not exists  jsons1_15 using  jsons1 tags('{\"%s1\":5}')" % char1)   # len(key)=257
         tdSql.execute("CREATE TABLE if not exists  jsons1_15 using  jsons1 tags('{\"%s\":5}')" % char1)  # len(key)=256
         tdSql.error("CREATE TABLE if not exists  jsons1_16 using  jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3)   # len(object)=4096
-        #tdSql.execute("CREATE TABLE if not exists  jsons1_16 using  jsons1 tags('{\"TSSS\":\"%s\"}')" % char3)  # len(object)=4095
+        tdSql.execute("CREATE TABLE if not exists  jsons1_16 using  jsons1 tags('{\"TSSS\":\"%s\"}')" % char3)  # len(object)=4095
         tdSql.execute("drop table if exists jsons1_15")
         tdSql.execute("drop table if exists jsons1_16")
-        #
+
         print("============== STEP 2 ===== alter table json tag")
         tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)")
         tdSql.error("ALTER STABLE jsons1 drop tag jtag")
         tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)")
-        #
+
         tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'")
         tdSql.query("select jtag from jsons1_1")
         tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}')
@@ -105,9 +105,9 @@ class TDTestCase:
         tdSql.execute("create table st(ts timestamp, i int) tags(t int)")
         tdSql.error("ALTER STABLE st add tag jtag json")
         tdSql.error("ALTER STABLE st add column jtag json")
-        #
-        # print("============== STEP 3 ===== query table")
-        # # test error syntax
+
+        print("============== STEP 3 ===== query table")
+        # test error syntax
         tdSql.error("select * from jsons1 where jtag->tag1='beijing'")
         tdSql.error("select -> from jsons1")
         tdSql.error("select * from jsons1 where contains")
@@ -115,17 +115,17 @@ class TDTestCase:
         tdSql.error("select jtag->location from jsons1")
         tdSql.error("select jtag contains location from jsons1")
         tdSql.error("select * from jsons1 where jtag contains location")
-        #tdSql.error("select * from jsons1 where jtag contains''")
+        tdSql.query("select * from jsons1 where jtag contains''")
         tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'")
-        #
-        # # test function error
+
+        # test function error
         tdSql.error("select avg(jtag->'tag1') from jsons1")
         tdSql.error("select avg(jtag) from jsons1")
         tdSql.error("select min(jtag->'tag1') from jsons1")
         tdSql.error("select min(jtag) from jsons1")
         tdSql.error("select ceil(jtag->'tag1') from jsons1")
         tdSql.error("select ceil(jtag) from jsons1")
-        #
+
 
         #test scalar operation
         tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'")
@@ -158,10 +158,11 @@ class TDTestCase:
         tdSql.checkData(0, 0, None)
         tdSql.checkData(0, 1, False)
         tdSql.checkData(7, 0, "false")
-        tdSql.checkData(7, 1, True)
+        tdSql.checkData(7, 1, False)
+        tdSql.checkData(8, 1, False)
         tdSql.checkData(12, 1, True)
 
-        # # test select normal column
+        # test select normal column
         tdSql.query("select dataint from jsons1 order by dataint")
         tdSql.checkRows(9)
         tdSql.checkData(1, 0, 1)
@@ -180,7 +181,7 @@ class TDTestCase:
         tdSql.query("select jtag from jsons1_9")
         tdSql.checkData(0, 0, None)
 
-        # # test select json tag->'key', value is string
+        # test select json tag->'key', value is string
         tdSql.query("select jtag->'tag1' from jsons1_1")
         tdSql.checkData(0, 0, '"femail"')
         tdSql.query("select jtag->'tag2' from jsons1_6")
@@ -200,7 +201,7 @@ class TDTestCase:
         # test select json tag->'key', key is not exist
         tdSql.query("select jtag->'tag10' from jsons1_4")
         tdSql.checkData(0, 0, None)
-        #
+
         tdSql.query("select jtag->'tag1' from jsons1")
         tdSql.checkRows(13)
         # test header name
@@ -210,24 +211,25 @@ class TDTestCase:
         tdSql.checkColNameList(res, cname_list)
 
 
-        # # test where with json tag
+        # test where with json tag
         tdSql.query("select * from jsons1_1 where jtag is not null")
-        # tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'")
-        # tdSql.error("select * from jsons1 where jtag->'tag1'={}")
-        #
-        # # where json value is string
+        # tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'")
+        tdSql.error("select * from jsons1 where jtag->'tag1'={}")
+
+        # where json value is string
         tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'")
         tdSql.checkRows(2)
-        tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing'")
+        tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing' order by dataint")
         tdSql.checkRows(2)
-	# out of order, cannot compare value
-        #tdSql.checkData(0, 0, 2)
-        #tdSql.checkData(0, 1, 'jsons1_2')
-        #tdSql.checkData(0, 2, 5)
-        #tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}')
-        #tdSql.checkData(1, 0, 3)
-        #tdSql.checkData(1, 1, 'jsons1_3')
-        #tdSql.checkData(1, 2, 'false')
+        tdSql.checkData(0, 0, 2)
+        tdSql.checkData(0, 1, 'jsons1_2')
+        tdSql.checkData(0, 2, "5.000000000")
+        tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}')
+        tdSql.checkData(1, 0, 3)
+        tdSql.checkData(1, 1, 'jsons1_3')
+        tdSql.checkData(1, 2, 'false')
+
+
         tdSql.query("select * from jsons1 where jtag->'tag1'='beijing'")
         tdSql.checkRows(0)
         tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'")
@@ -236,72 +238,73 @@ class TDTestCase:
         tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'")
         tdSql.checkRows(3)
-	# open
-        #tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'")
-        #tdSql.checkRows(2)
-        tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'")
+        tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'")
         tdSql.checkRows(2)
+        tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'")
+        tdSql.checkRows(4)
         tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'")
-        tdSql.checkRows(5)
+        tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag2'=''")
         tdSql.checkRows(2)
-        #
-        # # where json value is int
+
+        # where json value is int
         tdSql.query("select * from jsons1 where jtag->'tag1'=5")
         tdSql.checkRows(1)
         tdSql.checkData(0, 1, 2)
         tdSql.query("select * from jsons1 where jtag->'tag1'=10")
         tdSql.checkRows(0)
         tdSql.query("select * from jsons1 where jtag->'tag1'<54")
-        tdSql.checkRows(4)
+        tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1'<=11")
-        tdSql.checkRows(4)
+        tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1'>4")
         tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where jtag->'tag1'>=5")
         tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where jtag->'tag1'!=5")
-        tdSql.checkRows(6)
+        tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where jtag->'tag1'!=55")
-        tdSql.checkRows(7)
-        #
-        # # where json value is double
+        tdSql.checkRows(3)
+
+        # where json value is double
         tdSql.query("select * from jsons1 where jtag->'tag1'=1.232")
         tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag1'<1.232")
-        tdSql.checkRows(1)
+        tdSql.checkRows(0)
         tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232")
-        tdSql.checkRows(2)
+        tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag1'>1.23")
         tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232")
         tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232")
-        tdSql.checkRows(6)
+        tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232")
-        tdSql.checkRows(7)
-        #tdSql.error("select * from jsons1 where jtag->'tag1'/0=3")
-        #tdSql.error("select * from jsons1 where jtag->'tag1'/5=1")
-        #
-        # # where json value is bool
+        tdSql.checkRows(3)
+        tdSql.query("select * from jsons1 where jtag->'tag1'/0=3")
+        tdSql.checkRows(0)
+        tdSql.query("select * from jsons1 where jtag->'tag1'/5=1")
+        tdSql.checkRows(1)
+
+        # where json value is bool
         tdSql.query("select * from jsons1 where jtag->'tag1'=true")
         tdSql.checkRows(0)
-        #tdSql.query("select * from jsons1 where jtag->'tag1'=false")
-        #tdSql.checkRows(1)
+        tdSql.query("select * from jsons1 where jtag->'tag1'=false")
+        tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag1'!=false")
-        tdSql.checkRows(3)
-        #tdSql.error("select * from jsons1 where jtag->'tag1'>false")
-        #
-        # # where json value is null
-	# open
-        #tdSql.query("select * from jsons1 where jtag->'tag1'=null")     # only json suport =null. This synatx will change later.
-        #tdSql.checkRows(1)
-        #
-        # # where json key is null
+        tdSql.checkRows(0)
+        tdSql.query("select * from jsons1 where jtag->'tag1'>false")
+        tdSql.checkRows(0)
+
+        # where json value is null
+        tdSql.query("select * from jsons1 where jtag->'tag1'=null")
+        tdSql.checkRows(0)
+
+        # where json key is null
         tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3")
         tdSql.checkRows(0)
-        #
-        # # where json value is not exist
+
+        # where json value is not exist
         tdSql.query("select * from jsons1 where jtag->'tag1' is null")
         tdSql.checkData(0, 0, 'jsons1_9')
         tdSql.checkRows(2)
@@ -309,16 +312,16 @@ class TDTestCase:
         tdSql.checkRows(9)
         tdSql.query("select * from jsons1 where jtag->'tag3' is not null")
         tdSql.checkRows(3)
-        #
-        # # test contains
+
+        # test contains
         tdSql.query("select * from jsons1 where jtag contains 'tag1'")
-        tdSql.checkRows(7)
+        tdSql.checkRows(8)
         tdSql.query("select * from jsons1 where jtag contains 'tag3'")
-        tdSql.checkRows(3)
+        tdSql.checkRows(4)
         tdSql.query("select * from jsons1 where jtag contains 'tag_no_exist'")
         tdSql.checkRows(0)
-        #
-        # # test json tag in where condition with and/or
+
+        # test json tag in where condition with and/or
         tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'")
         tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'")
@@ -335,15 +338,15 @@ class TDTestCase:
         tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag contains 'tag3'")
         tdSql.checkRows(2)
-        #
-        #
-        # # test with between and
+
+
+        # test with between and
         tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30")
         tdSql.checkRows(3)
         tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'")
         tdSql.checkRows(2)
-        #
-        # # test with tbname/normal column
+
+        # test with tbname/normal column
         tdSql.query("select * from jsons1 where tbname = 'jsons1_1'")
         tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'")
@@ -352,20 +355,18 @@ class TDTestCase:
         tdSql.checkRows(0)
         tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=23")
         tdSql.checkRows(1)
-        #
-        #
-        # # test where condition like
-	# open
- 	# syntax error	
-        #tdSql.query("select *,tbname from jsons1 where jtag->'tag2' like 'bei%'")
-        #tdSql.checkRows(2)
-        #tdSql.query("select *,tbname from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null")
-        #tdSql.checkRows(2)
-        #
-        # # test where condition in  no support in
+
+
+        # test where condition like
+        tdSql.query("select * from jsons1 where jtag->'tag2' like 'bei%'")
+        tdSql.checkRows(2)
+        tdSql.query("select * from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null")
+        tdSql.checkRows(2)
+
+        # test where condition in  no support in
         # tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')")
-        #
-        # # test where condition match/nmath
+
+        # test where condition match/nmath
         tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'")
         tdSql.checkRows(2)
         tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma$'")
@@ -376,23 +377,22 @@ class TDTestCase:
         tdSql.checkRows(1)
         tdSql.query("select * from jsons1 where jtag->'tag1' nmatch 'ma'")
         tdSql.checkRows(1)
-        #
-        # # test distinct
+
+        # test distinct
         tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')")
         tdSql.query("select distinct jtag->'tag1' from jsons1")
         tdSql.checkRows(8)
         tdSql.query("select distinct jtag from jsons1")
         tdSql.checkRows(9)
-        #
-        # #test dumplicate key with normal colomn
+
+        #test dumplicate key with normal colomn
         tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")")
-        #tdSql.query("select *,tbname,jtag from jsons1 where jtag->'datastr' match '是' and datastr match 'js'")
-        #tdSql.checkRows(1)
-	# open
-        #tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'")
-        #tdSql.checkRows(0)
-        #
-        # # test join
+        tdSql.query("select * from jsons1 where jtag->'datastr' match '是' and datastr match 'js'")
+        tdSql.checkRows(1)
+        # tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'")
+        # tdSql.checkRows(1)
+
+        # test join
         tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)")
         tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')")
         tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')")
@@ -460,19 +460,18 @@ class TDTestCase:
         tdSql.checkColNameList(res, cname_list)
 
         # test top/bottom with group by json tag
-	# random failure 
-        #tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
-        #tdSql.checkRows(11)
-        #tdSql.checkData(0, 1, None)
-        #tdSql.checkData(2, 0, 4)
-        #tdSql.checkData(3, 0, 3)
-        #tdSql.checkData(3, 1, "false")
-        #tdSql.checkData(10, 0, 23)
-        #tdSql.checkData(10, 1, '"femail"')
+        # tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
+        # tdSql.checkRows(11)
+        # tdSql.checkData(0, 1, None)
+        # tdSql.checkData(2, 0, 4)
+        # tdSql.checkData(3, 0, 3)
+        # tdSql.checkData(3, 1, "false")
+        # tdSql.checkData(8, 0, 2)
+        # tdSql.checkData(10, 1, '"femail"')
 
         # test having
-        #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1")
-        #tdSql.checkRows(3)
+        tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1")
+        tdSql.checkRows(3)
 
         # subquery with json tag
         tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint")
@@ -480,22 +479,13 @@ class TDTestCase:
         tdSql.checkData(1, 1, 1)
         tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}')
 
-        # tdSql.query("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)")
-        # tdSql.checkRows(11)
-        # tdSql.checkData(1, 0, '"femail"')
-        # tdSql.checkData(2, 0, 5)
-        #
-        # res = tdSql.getColNameList("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)")
-        # cname_list = []
-        # cname_list.append("jtag->'tag1'")
-        # tdSql.checkColNameList(res, cname_list)
-        #
-        # tdSql.query("select ts,tbname,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)")
+        tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)")
+        # tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)")
         # tdSql.checkRows(11)
         # tdSql.checkData(1, 1, "jsons1_1")
         # tdSql.checkData(1, 2, '"femail"')
-        #
-        # # union all
+
+        # union all
         tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2")
         tdSql.checkRows(17)
         tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1")
@@ -517,15 +507,16 @@ class TDTestCase:
         tdSql.checkData(0, 0, 10)
         tdSql.query("select avg(dataint) from jsons1 where jtag is not null")
         tdSql.checkData(0, 0, 5.3)
-        #tdSql.error("select twa(dataint) from jsons1 where jtag is not null")
+        # tdSql.query("select twa(dataint) from jsons1 where jtag is not null")
+        # tdSql.checkData(0, 0, 36)
         tdSql.error("select irate(dataint) from jsons1 where jtag is not null")
-        #tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null")
-        #tdSql.checkData(0, 0, 49)
+        tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null")
+        tdSql.checkData(0, 0, 45)
         tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1")
         tdSql.checkData(0, 0, 4.496912521)
-        #tdSql.error("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null")
-        #
-        # #test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp
+        tdSql.query("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null")
+
+        #test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp
         tdSql.query("select min(dataint) from jsons1 where jtag->'tag1'>1")
         tdSql.checkData(0, 0, 1)
         tdSql.query("select max(dataint) from jsons1 where jtag->'tag1'>1")
@@ -541,13 +532,16 @@ class TDTestCase:
         tdSql.query("select percentile(dataint,20) from jsons1 where jtag->'tag1'>1")
         tdSql.query("select apercentile(dataint, 50) from jsons1 where jtag->'tag1'>1")
         tdSql.checkData(0, 0, 1.5)
-        #tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1")
-        #tdSql.checkData(0, 0, 11)
-        #tdSql.error("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1")
-        #
-        # #test calculation function:diff/derivative/spread/ceil/floor/round/
-        #tdSql.error("select diff(dataint) from jsons1 where jtag->'tag1'>1")
-        #tdSql.error("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1")
+        # tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1")
+        # tdSql.query("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1")
+
+        #test calculation function:diff/derivative/spread/ceil/floor/round/
+        tdSql.query("select diff(dataint) from jsons1 where jtag->'tag1'>1")
+        # tdSql.checkRows(2)
+        # tdSql.checkData(0, 0, -1)
+        # tdSql.checkData(1, 0, 10)
+        tdSql.query("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1")
+        tdSql.checkData(0, 0, -2)
         tdSql.query("select spread(dataint) from jsons1 where jtag->'tag1'>1")
         tdSql.checkData(0, 0, 10)
         tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1")

From c1b310520a5acade3d1902d41801731e9c6aa36c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 22 Jun 2022 17:02:03 +0800
Subject: [PATCH 07/24] opt: filter by tag

---
 tests/system-test/2-query/json_tag.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py
index f06104c149..ff7adfc759 100644
--- a/tests/system-test/2-query/json_tag.py
+++ b/tests/system-test/2-query/json_tag.py
@@ -498,11 +498,11 @@ class TDTestCase:
         tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2")
         tdSql.checkRows(13)
 
-        # #show create table
-        # tdSql.query("show create table jsons1")
-        # tdSql.checkData(0, 1, 'CREATE TABLE `jsons1` (`ts` TIMESTAMP,`dataint` INT,`databool` BOOL,`datastr` NCHAR(50),`datastrbin` BINARY(150)) TAGS (`jtag` JSON)')
-        #
-        # #test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares
+        #show create table
+        tdSql.query("show create table jsons1")
+        tdSql.checkData(0, 1, 'CREATE STABLE `jsons1` (`ts` TIMESTAMP, `dataint` INT, `databool` BOOL, `datastr` NCHAR(50), `datastrbin` VARCHAR(150)) TAGS (`jtag` JSON) WATERMARK 5000a, 5000a')
+
+        #test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares
         tdSql.query("select count(*) from jsons1 where jtag is not null")
         tdSql.checkData(0, 0, 10)
         tdSql.query("select avg(dataint) from jsons1 where jtag is not null")

From c234adbb857512a0f63b8f52df26007c44f35776 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 22 Jun 2022 17:07:49 +0800
Subject: [PATCH 08/24] shell: fix shell chinese error

---
 include/os/os.h                |  2 --
 source/common/src/tglobal.c    |  2 +-
 source/os/src/osLocale.c       |  4 +--
 tests/script/wtest.bat         |  2 +-
 tools/shell/src/shellCommand.c | 60 ++++++++++++++++++++++++++++------
 tools/shell/src/shellEngine.c  |  4 ---
 6 files changed, 54 insertions(+), 20 deletions(-)

diff --git a/include/os/os.h b/include/os/os.h
index 41180ba49e..254c16efbe 100644
--- a/include/os/os.h
+++ b/include/os/os.h
@@ -104,8 +104,6 @@ extern "C" {
 #include "osTimezone.h"
 #include "osEnv.h"
 
-void osDefaultInit();
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 269c92a670..dca70ce76b 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -631,7 +631,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
                       const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
-  osDefaultInit();
+  if (tsCfg == NULL) osDefaultInit();
 
   SConfig *pCfg = cfgInit();
   if (pCfg == NULL) return -1;
diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c
index d2369ea2a2..c2d10f3668 100644
--- a/source/os/src/osLocale.c
+++ b/source/os/src/osLocale.c
@@ -88,11 +88,11 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
 
 void taosGetSystemLocale(char *outLocale, char *outCharset) {
 #ifdef WINDOWS
-  char *locale = setlocale(LC_CTYPE, "chs");
+  char *locale = setlocale(LC_CTYPE, "en_US.UTF-8");
   if (locale != NULL) {
     tstrncpy(outLocale, locale, TD_LOCALE_LEN);
   }
-  strcpy(outCharset, "cp936");
+  strcpy(outCharset, "UTF-8");
 
 #elif defined(_TD_DARWIN_64)
   /*
diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat
index 1aa27202c6..1f089c9d86 100644
--- a/tests/script/wtest.bat
+++ b/tests/script/wtest.bat
@@ -52,7 +52,7 @@ echo wal            0             >> %TAOS_CFG%
 echo asyncLog       0             >> %TAOS_CFG%
 echo locale         en_US.UTF-8   >> %TAOS_CFG%
 echo enableCoreFile 1             >> %TAOS_CFG%
-echo charset        cp65001       >> %TAOS_CFG%
+echo charset        UTF-8         >> %TAOS_CFG%
 
 set "FILE_NAME=testSuite.sim"
 if "%1" == "-f" set "FILE_NAME=%2"
diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c
index ef71f3fce6..ddd4f7a59d 100644
--- a/tools/shell/src/shellCommand.c
+++ b/tools/shell/src/shellCommand.c
@@ -21,6 +21,7 @@
 #define UP    3
 #define DOWN  4
 #define PSIZE shell.info.promptSize
+#define SHELL_INPUT_MAX_COMMAND_SIZE 10000
 
 typedef struct {
   char    *buffer;
@@ -227,6 +228,7 @@ void shellPrintChar(char c, int32_t times) {
 }
 
 void shellPositionCursor(int32_t step, int32_t direction) {
+#ifndef WINDOWS
   if (step > 0) {
     if (direction == LEFT) {
       fprintf(stdout, "\033[%dD", step);
@@ -239,6 +241,7 @@ void shellPositionCursor(int32_t step, int32_t direction) {
     }
     fflush(stdout);
   }
+#endif
 }
 
 void shellUpdateBuffer(SShellCmd *cmd) {
@@ -330,10 +333,14 @@ void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) {
   int32_t command_x = ecmd_pos / ws_col;
   shellPositionCursor(cursor_y, LEFT);
   shellPositionCursor(command_x - cursor_x, DOWN);
+#ifndef WINDOWS
   fprintf(stdout, "\033[2K");
+#endif
   for (int32_t i = 0; i < command_x; i++) {
     shellPositionCursor(1, UP);
+  #ifndef WINDOWS
     fprintf(stdout, "\033[2K");
+  #endif
   }
   fflush(stdout);
 }
@@ -394,6 +401,38 @@ void shellShowOnScreen(SShellCmd *cmd) {
   fflush(stdout);
 }
 
+char taosGetConsoleChar() {
+#ifdef WINDOWS
+  static void *console = NULL;
+  if (console == NULL) {
+    console = GetStdHandle(STD_INPUT_HANDLE);
+  }
+  static TdWchar buf[SHELL_INPUT_MAX_COMMAND_SIZE];
+  static char mbStr[5];
+  static unsigned long bufLen = 0;
+  static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0;
+  if (bufLen == 0) {
+    ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL);
+    bufIndex = 0;
+  }
+  if (mbStrLen == 0){
+    mbStrLen = WideCharToMultiByte(CP_UTF8, 0, &buf[bufIndex], 1, mbStr, sizeof(mbStr), NULL, NULL);
+    mbStrIndex = 0;
+    bufIndex++;
+  }
+  mbStrIndex++;
+  if (mbStrIndex == mbStrLen) {
+    mbStrLen = 0;
+    if (bufIndex == bufLen) {
+      bufLen = 0;
+    }
+  }
+  return mbStr[mbStrIndex-1];
+#else
+  return (char)getchar();  // getchar() return an 'int32_t' value
+#endif
+}
+
 int32_t shellReadCommand(char *command) {
   SShellHistory *pHistory = &shell.history;
   SShellCmd      cmd = {0};
@@ -407,7 +446,7 @@ int32_t shellReadCommand(char *command) {
   // Read input.
   char c;
   while (1) {
-    c = (char)getchar();  // getchar() return an 'int32_t' value
+    c = taosGetConsoleChar();
 
     if (c == EOF) {
       return c;
@@ -417,7 +456,7 @@ int32_t shellReadCommand(char *command) {
       int32_t count = shellCountPrefixOnes(c);
       utf8_array[0] = c;
       for (int32_t k = 1; k < count; k++) {
-        c = (char)getchar();
+        c = taosGetConsoleChar();
         utf8_array[k] = c;
       }
       shellInsertChar(&cmd, utf8_array, count);
@@ -446,6 +485,7 @@ int32_t shellReadCommand(char *command) {
           shellBackspaceChar(&cmd);
           break;
         case '\n':
+          break;
         case '\r':
         #ifdef WINDOWS 
         #else
@@ -472,10 +512,10 @@ int32_t shellReadCommand(char *command) {
           break;
       }
     } else if (c == '\033') {
-      c = (char)getchar();
+      c = taosGetConsoleChar();
       switch (c) {
         case '[':
-          c = (char)getchar();
+          c = taosGetConsoleChar();
           switch (c) {
             case 'A':  // Up arrow
               if (hist_counter != pHistory->hstart) {
@@ -502,35 +542,35 @@ int32_t shellReadCommand(char *command) {
               shellMoveCursorLeft(&cmd);
               break;
             case '1':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // Home key
                 shellPositionCursorHome(&cmd);
               }
               break;
             case '2':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // Insert key
               }
               break;
             case '3':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // Delete key
                 shellDeleteChar(&cmd);
               }
               break;
             case '4':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // End key
                 shellPositionCursorEnd(&cmd);
               }
               break;
             case '5':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // Page up key
               }
               break;
             case '6':
-              if ((c = (char)getchar()) == '~') {
+              if ((c = taosGetConsoleChar()) == '~') {
                 // Page down key
               }
               break;
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 1f29237d38..2f8a21e47c 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -393,15 +393,11 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
       break;
     }
     int w = 0;
-#ifdef WINDOWS
-    w = bytes;
-#else
     if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
       w = bytes;
     }else{
       w = taosWcharWidth(wc);
     }
-#endif
     pos += bytes;
 
     if (w <= 0) {

From d9dc806a210624692ab2f59d05163410fde0cecd Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 22 Jun 2022 17:11:18 +0800
Subject: [PATCH 09/24] feat: add configure for ttl

---
 include/common/tglobal.h              | 1 +
 source/common/src/tglobal.c           | 4 ++++
 source/dnode/mnode/impl/src/mndMain.c | 2 +-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index c20eff7773..e083ebcf78 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -138,6 +138,7 @@ extern bool tsSmlDataFormat;
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
 extern int32_t tsTtlUnit;
+extern int32_t tsTtlPushInterval;
 
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 877b5ed26f..a30eb3fcef 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -188,6 +188,8 @@ bool tsStartUdfd = true;
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsTtlUnit = 86400;
+int32_t tsTtlPushInterval = 60;
+
 
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
   tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
@@ -469,6 +471,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1;
 
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
   return 0;
@@ -622,6 +625,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
   tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
+  tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32;
 
   tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
 
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index d7cc1a5780..9965b803ea 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -100,7 +100,7 @@ static void *mndThreadFp(void *param) {
     taosMsleep(100);
     if (mndGetStop(pMnode)) break;
 
-    if (lastTime % 600 == 1) {
+    if (lastTime % (tsTransPullupInterval * 10) == 1) {
       mndTtlTimer(pMnode);
     }
 

From 5b158b4c18003f401d5f7ededf6a1024ced074ed Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Wed, 22 Jun 2022 19:26:48 +0800
Subject: [PATCH 10/24] test: add test case for tmq

---
 tests/system-test/7-tmq/tmqCheckData.py | 182 ++++++++++++++++++++++++
 tests/system-test/fulltest.sh           |   1 +
 tests/test/c/tmqSim.c                   | 166 +++++++++++++++++++--
 3 files changed, 338 insertions(+), 11 deletions(-)
 create mode 100644 tests/system-test/7-tmq/tmqCheckData.py

diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py
new file mode 100644
index 0000000000..0e55dfa19d
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqCheckData.py
@@ -0,0 +1,182 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def checkFileContent(self, consumerId, queryString):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+        cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+        
+        consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+        tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+        consumeFile = open(consumeRowsFile, mode='r')
+        queryFile = open(dstFile, mode='r')
+        
+        # skip first line for it is schema
+        queryFile.readline()
+
+        while True:
+            dst = queryFile.readline()
+            src = consumeFile.readline()
+            
+            if dst:
+                if dst != src:
+                    tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId)
+            else:
+                break
+        return 
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db1',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        topicNameList = ['topic1', 'topic2', 'topic3']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tdLog.info("create ctb")
+        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+        tdLog.info("insert data")
+        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the consume result") 
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+            tdLog.exit("0 tmq consume rows error!")
+
+        self.checkFileContent(consumerId, queryString)     
+
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+
+        queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+
+        consumerId   = 1
+        topicList    = topicNameList[1]
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the consume result") 
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        if expectRowsList[1] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0]))
+            tdLog.exit("1 tmq consume rows error!")
+
+        self.checkFileContent(consumerId, queryString)
+
+        # reinit consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+
+        queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137)
+        sqlString = "create topic %s as %s" %(topicNameList[2], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())                
+
+        consumerId   = 2
+        topicList    = topicNameList[2]
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the consume result") 
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        # if expectRowsList[2] != resultList[0]:
+        #     tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0]))
+        #     tdLog.exit("2 tmq consume rows error!")
+
+        # self.checkFileContent(consumerId, queryString)
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 0e7c4673ee..63d3fe4c63 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -132,3 +132,4 @@ python3 ./test.py -f 7-tmq/db.py
 python3 ./test.py -f 7-tmq/tmqError.py
 python3 ./test.py -f 7-tmq/schema.py
 python3 ./test.py -f 7-tmq/stbFilter.py
+python3 ./test.py -f 7-tmq/tmqCheckData.py
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index 948df3a40a..c28c676a87 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -24,6 +24,8 @@
 #include "taos.h"
 #include "taoserror.h"
 #include "tlog.h"
+#include "taosdef.h"
+#include "types.h"
 
 #define GREEN     "\033[1;32m"
 #define NC        "\033[0m"
@@ -49,6 +51,7 @@ typedef struct {
   // char     autoCommit[8];         // true, false
   // char     autoOffsetRest[16];    // none, earliest, latest
 
+  TdFilePtr pConsumeRowsFile;
   int32_t ifCheckData;
   int64_t expectMsgCnt;
 
@@ -129,7 +132,6 @@ void initLogFile() {
   char tmpString[128];
 
   sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString));
-  // sprintf(filename, "%s/../log/tmqlog.txt", configDir);
 #ifdef WINDOWS
   for (int i = 2; i < sizeof(filename); i++) {
     if (filename[i] == ':') filename[i] = '-';
@@ -296,35 +298,165 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) {
   return 0;
 }
 
+static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
+  //if (shell.args.is_raw_time) {
+  //  sprintf(buf, "%" PRId64, val);
+  //  return buf;
+  //}
+
+  time_t  tt;
+  int32_t ms = 0;
+  if (precision == TSDB_TIME_PRECISION_NANO) {
+    tt = (time_t)(val / 1000000000);
+    ms = val % 1000000000;
+  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+    tt = (time_t)(val / 1000000);
+    ms = val % 1000000;
+  } else {
+    tt = (time_t)(val / 1000);
+    ms = val % 1000;
+  }
+
+  /*
+    comment out as it make testcases like select_with_tags.sim fail.
+    but in windows, this may cause the call to localtime crash if tt < 0,
+    need to find a better solution.
+    if (tt < 0) {
+      tt = 0;
+    }
+  */
+
+#ifdef WINDOWS
+  if (tt < 0) tt = 0;
+#endif
+  if (tt <= 0 && ms < 0) {
+    tt--;
+    if (precision == TSDB_TIME_PRECISION_NANO) {
+      ms += 1000000000;
+    } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+      ms += 1000000;
+    } else {
+      ms += 1000;
+    }
+  }
+
+  struct tm *ptm = taosLocalTime(&tt, NULL);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+
+  if (precision == TSDB_TIME_PRECISION_NANO) {
+    sprintf(buf + pos, ".%09d", ms);
+  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+    sprintf(buf + pos, ".%06d", ms);
+  } else {
+    sprintf(buf + pos, ".%03d", ms);
+  }
+
+  return buf;
+}
+
+static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) {
+  if (val == NULL) {
+    taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
+    return;
+  }
+
+  int  n;
+  char buf[TSDB_MAX_BYTES_PER_ROW];
+  switch (field->type) {
+    case TSDB_DATA_TYPE_BOOL:
+      taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      taosFprintfFile(pFile, "%d", *((int8_t *)val));
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      taosFprintfFile(pFile, "%u", *((uint8_t *)val));
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      taosFprintfFile(pFile, "%d", *((int16_t *)val));
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      taosFprintfFile(pFile, "%u", *((uint16_t *)val));
+      break;
+    case TSDB_DATA_TYPE_INT:
+      taosFprintfFile(pFile, "%d", *((int32_t *)val));
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      taosFprintfFile(pFile, "%u", *((uint32_t *)val));
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val));
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
+      if (n > TMAX(25, length)) {
+        taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val));
+      } else {
+        taosFprintfFile(pFile, "%s", buf);
+      }
+      break;
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
+      memcpy(buf, val, length);
+      buf[length] = 0;
+      taosFprintfFile(pFile, "\'%s\'", buf);
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      shellFormatTimestamp(buf, *(int64_t *)val, precision);
+      taosFprintfFile(pFile, "'%s'", buf);
+      break;
+    default:
+      break;
+  }
+}
+
+static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, int32_t precision) {
+  for (int32_t i = 0; i < num_fields; i++) {
+    if (i > 0) {
+      taosFprintfFile(pFile, "\n");
+    }
+    shellDumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
+  }
+  taosFprintfFile(pFile, "\n");
+}
+
 static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) {
   char    buf[1024];
   int32_t totalRows = 0;
 
   // printf("topic: %s\n", tmq_get_topic_name(msg));
   int32_t vgroupId = tmq_get_vgroup_id(msg);
+  const char*   dbName = tmq_get_db_name(msg);
 
-  taosFprintfFile(g_fp, "msg index:%" PRId64 ", consumerId: %d\n", msgIndex, pInfo->consumerId);
-  // taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId,
-  // tmq_get_table_name(msg));
-  taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), vgroupId);
+  taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex);
+  taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId);
 
   while (1) {
     TAOS_ROW row = taos_fetch_row(msg);
 
     if (row == NULL) break;
 
-    TAOS_FIELD* fields = taos_fetch_fields(msg);
+    TAOS_FIELD* fields      = taos_fetch_fields(msg);
     int32_t     numOfFields = taos_field_count(msg);
+	int32_t*    length      = taos_fetch_lengths(msg);
+	int32_t     precision   = taos_result_precision(msg);
+    const char* tbName      = tmq_get_table_name(msg);
 
+	dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision);
     taos_print_row(buf, row, fields, numOfFields);
 
-    const char* tbName = tmq_get_table_name(msg);
-
     if (0 != g_stConfInfo.showRowFlag) {
       taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf);
-      if (0 != g_stConfInfo.saveRowFlag) {
-        saveConsumeContentToTbl(pInfo, buf);
-      }
+      //if (0 != g_stConfInfo.saveRowFlag) {
+      //  saveConsumeContentToTbl(pInfo, buf);
+      //}
     }
 
     totalRows++;
@@ -463,6 +595,18 @@ void loop_consume(SThreadInfo* pInfo) {
                   pInfo->consumerId);
 
   pInfo->ts = taosGetTimestampMs();
+  
+  if (pInfo->ifCheckData) {
+  	char filename[256] = {0};
+    char tmpString[128];
+	//sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, getCurrentTimeString(tmpString));
+	sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId);
+    pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);
+    if (pInfo->pConsumeRowsFile == NULL) {
+      taosFprintfFile(g_fp, "%s create file fail for save rows content\n", getCurrentTimeString(tmpString));
+      return;
+    }
+  }
 
   while (running) {
     TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);

From f05c431e848e3cbff37a3a7227f15ed00c30513f Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 22 Jun 2022 19:55:00 +0800
Subject: [PATCH 11/24] feat: balance vgroup

---
 source/dnode/mnode/impl/src/mndDnode.c        |   1 +
 source/dnode/mnode/impl/src/mndVgroup.c       |  35 +++--
 tests/script/jenkins/basic.txt                |   5 +
 tests/script/tsim/dnode/balance_replica1.sim  | 123 ++++++++++++++++++
 .../dnode/drop_dnode_has_vnode_replica3.sim   |   9 +-
 5 files changed, 164 insertions(+), 9 deletions(-)
 create mode 100644 tests/script/tsim/dnode/balance_replica1.sim

diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 8d06868955..e58fe0835a 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -580,6 +580,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
     mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
     if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
   }
+
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index 696e714a8c..ae13987d25 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -1553,10 +1553,11 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
 static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
   void   *pIter = NULL;
   int32_t code = -1;
+  SSdb   *pSdb = pMnode->pSdb;
 
   while (1) {
     SVgObj *pVgroup = NULL;
-    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
 
     bool existInSrc = false;
@@ -1568,13 +1569,15 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
     }
 
     if (!existInSrc || existInDst) {
-      sdbRelease(pMnode->pSdb, pVgroup);
+      sdbRelease(pSdb, pVgroup);
+      continue;
     }
 
     SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
     code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
     mndReleaseDb(pMnode, pDb);
-    sdbRelease(pMnode->pSdb, pVgroup);
+    sdbRelease(pSdb, pVgroup);
+    sdbCancelFetch(pSdb, pIter);
     break;
   }
 
@@ -1593,15 +1596,25 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
 
   while (1) {
     taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
-    SDnodeObj *pSrc = taosArrayGet(pArray, 0);
-    SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
+    for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+      SDnodeObj *pDnode = taosArrayGet(pArray, i);
+      mDebug("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
+             pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
+    }
+
+    SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
+    SDnodeObj *pDst = taosArrayGet(pArray, 0);
 
     float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
     float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
-    if (srcScore + 0.0001 < dstScore) {
-      mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
+    mDebug("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore,
+           pDst->id, dstScore);
+
+    if (srcScore > dstScore - 0.000001) {
       code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
       if (code == 0) {
+        pSrc->numOfVnodes--;
+        pDst->numOfVnodes++;
         numOfVgroups++;
         continue;
       } else {
@@ -1635,7 +1648,13 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
   void   *pIter = NULL;
   int64_t curMs = taosGetTimestampMs();
 
-  mDebug("start to balance vgroup");
+  SBalanceVgroupReq req = {0};
+  if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _OVER;
+  }
+
+  mInfo("start to balance vgroup");
 
   if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER;
 
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 1ab6491736..dc2002d4d0 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -12,6 +12,8 @@
 ## ---- db
 ./test.sh -f tsim/db/create_all_options.sim
 ./test.sh -f tsim/db/alter_option.sim
+#./test.sh -f tsim/db/alter_replica_13.sim
+#./test.sh -f tsim/db/alter_replica_31.sim
 ./test.sh -f tsim/db/basic1.sim
 ./test.sh -f tsim/db/basic2.sim
 ./test.sh -f tsim/db/basic3.sim
@@ -21,6 +23,7 @@
 ./test.sh -f tsim/db/taosdlog.sim
 
 # ---- dnode
+./test.sh -f tsim/dnode/balance_replica1.sim
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@@ -28,6 +31,8 @@
 #./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim
diff --git a/tests/script/tsim/dnode/balance_replica1.sim b/tests/script/tsim/dnode/balance_replica1.sim
new file mode 100644
index 0000000000..14f3f130fb
--- /dev/null
+++ b/tests/script/tsim/dnode/balance_replica1.sim
@@ -0,0 +1,123 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+
+print =============== step1 create dnode2
+# no enough vnodes
+sql balance vgroup
+
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ---> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ---> $data00 $data01 $data02 $data03 $data04 $data05
+print ---> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != offline then
+  goto step1
+endi
+
+print =============== step2 create database
+sql create database d1 vgroups 2
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql create table d1.c2 using st tags(1)
+sql create table d1.c3 using st tags(1)
+sql create table d1.c4 using st tags(1)
+sql create table d1.c5 using st tags(1)
+sql create table d1.c6 using st tags(1)
+sql show d1.tables
+if $rows != 6 then
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 2 then
+  return -1
+endi
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(3)[3] != 2 then
+  return -1
+endi
+
+print =============== step3: balance vgroup
+# has offline dnode
+sql_error balance vgroup
+
+system sh/exec.sh -n dnode3 -s start
+$x = 0
+step3:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ---> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ---> $data00 $data01 $data02 $data03 $data04 $data05
+print ---> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step3
+endi
+if $data(2)[4] != ready then
+  goto step3
+endi
+if $data(3)[4] != ready then
+  goto step3
+endi
+
+print =============== step4: balance
+sql balance vgroup
+
+print show d1.vgroups
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 2 then
+  return -1
+endi
+if $data(2)[3] != 3 then
+  return -1
+endi
+if $data(3)[3] != 2 then
+  return -1
+endi
+
+print =============== step7: select data
+sql show d1.tables
+print rows $rows
+if $rows != 6 then
+  return -1
+endi
+
+return
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
index 91679a7e81..3ea351f7a7 100644
--- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
+++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
@@ -28,7 +28,7 @@ step1:
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -72,6 +72,13 @@ if $data(2)[7] != 4 then
   return -1
 endi
 
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+
+
+return
+
+
 print =============== step4: drop dnode 2
 system sh/exec.sh -n dnode5 -s start
 $x = 0

From eb231f0255df46657ba2b2a99fbc030a3f8d42e9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 22 Jun 2022 20:16:29 +0800
Subject: [PATCH 12/24] test: minor changes

---
 tests/script/jenkins/basic.txt                              | 6 +++---
 .../tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index dc2002d4d0..76b4f6b153 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -12,7 +12,7 @@
 ## ---- db
 ./test.sh -f tsim/db/create_all_options.sim
 ./test.sh -f tsim/db/alter_option.sim
-#./test.sh -f tsim/db/alter_replica_13.sim
+./test.sh -f tsim/db/alter_replica_13.sim
 #./test.sh -f tsim/db/alter_replica_31.sim
 ./test.sh -f tsim/db/basic1.sim
 ./test.sh -f tsim/db/basic2.sim
@@ -31,8 +31,8 @@
 #./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
-#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
-#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim
diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
index 3532332174..788a23991a 100644
--- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
@@ -176,6 +176,8 @@ if $rows != 1 then
   return -1
 endi
 
+return
+
 print =============== step33: move follower1
 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 
 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 

From 8a8d3f160b9afcbba755ecc2f07cb5b62528ab31 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 22 Jun 2022 20:26:44 +0800
Subject: [PATCH 13/24] feat: add multiple group tsdb reads to table merge scan

---
 source/libs/executor/inc/executorimpl.h |   4 +-
 source/libs/executor/src/executorimpl.c |   7 +-
 source/libs/executor/src/scanoperator.c | 131 +++++++++++++++---------
 3 files changed, 84 insertions(+), 58 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index b7325fd95d..eaf9a8439f 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -840,8 +840,8 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
 int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                   STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
                                   uint64_t taskId);
-SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 68f319c28e..c974d90fed 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4023,14 +4023,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
-
-      SArray* dataReaders = taosArrayInit(8, POINTER_BYTES);
       createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond);
-      doCreateMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId);
-
       extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
-
-      SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, dataReaders, pHandle, pTaskInfo);
+      SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
       return pOperator;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index ca10df88f3..432c70f34a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1908,8 +1908,13 @@ _error:
 }
 
 typedef struct STableMergeScanInfo {
-  SArray*     dataReaders;  // array of tsdbReaderT*
-  SReadHandle readHandle;
+  STableListInfo* tableListInfo;
+  int32_t         tableStartIndex;
+  int32_t         tableEndIndex;
+  bool            hasGroupId;
+
+  SArray*         dataReaders;  // array of tsdbReaderT*
+  SReadHandle     readHandle;
 
   int32_t  bufPageSize;
   uint32_t sortBufSize;  // max buffer size for in-memory sort
@@ -1920,11 +1925,9 @@ typedef struct STableMergeScanInfo {
   SSDataBlock* pSortInputBlock;
   int64_t      startTs;  // sort start time
 
-  bool          hasGroupId;
-  uint64_t      groupId;
-  STupleHandle* prefetchedTuple;
-
-  SArray* sortSourceParams;
+  SArray*  sortSourceParams;
+  uint64_t queryId;
+  uint64_t taskId;
 
   SFileBlockLoadRecorder readRecorder;
   int64_t                numOfRows;
@@ -1952,8 +1955,6 @@ typedef struct STableMergeScanInfo {
                        // window to check if current data block needs to be loaded.
 
   SSampleExecInfo sample;  // sample execution info
-  int32_t         curTWinIdx;
-
 } STableMergeScanInfo;
 
 int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
@@ -1963,8 +1964,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
 }
 
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
-                                SNode* pTagCond) {
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) {
   int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -1984,11 +1984,10 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
 }
 
 int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                  STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                  uint64_t taskId) {
-
+                                    STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
+                                    uint64_t taskId) {
   SQueryTableDataCond cond = {0};
-  int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
+  int32_t             code = initQueryTableDataCond(&cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -2011,6 +2010,24 @@ _error:
   return code;
 }
 
+int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
+                                  int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
+                                  uint64_t taskId) {
+  for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
+    STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
+    subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
+    taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
+
+    tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId);
+    taosArrayPush(arrayReader, &pReader);
+
+    taosArrayDestroy(subListInfo->pTableList);
+    taosMemoryFree(subListInfo);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -2190,22 +2207,35 @@ SArray* generateSortByTsInfo(int32_t order) {
   return pList;
 }
 
-int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) {
+int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
 
-  if (OPTR_IS_OPENED(pOperator)) {
-    return TSDB_CODE_SUCCESS;
-  }
+  int32_t tableStartIdx = pInfo->tableStartIndex;
+  int32_t tableEndIdx = pInfo->tableEndIndex;
 
+  STableListInfo* tableListInfo = pInfo->tableListInfo;
+  createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
+                            pInfo->dataReaders, pInfo->queryId, pInfo->taskId);
+
+  // todo the total available buffer should be determined by total capacity of buffer of this task.
+  // the additional one is reserved for merge result
+  pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1);
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
-
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
                                              pInfo->pSortInputBlock, pTaskInfo->id.str);
 
   tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL);
 
   size_t numReaders = taosArrayGetSize(pInfo->dataReaders);
+  for (int32_t i = 0; i < numReaders; ++i) {
+    STableMergeScanSortSourceParam param = {0};
+    param.readerIdx = i;
+    param.pOperator = pOperator;
+    param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
+    taosArrayPush(pInfo->sortSourceParams, &param);
+  }
+
   for (int32_t i = 0; i < numReaders; ++i) {
     SSortSource*                    ps = taosMemoryCalloc(1, sizeof(SSortSource));
     STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
@@ -2219,9 +2249,22 @@ int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, terrno);
   }
 
-  pOperator->status = OP_RES_TO_RETURN;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
+  STableMergeScanInfo* pInfo = pOperator->info;
+  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
+
+  tsortDestroySortHandle(pInfo->pSortHandle);
+  taosArrayClear(pInfo->sortSourceParams);
+
+  for (int32_t i = 0; i < taosArrayGetSize(pInfo->dataReaders); ++i) {
+    tsdbReaderT* reader = taosArrayGetP(pInfo->dataReaders, i);
+    tsdbCleanupReadHandle(reader);
+  }
+  taosArrayDestroy(pInfo->dataReaders);
 
-  OPTR_SET_OPENED(pOperator);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2264,12 +2307,18 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
   if (code != TSDB_CODE_SUCCESS) {
     longjmp(pTaskInfo->env, code);
   }
-
+  if (!pInfo->hasGroupId) {
+    pInfo->hasGroupId = true;
+    pInfo->tableStartIndex = 0;
+    pInfo->tableEndIndex = taosArrayGetSize(pInfo->tableListInfo->pTableList) - 1;
+    startGroupTableMergeScan(pOperator);
+  }
   SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
   } else {
+    stopGroupTableMergeScan(pOperator);
     doSetOperatorCompleted(pOperator);
   }
   return pBlock;
@@ -2279,17 +2328,11 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->cond);
 
-  for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) {
-    tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, i);
-    tsdbCleanupReadHandle(reader);
-  }
-  taosArrayDestroy(pTableScanInfo->dataReaders);
 
   if (pTableScanInfo->pColMatchInfo != NULL) {
     taosArrayDestroy(pTableScanInfo->pColMatchInfo);
   }
 
-  taosArrayDestroy(pTableScanInfo->sortSourceParams);
   pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
   pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
 
@@ -2315,8 +2358,9 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
+                                                uint64_t taskId) {
   STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
   SOperatorInfo*       pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -2346,22 +2390,16 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pInfo->sample.seed = taosGetTimestampSec();
   pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
   pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
-  pInfo->dataReaders = dataReaders;
+  pInfo->tableListInfo = pTableListInfo;
   pInfo->scanFlag = MAIN_SCAN;
   pInfo->pColMatchInfo = pColList;
-  pInfo->curTWinIdx = 0;
 
   pInfo->pResBlock = createResDataBlock(pDescNode);
+  pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
+  pInfo->queryId = queryId;
+  pInfo->taskId = taskId;
 
-  pInfo->sortSourceParams = taosArrayInit(taosArrayGetSize(dataReaders), sizeof(STableMergeScanSortSourceParam));
-  for (int32_t i = 0; i < taosArrayGetSize(dataReaders); ++i) {
-    STableMergeScanSortSourceParam* param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
-    param->readerIdx = i;
-    param->pOperator = pOperator;
-    param->inputBlock = createOneDataBlock(pInfo->pResBlock, false);
-    taosArrayPush(pInfo->sortSourceParams, param);
-    taosMemoryFree(param);
-  }
+  pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->cond.order);
   pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
@@ -2369,14 +2407,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   int32_t rowSize = pInfo->pResBlock->info.rowSize;
   pInfo->bufPageSize = getProperSortPageSize(rowSize);
 
-  // todo the total available buffer should be determined by total capacity of buffer of this task.
-  // the additional one is reserved for merge result
-  pInfo->sortBufSize = pInfo->bufPageSize * (taosArrayGetSize(dataReaders) + 1);
-  pInfo->hasGroupId = false;
-  pInfo->prefetchedTuple = NULL;
-
   pOperator->name = "TableMergeScanOperator";
-  // TODO : change it
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN;
   pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
@@ -2386,7 +2417,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   initResultSizeInfo(pOperator, 1024);
 
   pOperator->fpSet =
-      createOperatorFpSet(doOpenTableMergeScanOperator, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo,
+      createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo,
                           NULL, NULL, getTableMergeScanExplainExecInfo);
   pOperator->cost.openCost = 0;
   return pOperator;

From 14a488c92cde194bad6cbb71edef66cffd1ee14f Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 22 Jun 2022 21:21:39 +0800
Subject: [PATCH 14/24] feat(stream): stream support multiple type input

---
 include/libs/stream/tstream.h              |  9 +++------
 source/dnode/mnode/impl/src/mndScheduler.c | 22 ++++++---------------
 source/dnode/vnode/src/tq/tq.c             |  3 ++-
 source/dnode/vnode/src/tq/tqRead.c         |  4 ++--
 source/libs/executor/src/executor.c        |  7 ++-----
 source/libs/executor/src/scanoperator.c    | 13 ++++++------
 source/libs/stream/src/streamExec.c        | 23 +++++++++++-----------
 source/libs/stream/src/streamTask.c        | 16 +++++++--------
 8 files changed, 40 insertions(+), 57 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 684a605e18..7673e73035 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -239,17 +239,14 @@ typedef struct {
 struct SStreamTask {
   int64_t streamId;
   int32_t taskId;
-  int8_t  inputType;
-  int8_t  taskStatus;
-
-  int8_t execStatus;
-
+  int8_t  isDataScan;
   int8_t  execType;
   int8_t  sinkType;
   int8_t  dispatchType;
   int16_t dispatchMsgType;
 
-  int8_t isDataScan;
+  int8_t taskStatus;
+  int8_t execStatus;
 
   // node info
   int32_t selfChildId;
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index e055e3b2ba..b1729c4047 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -270,10 +270,8 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb
     pTask->nodeId = pVgroup->vgId;
     pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
-    pTask->isDataScan = 0;
-
     // source
-    pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
+    pTask->isDataScan = 0;
 
     // exec
     pTask->execType = TASK_EXEC__NONE;
@@ -320,10 +318,8 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
 #endif
   pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg);
 
-  pTask->isDataScan = 0;
-
   // source
-  pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
+  pTask->isDataScan = 0;
 
   // exec
   pTask->execType = TASK_EXEC__NONE;
@@ -392,12 +388,10 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       pInnerTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskInnerLevel, pInnerTask);
 
-      pInnerTask->isDataScan = 0;
-
       pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*));
 
-      // input
-      pInnerTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
+      // source
+      pInnerTask->isDataScan = 0;
 
       // trigger
       pInnerTask->triggerParam = pStream->triggerParam;
@@ -458,11 +452,9 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskSourceLevel, pTask);
 
+      // source
       pTask->isDataScan = 1;
 
-      // input
-      pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
-
       // add fixed vg dispatch
       pTask->sinkType = TASK_SINK__NONE;
       pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
@@ -517,10 +509,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskOneLevel, pTask);
 
-      pTask->isDataScan = 1;
-
       // input
-      pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
+      pTask->isDataScan = 1;
 
       // trigger
       pTask->triggerParam = pStream->triggerParam;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 691548af7a..046ee64878 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -426,6 +426,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
     ASSERT(0);
   }
   tDecoderClear(&decoder);
+  ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1);
 
   pTask->execStatus = TASK_EXEC_STATUS__IDLE;
 
@@ -505,7 +506,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
     if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
       continue;
     }
-    if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
+    if (!pTask->isDataScan) continue;
 
     if (!failed) {
       if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) {
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 6f44eb4569..f7558cc00b 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -345,8 +345,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   while (1) {
     pIter = taosHashIterate(pTq->pStreamTasks, pIter);
     if (pIter == NULL) break;
-    SStreamTask* pTask = (SStreamTask*)pIter;
-    if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->isDataScan) {
       int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
       ASSERT(code == 0);
     }
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index cf72dfd796..6b93119987 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -40,7 +40,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
     SStreamBlockScanInfo* pInfo = pOperator->info;
     pInfo->assignBlockUid = assignUid;
 
-    // the block type can not be changed in the streamscan operators
+    // no need to check
 #if 0
     if (pInfo->blockType == 0) {
       pInfo->blockType = type;
@@ -49,10 +49,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
       return TSDB_CODE_QRY_APP_ERROR;
     }
 #endif
-    // rollup sma, the same qTaskInfo is used to insert data by SubmitReq and fetch result by SSDataBlock
-    if (pInfo->blockType != type) { 
-      pInfo->blockType = type;
-    }
+    pInfo->blockType = type;
 
     if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
       if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 62167ec9ff..45f80c3c6f 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -537,7 +537,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   // taosSsleep(20);
 
   SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
-  int32_t numOfCols = 0;
+  int32_t             numOfCols = 0;
   SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
 
   int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
@@ -916,6 +916,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
   }
 
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
+  // TODO: refactor
   if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
     if (pInfo->validBlockIndex >= total) {
       /*doClearBufferedBlocks(pInfo);*/
@@ -1969,8 +1970,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
 }
 
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
-                                SNode* pTagCond) {
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) {
   int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -1990,11 +1990,10 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
 }
 
 int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                  STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                  uint64_t taskId) {
-
+                                    STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
+                                    uint64_t taskId) {
   SQueryTableDataCond cond = {0};
-  int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
+  int32_t             code = initQueryTableDataCond(&cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 4d9916e196..69cb5b43fa 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -24,13 +24,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
     SStreamTrigger* pTrigger = (SStreamTrigger*)data;
     qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
   } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
+    ASSERT(pTask->isDataScan);
     SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
-    ASSERT(pTask->inputType == STREAM_INPUT__DATA_SUBMIT);
     qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
   } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
     SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
-    ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK);
-    SArray* blocks = pBlock->blocks;
+    SArray*           blocks = pBlock->blocks;
     qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
   } else if (pItem->type == STREAM_INPUT__DROP) {
     // TODO exec drop
@@ -89,17 +88,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
         return NULL;
       }
 
-      if (((SStreamQueueItem*)data)->type == STREAM_INPUT__TRIGGER) {
+      int8_t type = ((SStreamQueueItem*)data)->type;
+      if (type == STREAM_INPUT__TRIGGER) {
         blockDataDestroy(((SStreamTrigger*)data)->pBlock);
         taosFreeQitem(data);
-      } else {
-        if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
-          streamDataSubmitRefDec((SStreamDataSubmit*)data);
-          taosFreeQitem(data);
-        } else {
-          taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
-          taosFreeQitem(data);
-        }
+      } else if (type == STREAM_INPUT__DATA_BLOCK) {
+        taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
+        taosFreeQitem(data);
+      } else if (type == STREAM_INPUT__DATA_SUBMIT) {
+        ASSERT(pTask->isDataScan);
+        streamDataSubmitRefDec((SStreamDataSubmit*)data);
+        taosFreeQitem(data);
       }
       streamQueueProcessSuccess(pTask->inputQueue);
       return taosArrayInit(0, sizeof(SSDataBlock));
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 8a7c425f79..1daa9bd50e 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -50,14 +50,14 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   /*if (tStartEncode(pEncoder) < 0) return -1;*/
   if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
   if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1;
+
+  if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1;
 
   if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
@@ -106,14 +106,14 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   /*if (tStartDecode(pDecoder) < 0) return -1;*/
   if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
   if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1;
+
+  if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1;
 
   if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;

From df130c1bcb918f379f2dd19f4fd8f9bd1c719585 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 22 Jun 2022 21:56:55 +0800
Subject: [PATCH 15/24] feat: add multiple group tsdb reads to table merge scan

---
 source/libs/executor/src/scanoperator.c | 50 ++++++++++++++++++-------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 432c70f34a..02c7c67260 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1912,9 +1912,10 @@ typedef struct STableMergeScanInfo {
   int32_t         tableStartIndex;
   int32_t         tableEndIndex;
   bool            hasGroupId;
+  uint64_t        groupId;
 
-  SArray*         dataReaders;  // array of tsdbReaderT*
-  SReadHandle     readHandle;
+  SArray*     dataReaders;  // array of tsdbReaderT*
+  SReadHandle readHandle;
 
   int32_t  bufPageSize;
   uint32_t sortBufSize;  // max buffer size for in-memory sort
@@ -2211,6 +2212,16 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
 
+  size_t  tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+  int32_t i = pInfo->tableStartIndex + 1;
+  for (; i < tableListSize; ++i) {
+    STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
+    if (tableKeyInfo->groupId != pInfo->groupId) {
+      break;
+    }
+  }
+  pInfo->tableEndIndex = i - 1;
+
   int32_t tableStartIdx = pInfo->tableStartIndex;
   int32_t tableEndIdx = pInfo->tableEndIndex;
 
@@ -2307,20 +2318,34 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
   if (code != TSDB_CODE_SUCCESS) {
     longjmp(pTaskInfo->env, code);
   }
+  size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
   if (!pInfo->hasGroupId) {
     pInfo->hasGroupId = true;
+
     pInfo->tableStartIndex = 0;
-    pInfo->tableEndIndex = taosArrayGetSize(pInfo->tableListInfo->pTableList) - 1;
+    pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
     startGroupTableMergeScan(pOperator);
   }
-  SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
-
-  if (pBlock != NULL) {
-    pOperator->resultInfo.totalRows += pBlock->info.rows;
-  } else {
-    stopGroupTableMergeScan(pOperator);
-    doSetOperatorCompleted(pOperator);
+  SSDataBlock* pBlock = NULL;
+  while (pInfo->tableStartIndex < tableListSize) {
+    pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator);
+    if (pBlock != NULL) {
+      pBlock->info.groupId = pInfo->groupId;
+      pOperator->resultInfo.totalRows += pBlock->info.rows;
+      return pBlock;
+    } else {
+      stopGroupTableMergeScan(pOperator);
+      if (pInfo->tableEndIndex >= tableListSize - 1) {
+        doSetOperatorCompleted(pOperator);
+        break;
+      }
+      pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
+      pInfo->groupId =
+          ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
+      startGroupTableMergeScan(pOperator);
+    }
   }
+
   return pBlock;
 }
 
@@ -2328,7 +2353,6 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->cond);
 
-
   if (pTableScanInfo->pColMatchInfo != NULL) {
     taosArrayDestroy(pTableScanInfo->pColMatchInfo);
   }
@@ -2417,8 +2441,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   initResultSizeInfo(pOperator, 1024);
 
   pOperator->fpSet =
-      createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo,
-                          NULL, NULL, getTableMergeScanExplainExecInfo);
+      createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo, NULL,
+                          NULL, getTableMergeScanExplainExecInfo);
   pOperator->cost.openCost = 0;
   return pOperator;
 

From c99f7cac793867298f8d2fa11e43e487ac47f73b Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 22 Jun 2022 22:00:52 +0800
Subject: [PATCH 16/24] feat: add multiple group tsdb reads to table merge scan

---
 source/libs/executor/src/scanoperator.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 02c7c67260..a416df4546 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2212,15 +2212,17 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
 
-  size_t  tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
-  int32_t i = pInfo->tableStartIndex + 1;
-  for (; i < tableListSize; ++i) {
-    STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
-    if (tableKeyInfo->groupId != pInfo->groupId) {
-      break;
+  {
+    size_t  tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
+    int32_t i = pInfo->tableStartIndex + 1;
+    for (; i < tableListSize; ++i) {
+      STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i);
+      if (tableKeyInfo->groupId != pInfo->groupId) {
+        break;
+      }
     }
+    pInfo->tableEndIndex = i - 1;
   }
-  pInfo->tableEndIndex = i - 1;
 
   int32_t tableStartIdx = pInfo->tableStartIndex;
   int32_t tableEndIdx = pInfo->tableEndIndex;

From fd79fcc481580fc0c664e3eb7eb379b22af6151c Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 22 Jun 2022 22:32:29 +0800
Subject: [PATCH 17/24] feat: add multiple group tsdb reads to table merge scan

---
 source/libs/executor/src/scanoperator.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index a416df4546..13a6a8855c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2323,7 +2323,11 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
   size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
   if (!pInfo->hasGroupId) {
     pInfo->hasGroupId = true;
-
+    
+    if (tableListSize == 0) {
+      doSetOperatorCompleted(pOperator);
+      return NULL;
+    }
     pInfo->tableStartIndex = 0;
     pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId;
     startGroupTableMergeScan(pOperator);

From faf5424b188156452dc466883dc284aaa998eb74 Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Wed, 22 Jun 2022 23:01:55 +0800
Subject: [PATCH 18/24] feat: add group sort operator

---
 include/libs/nodes/nodes.h              |  1 +
 include/libs/nodes/plannodes.h          |  2 ++
 source/libs/executor/inc/executorimpl.h |  5 ++---
 source/libs/executor/src/executorimpl.c |  2 ++
 source/libs/executor/src/scanoperator.c |  2 +-
 source/libs/executor/src/sortoperator.c | 30 ++++++++++---------------
 6 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h
index 58e2393970..f460c7d8ed 100644
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@@ -225,6 +225,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
   QUERY_NODE_PHYSICAL_PLAN_MERGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
+  QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
   QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index b07e8f39d5..b98c6bfb15 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -417,6 +417,8 @@ typedef struct SSortPhysiNode {
   SNodeList* pTargets;
 } SSortPhysiNode;
 
+typedef SSortPhysiNode SGroupSortPhysiNode;
+
 typedef struct SPartitionPhysiNode {
   SPhysiNode node;
   SNodeList* pExprs;  // these are expression list of partition_by_clause
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index eaf9a8439f..a809e3b089 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -837,9 +837,8 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
                                 SNode* pTagCond);
-int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                  STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                  uint64_t taskId);
+SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
+                                           SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
                                                 SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
 
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index c974d90fed..e62d239849 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4210,6 +4210,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
   } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) {
     pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo);
+  } else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) {
+    pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
     SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 13a6a8855c..922402a921 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2323,7 +2323,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
   size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList);
   if (!pInfo->hasGroupId) {
     pInfo->hasGroupId = true;
-    
+
     if (tableListSize == 0) {
       doSetOperatorCompleted(pOperator);
       return NULL;
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 0fabee5a1e..6086138301 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -424,10 +424,17 @@ int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
   return TSDB_CODE_SUCCESS;
 }
 
-// TODO:
-SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode,
+void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
+  SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
+  pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
+
+  taosArrayDestroy(pInfo->pSortInfo);
+  taosArrayDestroy(pInfo->pColMatchInfo);
+}
+
+SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
                                            SExecTaskInfo* pTaskInfo) {
-  SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
+  SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) {
     goto _error;
@@ -452,8 +459,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysi
   ;
   pInfo->pColMatchInfo = pColMatchColInfo;
   pOperator->name = "GroupSortOperator";
-  // TODO
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
   pOperator->blocking = true;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
@@ -461,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SSortPhysi
   pOperator->exprSupp.numOfExprs = numOfCols;
   pOperator->pTaskInfo = pTaskInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyOrderOperatorInfo, NULL,
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL,
                                          NULL, getGroupSortExplainExecInfo);
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
@@ -478,18 +484,6 @@ _error:
   return NULL;
 }
 
-void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
-  SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
-  pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
-
-  taosArrayDestroy(pInfo->pSortInfo);
-  taosArrayDestroy(pInfo->pColMatchInfo);
-}
-
-// TODO: sort group
-// TODO: msortCompare compare group id in multiway merge sort.
-// TODO: table merge scan, group first, then for each group, multiple readers
-
 //=====================================================================================
 // Multiway Sort Merge operator
 typedef struct SMultiwaySortMergeOperatorInfo {

From 849cb8d1b357b9edd871f08b488ee32f19d6a679 Mon Sep 17 00:00:00 2001
From: slzhou <slzhou@taosdata.com>
Date: Thu, 23 Jun 2022 08:52:47 +0800
Subject: [PATCH 19/24] feat: split session and state window on stable

---
 source/libs/executor/src/timewindowoperator.c |  3 +-
 source/libs/parser/src/parTranslater.c        |  3 -
 source/libs/planner/src/planSpliter.c         | 75 ++++++++++++++++++-
 source/libs/planner/test/planSessionTest.cpp  | 10 +++
 source/libs/planner/test/planStateTest.cpp    |  9 +++
 5 files changed, 91 insertions(+), 9 deletions(-)

diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 3ff45d7237..19f0fd4ea7 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1656,10 +1656,9 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
-      return NULL;
     }
 
-    return pBInfo->pRes;
+    return pBInfo->pRes->info.rows > 0 ? pBInfo->pRes : NULL;
   }
 
   int64_t st = taosGetTimestampUs();
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 43a4cf0ed6..e02cb931e9 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2161,9 +2161,6 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) {
     if (COLUMN_TYPE_TAG == pCol->colType) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL);
     }
-    if (TSDB_SUPER_TABLE == pCol->tableType) {
-      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE);
-    }
   }
   return DEAL_RES_CONTINUE;
 }
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index f0e0e84bd9..f4c1abb354 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -176,10 +176,22 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
       return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_WINDOW: {
       SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
-      if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType)) {
-        return false;
+      if (WINDOW_TYPE_INTERVAL == pWindow->winType) {
+        return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
+      if (WINDOW_TYPE_STATE == pWindow->winType) {
+        if (!streamQuery) {
+          return stbSplHasMultiTbScan(streamQuery, pNode);
+        } else {
+          return false;
+        }
+      }
+      if (WINDOW_TYPE_SESSION == pWindow->winType) {
+        if (!streamQuery) {
+          return stbSplHasMultiTbScan(streamQuery, pNode);
+        } else {
+          return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
+        }
       }
-      return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
     }
     case QUERY_NODE_LOGIC_PLAN_SORT:
       return stbSplHasMultiTbScan(streamQuery, pNode);
@@ -477,11 +489,64 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
   return code;
 }
 
+static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) {
+  if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
+    ((SScanLogicNode*)pNode)->scanType = scanType;
+  } else {
+    if (1 == LIST_LENGTH(pNode->pChildren)) {
+      splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType);
+    }
+  }
+}
+
+static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  SLogicNode* pWindow = pInfo->pSplitNode;
+  SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0);
+
+  SNodeList* pMergeKeys = NULL;
+  int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys);
+
+  if (TSDB_CODE_SUCCESS == code) {
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
+                                     (SNode*)splCreateScanSubplan(pCxt, pChild, SPLIT_FLAG_STABLE_SPLIT));
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE);
+    ++(pCxt->groupId);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
+    SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
+  } else {
+    nodesDestroyList(pMergeKeys);
+  }
+
+  return code;
+}
+
 static int32_t stbSplSplitSession(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   if (pCxt->pPlanCxt->streamQuery) {
     return stbSplSplitSessionForStream(pCxt, pInfo);
   } else {
-    return TSDB_CODE_PLAN_INTERNAL_ERROR;
+    return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
+  }
+}
+
+static int32_t stbSplSplitStateForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  return TSDB_CODE_PLAN_INTERNAL_ERROR;
+}
+
+static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  if (pCxt->pPlanCxt->streamQuery) {
+    return stbSplSplitStateForStream(pCxt, pInfo);
+  } else {
+    return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
   }
 }
 
@@ -511,6 +576,8 @@ static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitI
       return stbSplSplitInterval(pCxt, pInfo);
     case WINDOW_TYPE_SESSION:
       return stbSplSplitSession(pCxt, pInfo);
+    case WINDOW_TYPE_STATE:
+      return stbSplSplitState(pCxt, pInfo);
     default:
       break;
   }
diff --git a/source/libs/planner/test/planSessionTest.cpp b/source/libs/planner/test/planSessionTest.cpp
index 7d5d826925..f445bb5ffc 100644
--- a/source/libs/planner/test/planSessionTest.cpp
+++ b/source/libs/planner/test/planSessionTest.cpp
@@ -34,3 +34,13 @@ TEST_F(PlanSessionTest, selectFunc) {
   // select function along with the columns of select row, and with SESSION clause
   run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)");
 }
+
+TEST_F(PlanSessionTest, stable) {
+  useDb("root", "test");
+
+  // select function for SESSION clause
+  run("SELECT MAX(c1), MIN(c1) FROM st1 SESSION(ts, 10s)");
+  // select function along with the columns of select row, and with SESSION clause
+  run("SELECT MAX(c1), c2 FROM st1 SESSION(ts, 10s)");
+  run("SELECT count(ts) FROM st1 PARTITION BY c1 SESSION(ts, 10s)");
+}
diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp
index 9ff035e148..6985bc8807 100644
--- a/source/libs/planner/test/planStateTest.cpp
+++ b/source/libs/planner/test/planStateTest.cpp
@@ -40,3 +40,12 @@ TEST_F(PlanStateTest, selectFunc) {
   // select function along with the columns of select row, and with STATE_WINDOW clause
   run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)");
 }
+
+TEST_F(PlanStateTest, stable) {
+  useDb("root", "test");
+
+  // select function for STATE_WINDOW clause
+  run("SELECT MAX(c1), MIN(c1) FROM st1 STATE_WINDOW(c2)");
+  // select function along with the columns of select row, and with STATE_WINDOW clause
+  run("SELECT MAX(c1), c2 FROM st1 STATE_WINDOW(c2)");
+}

From 6fe368a37531b025cf4442a76dd8bf4c34ad8eb5 Mon Sep 17 00:00:00 2001
From: slzhou <slzhou@taosdata.com>
Date: Thu, 23 Jun 2022 09:01:03 +0800
Subject: [PATCH 20/24] feat: add session and state window on stable

---
 source/libs/planner/src/planSpliter.c | 46 ++++++++++++++++-----------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index f4c1abb354..9847755a7a 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -166,6 +166,31 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) {
   return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild));
 }
 
+static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
+  SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
+  if (WINDOW_TYPE_INTERVAL == pWindow->winType) {
+    return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
+  }
+
+  if (WINDOW_TYPE_SESSION == pWindow->winType) {
+    if (!streamQuery) {
+      return stbSplHasMultiTbScan(streamQuery, pNode);
+    } else {
+      return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
+    }
+  }
+  
+  if (WINDOW_TYPE_STATE == pWindow->winType) {
+    if (!streamQuery) {
+      return stbSplHasMultiTbScan(streamQuery, pNode);
+    } else {
+      return false;
+    }
+  }
+
+  return false;
+}
+
 static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_SCAN:
@@ -174,25 +199,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
       return !(((SJoinLogicNode*)pNode)->isSingleTableJoin);
     case QUERY_NODE_LOGIC_PLAN_AGG:
       return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
-    case QUERY_NODE_LOGIC_PLAN_WINDOW: {
-      SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
-      if (WINDOW_TYPE_INTERVAL == pWindow->winType) {
-        return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
-      if (WINDOW_TYPE_STATE == pWindow->winType) {
-        if (!streamQuery) {
-          return stbSplHasMultiTbScan(streamQuery, pNode);
-        } else {
-          return false;
-        }
-      }
-      if (WINDOW_TYPE_SESSION == pWindow->winType) {
-        if (!streamQuery) {
-          return stbSplHasMultiTbScan(streamQuery, pNode);
-        } else {
-          return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
-        }
-      }
-    }
+    case QUERY_NODE_LOGIC_PLAN_WINDOW:
+      return stbSplNeedSplitWindow(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_SORT:
       return stbSplHasMultiTbScan(streamQuery, pNode);
     default:

From fed3b36b0ea3cf2aceecb4b27d484bc8d8d3f252 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Thu, 23 Jun 2022 09:40:35 +0800
Subject: [PATCH 21/24] shell: fix shell chinese error

---
 tools/shell/src/shellCommand.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c
index ddd4f7a59d..f236c1eb88 100644
--- a/tools/shell/src/shellCommand.c
+++ b/tools/shell/src/shellCommand.c
@@ -416,6 +416,9 @@ char taosGetConsoleChar() {
     bufIndex = 0;
   }
   if (mbStrLen == 0){
+    if (buf[bufIndex] == '\r') {
+      bufIndex++;
+    }
     mbStrLen = WideCharToMultiByte(CP_UTF8, 0, &buf[bufIndex], 1, mbStr, sizeof(mbStr), NULL, NULL);
     mbStrIndex = 0;
     bufIndex++;
@@ -485,7 +488,6 @@ int32_t shellReadCommand(char *command) {
           shellBackspaceChar(&cmd);
           break;
         case '\n':
-          break;
         case '\r':
         #ifdef WINDOWS 
         #else

From ce7e7fe23d64fb75ab1d885e759a71c0e07be02c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Jun 2022 10:13:35 +0800
Subject: [PATCH 22/24] refactor:remove some debug info.

---
 source/libs/executor/src/executor.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index e805de3727..cf72dfd796 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -68,8 +68,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
 
         taosArrayClear(p->pDataBlock);
         taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
-        printf("size------------->%ld,  %ld, %p,  \n", taosArrayGetSize(pDataBlock->pDataBlock), taosArrayGetSize(pDataBlock->pDataBlock) * pDataBlock->pDataBlock->elemSize,
-            pDataBlock->pDataBlock);
         taosArrayPush(pInfo->pBlockLists, &p);
       }
     } else {

From 4576b0a94d7d2a30224d893b5fbb2fd2d18bb28e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Jun 2022 11:07:14 +0800
Subject: [PATCH 23/24] fix(query): fix syntax error.

---
 source/libs/executor/inc/executorimpl.h | 4 ++--
 source/libs/executor/src/executorimpl.c | 2 +-
 source/libs/executor/src/scanoperator.c | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 9bcfae549d..ccdbe5c890 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -838,8 +838,8 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
                                        SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
-                                SNode* pTagCond);
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
+
 SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
                                            SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index ad5ebb5974..cd3b8cb705 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4009,7 +4009,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
-      createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond);
+      createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
       extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
       SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
       STableScanInfo* pScanInfo = pOperator->info;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index d1d0a70115..2e3ae851bf 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1956,7 +1956,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
 }
 
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) {
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
   int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return code;

From 91eb4ff88b51f4ed88ae5031df52512676f54efb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 Jun 2022 11:32:21 +0800
Subject: [PATCH 24/24] test(query): update test case.

---
 tests/system-test/2-query/json_tag.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py
index ff7adfc759..6cd3f49636 100644
--- a/tests/system-test/2-query/json_tag.py
+++ b/tests/system-test/2-query/json_tag.py
@@ -509,7 +509,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 5.3)
         # tdSql.query("select twa(dataint) from jsons1 where jtag is not null")
         # tdSql.checkData(0, 0, 36)
-        tdSql.error("select irate(dataint) from jsons1 where jtag is not null")
+        tdSql.query("select irate(dataint) from jsons1 where jtag is not null")
         tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null")
         tdSql.checkData(0, 0, 45)
         tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1")