other:merge 3.0

source/client/src/clientHb.c

@@ -173,7 +173,8 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
       pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes;
       pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes;
       pTscObj->connId = pRsp->query->connId;
-      tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes);
+      tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes,
+               pTscObj->pAppInfo->totalDnodes);
 
       if (pRsp->query->killRid) {
         tscDebug("request rid %" PRIx64 " need to be killed now", pRsp->query->killRid);
@@ -297,7 +298,8 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
 
   if (code != 0) {
     (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1);
-    tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes);
+    tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes,
+             (*pInst)->totalDnodes);
   }
 
   if (rspNum) {
@@ -414,6 +416,9 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) {
   int32_t code = hbBuildQueryDesc(hbBasic, pTscObj);
   if (code) {
     releaseTscObj(connKey->tscRid);
+    if (hbBasic->queryDesc) {
+      taosArrayDestroyEx(hbBasic->queryDesc, tFreeClientHbQueryDesc);
+    }
     taosMemoryFree(hbBasic);
     return code;
   }
@@ -654,6 +659,8 @@ int32_t hbGatherAppInfo(void) {
 
   for (int32_t i = 0; i < sz; ++i) {
     SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
+    if (pAppHbMgr == NULL) continue;
+
     uint64_t   clusterId = pAppHbMgr->pAppInstInfo->clusterId;
     SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
     if (NULL == pApp) {
@@ -691,15 +698,21 @@ static void *hbThreadFunc(void *param) {
       hbGatherAppInfo();
     }
 
+    SArray *mgr = taosArrayInit(sz, sizeof(void *));
     for (int i = 0; i < sz; i++) {
       SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
+      if (pAppHbMgr == NULL) {
+        continue;
+      }
 
       int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
       if (connCnt == 0) {
+        taosArrayPush(mgr, &pAppHbMgr);
         continue;
       }
       SClientHbBatchReq *pReq = hbGatherAllInfo(pAppHbMgr);
-      if (pReq == NULL) {
+      if (pReq == NULL || taosArrayGetP(clientHbMgr.appHbMgrs, i) == NULL) {
+        tFreeClientHbBatchReq(pReq);
         continue;
       }
       int   tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq);
@@ -708,6 +721,7 @@ static void *hbThreadFunc(void *param) {
         terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
         tFreeClientHbBatchReq(pReq);
         // hbClearReqInfo(pAppHbMgr);
+        taosArrayPush(mgr, &pAppHbMgr);
         break;
       }
 
@@ -719,6 +733,7 @@ static void *hbThreadFunc(void *param) {
         tFreeClientHbBatchReq(pReq);
         // hbClearReqInfo(pAppHbMgr);
         taosMemoryFree(buf);
+        taosArrayPush(mgr, &pAppHbMgr);
         break;
       }
       pInfo->fp = hbAsyncCallBack;
@@ -738,8 +753,12 @@ static void *hbThreadFunc(void *param) {
       // hbClearReqInfo(pAppHbMgr);
 
       atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
+      taosArrayPush(mgr, &pAppHbMgr);
     }
 
+    taosArrayDestroy(clientHbMgr.appHbMgrs);
+    clientHbMgr.appHbMgrs = mgr;
+
     taosThreadMutexUnlock(&clientHbMgr.lock);
 
     taosMsleep(HEARTBEAT_INTERVAL);
@@ -831,7 +850,7 @@ void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr) {
     if (pItem == *pAppHbMgr) {
       hbFreeAppHbMgr(*pAppHbMgr);
       *pAppHbMgr = NULL;
-      taosArrayRemove(clientHbMgr.appHbMgrs, i);
+      taosArraySet(clientHbMgr.appHbMgrs, i, pAppHbMgr);
       break;
     }
   }
@@ -842,6 +861,7 @@ void appHbMgrCleanup(void) {
   int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
   for (int i = 0; i < sz; i++) {
     SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
+    if (pTarget == NULL) continue;
     hbFreeAppHbMgr(pTarget);
   }
 }
@@ -856,7 +876,14 @@ int hbMgrInit() {
 
   clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *));
-  taosThreadMutexInit(&clientHbMgr.lock, NULL);
+
+  TdThreadMutexAttr attr = {0};
+  taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE);
+  int ret = taosThreadMutexAttrInit(&attr);
+  assert(ret == 0);
+
+  taosThreadMutexInit(&clientHbMgr.lock, &attr);
+  taosThreadMutexAttrDestroy(&attr);
 
   // init handle funcs
   hbMgrInitHandle();

source/client/src/clientImpl.c

@@ -438,6 +438,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   }
   pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
   pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
+  ASSERT(numOfCols == pResInfo->numOfCols);
 
   for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
     pResInfo->fields[i].bytes = pSchema[i].bytes;
@@ -854,6 +855,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
   pRequest->metric.resultReady = taosGetTimestampUs();
 
   if (pResult) {
+    destroyQueryExecRes(&pRequest->body.resInfo.execRes);
     memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
   }
 
@@ -1390,6 +1392,7 @@ int32_t doProcessMsgFromServer(void* param) {
   pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
   rpcFreeCont(pMsg->pCont);
   destroySendMsgInfo(pSendInfo);
+
   taosMemoryFree(arg);
   return TSDB_CODE_SUCCESS;
 }

source/client/src/clientMain.c

@@ -870,11 +870,13 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
 
   if (code != TSDB_CODE_SUCCESS) {
     pRequest->code = code;
+    taosMemoryFreeClear(pResultInfo->pData);
     pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
     return;
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
+    taosMemoryFreeClear(pResultInfo->pData);
     pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
     return;
   }

source/client/src/clientMsgHandler.c

@@ -34,6 +34,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
     removeMeta(pRequest->pTscObj, pRequest->targetTableList);
   }
 
+  taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);
@@ -46,6 +47,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
 int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   if (code != TSDB_CODE_SUCCESS) {
+    taosMemoryFree(pMsg->pEpSet);
     taosMemoryFree(pMsg->pData);
     setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
@@ -62,6 +64,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   if (delta > timestampDeltaLimit) {
     code = TSDB_CODE_TIME_UNSYNCED;
     tscError("time diff:%ds is too big", delta);
+    taosMemoryFree(pMsg->pEpSet);
     taosMemoryFree(pMsg->pData);
     setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
@@ -70,6 +73,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   /*assert(connectRsp.epSet.numOfEps > 0);*/
   if (connectRsp.epSet.numOfEps == 0) {
+    taosMemoryFree(pMsg->pEpSet);
     taosMemoryFree(pMsg->pData);
     setErrno(pRequest, TSDB_CODE_MND_APP_ERROR);
     tsem_post(&pRequest->body.rspSem);
@@ -114,6 +118,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
            pTscObj->pAppInfo->numOfConns);
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
   tsem_post(&pRequest->body.rspSem);
   return 0;
 }
@@ -137,6 +142,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   // todo rsp with the vnode id list
   SRequestObj* pRequest = param;
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
   }
@@ -173,6 +179,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   if (code != TSDB_CODE_SUCCESS) {
     taosMemoryFree(pMsg->pData);
+    taosMemoryFree(pMsg->pEpSet);
     setErrno(pRequest, code);
 
     if (pRequest->body.queryFp != NULL) {
@@ -220,6 +227,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   setConnectionDB(pRequest->pTscObj, db);
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
@@ -246,6 +254,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
     pRequest->body.resInfo.execRes.res = createRsp.pMeta;
   }
 
+  taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
 
   if (pRequest->body.queryFp != NULL) {
@@ -284,6 +293,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);
@@ -309,6 +319,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
     SExecResult* pRes = &pRequest->body.resInfo.execRes;
@@ -420,6 +431,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);

source/client/src/clientTmq.c

@@ -841,7 +841,7 @@ void tmqFreeImpl(void* handle) {
   int32_t sz = taosArrayGetSize(tmq->clientTopics);
   for (int32_t i = 0; i < sz; i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    if (pTopic->schema.nCols) taosMemoryFree(pTopic->schema.pSchema);
+    if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
     int32_t vgSz = taosArrayGetSize(pTopic->vgs);
     taosArrayDestroy(pTopic->vgs);
   }
@@ -1077,6 +1077,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
     taosMemoryFree(pMsg->pData);
+    taosMemoryFree(pMsg->pEpSet);
     terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
     return -1;
   }
@@ -1115,6 +1116,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
             tmqEpoch);
     tsem_post(&tmq->rspSem);
     taosMemoryFree(pMsg->pData);
+    taosMemoryFree(pMsg->pEpSet);
     return 0;
   }
 
@@ -1128,6 +1130,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
   if (pRspWrapper == NULL) {
     taosMemoryFree(pMsg->pData);
+    taosMemoryFree(pMsg->pEpSet);
     tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch);
     goto CREATE_MSG_FAIL;
   }
@@ -1164,6 +1167,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
   tsem_post(&tmq->rspSem);
@@ -1218,6 +1222,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
     SMqClientTopic topic = {0};
     SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i);
     topic.schema = pTopicEp->schema;
+    pTopicEp->schema.nCols = 0;
+    pTopicEp->schema.pSchema = NULL;
     tstrncpy(topic.topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN);
     tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN);
 
@@ -1251,7 +1257,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
     int32_t sz = taosArrayGetSize(tmq->clientTopics);
     for (int32_t i = 0; i < sz; i++) {
       SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-      if (pTopic->schema.nCols) taosMemoryFree(pTopic->schema.pSchema);
+      if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
       int32_t vgSz = taosArrayGetSize(pTopic->vgs);
       taosArrayDestroy(pTopic->vgs);
     }

source/common/src/tdataformat.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "tdataformat.h"
+#include "tRealloc.h"
 #include "tcoding.h"
 #include "tdatablock.h"
 #include "tlog.h"
@@ -682,7 +683,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) {
   return n;
 }
 
-// STSchema
+// STSchema ========================================
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -722,9 +723,7 @@ void tTSchemaDestroy(STSchema *pTSchema) {
   if (pTSchema) taosMemoryFree(pTSchema);
 }
 
-// STSRowBuilder
+// STag ========================================
-
-// STag
 static int tTagValCmprFn(const void *p1, const void *p2) {
   if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) {
     return -1;
@@ -1175,3 +1174,494 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
 }
 
 #endif
+
+// SColData ========================================
+void tColDataDestroy(void *ph) {
+  SColData *pColData = (SColData *)ph;
+
+  tFree(pColData->pBitMap);
+  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->pData);
+}
+
+void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
+  pColData->cid = cid;
+  pColData->type = type;
+  pColData->smaOn = smaOn;
+  tColDataClear(pColData);
+}
+
+void tColDataClear(SColData *pColData) {
+  pColData->nVal = 0;
+  pColData->flag = 0;
+  pColData->nData = 0;
+}
+
+static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) {
+  int32_t code = 0;
+
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1));
+    if (code) goto _exit;
+    pColData->aOffset[pColData->nVal] = pColData->nData;
+
+    if (pColVal->value.nData) {
+      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
+      if (code) goto _exit;
+      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
+      pColData->nData += pColVal->value.nData;
+    }
+  } else {
+    ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal);
+    code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
+    if (code) goto _exit;
+    pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type);
+  }
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) {  // 0
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    pColData->flag = HAS_NONE;
+  } else if (pColVal->isNull) {
+    pColData->flag = HAS_NULL;
+  } else {
+    pColData->flag = HAS_VALUE;
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) {  // HAS_NONE
+  int32_t code = 0;
+
+  if (!pColVal->isNone) {
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    memset(pColData->pBitMap, 0, nBit);
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+    if (pColVal->isNull) {
+      pColData->flag |= HAS_NULL;
+    } else {
+      pColData->flag |= HAS_VALUE;
+
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) {  // HAS_NULL
+  int32_t code = 0;
+
+  if (!pColVal->isNull) {
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+
+      memset(pColData->pBitMap, 255, nBit);
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      pColData->flag |= HAS_VALUE;
+
+      memset(pColData->pBitMap, 0, nBit);
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+  } else if (pColVal->isNull) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+  } else {
+    pColData->flag |= HAS_VALUE;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 2);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+
+    if (pColData->nVal) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+        code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+        if (code) goto _exit;
+        memset(pColData->aOffset, 0, nOffset);
+      } else {
+        pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+        code = tRealloc(&pColData->pData, pColData->nData);
+        if (code) goto _exit;
+        memset(pColData->pData, 0, pColData->nData);
+      }
+    }
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE
+  int32_t code = 0;
+
+  if (pColVal->isNone || pColVal->isNull) {
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+    } else {
+      pColData->flag |= HAS_NULL;
+    }
+
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    memset(pColData->pBitMap, 255, nBit);
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  } else {
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 1);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+  } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+    }
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    pColData->flag |= HAS_NONE;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 0);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+  } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNull) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+    }
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
+                                                 SColVal  *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
+  int32_t code = 0;
+
+  code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
+  if (code) goto _exit;
+
+  if (pColVal->isNone) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
+  } else if (pColVal->isNull) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
+  } else {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = {
+    tColDataAppendValue0,  // 0
+    tColDataAppendValue1,  // HAS_NONE
+    tColDataAppendValue2,  // HAS_NULL
+    tColDataAppendValue3,  // HAS_NULL|HAS_NONE
+    tColDataAppendValue4,  // HAS_VALUE
+    tColDataAppendValue5,  // HAS_VALUE|HAS_NONE
+    tColDataAppendValue6,  // HAS_VALUE|HAS_NULL
+    tColDataAppendValue7   // HAS_VALUE|HAS_NULL|HAS_NONE
+};
+int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
+  ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
+  return tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
+}
+
+static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NONE
+  *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL
+  *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_VALUE
+  SValue value;
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    if (iVal + 1 < pColData->nVal) {
+      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
+    } else {
+      value.nData = pColData->nData - pColData->aOffset[iVal];
+    }
+    value.pData = pColData->pData + pColData->aOffset[iVal];
+  } else {
+    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
+  }
+  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
+}
+static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
+  switch (GET_BIT2(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 2:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = {
+    NULL,               // 0
+    tColDataGetValue1,  // HAS_NONE
+    tColDataGetValue2,  // HAS_NULL
+    tColDataGetValue3,  // HAS_NULL | HAS_NONE
+    tColDataGetValue4,  // HAS_VALUE
+    tColDataGetValue5,  // HAS_VALUE | HAS_NONE
+    tColDataGetValue6,  // HAS_VALUE | HAS_NULL
+    tColDataGetValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
+};
+void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
+  ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag);
+  tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
+}
+
+uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) {
+  uint8_t v;
+  switch (pColData->flag) {
+    case HAS_NONE:
+      v = 0;
+      break;
+    case HAS_NULL:
+      v = 1;
+      break;
+    case (HAS_NULL | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      break;
+    case HAS_VALUE:
+      v = 2;
+      break;
+    case (HAS_VALUE | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      if (v) v = 2;
+      break;
+    case (HAS_VALUE | HAS_NULL):
+      v = GET_BIT1(pColData->pBitMap, iVal) + 1;
+      break;
+    case (HAS_VALUE | HAS_NULL | HAS_NONE):
+      v = GET_BIT2(pColData->pBitMap, iVal);
+      break;
+    default:
+      ASSERT(0);
+      break;
+  }
+  return v;
+}
+
+int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
+  int32_t code = 0;
+  int32_t size;
+
+  ASSERT(pColDataSrc->nVal > 0);
+  ASSERT(pColDataDest->cid = pColDataSrc->cid);
+  ASSERT(pColDataDest->type = pColDataSrc->type);
+
+  pColDataDest->smaOn = pColDataSrc->smaOn;
+  pColDataDest->nVal = pColDataSrc->nVal;
+  pColDataDest->flag = pColDataSrc->flag;
+
+  // bitmap
+  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
+    size = BIT2_SIZE(pColDataSrc->nVal);
+    code = tRealloc(&pColDataDest->pBitMap, size);
+    if (code) goto _exit;
+    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
+  }
+
+  // offset
+  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
+    size = sizeof(int32_t) * pColDataSrc->nVal;
+
+    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
+    if (code) goto _exit;
+
+    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
+  }
+
+  // value
+  pColDataDest->nData = pColDataSrc->nData;
+  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
+  if (code) goto _exit;
+  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
+
+_exit:
+  return code;
+}

source/common/src/tglobal.c

@@ -63,7 +63,7 @@ int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
-int32_t tsNumOfQnodeFetchThreads = 4;
+int32_t tsNumOfQnodeFetchThreads = 1;
 int32_t tsNumOfSnodeSharedThreads = 2;
 int32_t tsNumOfSnodeUniqueThreads = 2;
 
@@ -385,9 +385,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4);
   if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;
 
-  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
+//  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
-  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
+//  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
-  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
+//  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
   tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
@@ -527,6 +527,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->stype = stype;
   }
 
+/*
   pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
     tsNumOfQnodeFetchThreads = numOfCores / 2;
@@ -534,6 +535,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->i32 = tsNumOfQnodeFetchThreads;
     pItem->stype = stype;
   }
+*/
 
   pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
@@ -691,7 +693,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
   tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
   tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
-  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
+//  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
   tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
   tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
   tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
@@ -939,8 +941,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
         tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
       } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {
         tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
+/*        
       } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) {
         tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
+*/
       } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) {
         tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
       } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) {

source/common/src/tmsg.c

@@ -3347,7 +3347,13 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) {
   return 0;
 }
 
-void tFreeSTableMetaRsp(void *pRsp) { taosMemoryFreeClear(((STableMetaRsp *)pRsp)->pSchemas); }
+void tFreeSTableMetaRsp(void *pRsp) { 
+  if (NULL == pRsp) {
+    return;
+  }
+  
+  taosMemoryFreeClear(((STableMetaRsp *)pRsp)->pSchemas); 
+}
 
 void tFreeSTableIndexRsp(void *info) {
   if (NULL == info) {
@@ -5439,6 +5445,8 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
     for (int32_t i = 0; i < pRsp->nBlocks; ++i) {
       SSubmitBlkRsp *sRsp = pRsp->pBlocks + i;
       taosMemoryFree(sRsp->tblFName);
+      tFreeSTableMetaRsp(sRsp->pMeta);
+      taosMemoryFree(sRsp->pMeta);
     }
 
     taosMemoryFree(pRsp->pBlocks);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -270,7 +270,7 @@ int32_t dmInitClient(SDnode *pDnode) {
 
   SRpcInit rpcInit = {0};
   rpcInit.label = "DND-C";
-  rpcInit.numOfThreads = 1;
+  rpcInit.numOfThreads = 4;
   rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;

source/dnode/mnode/impl/src/mndConsumer.c

@@ -487,6 +487,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    taosArrayDestroy(pConsumerNew->rebNewTopics);
     pConsumerNew->rebNewTopics = newSub;
     subscribe.topicNames = NULL;
 

source/dnode/mnode/impl/src/mndDef.c

@@ -145,7 +145,10 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
 }
 
 void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
-  if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
+  if (pVgEp) {
+    taosMemoryFreeClear(pVgEp->qmsg);
+    taosMemoryFree(pVgEp);
+  }
 }
 
 int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
@@ -200,19 +203,11 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L
 }
 
 void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
-  if (pConsumer->currentTopics) {
   taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
-  }
-  if (pConsumer->rebNewTopics) {
   taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
-  }
-  if (pConsumer->rebRemovedTopics) {
   taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
-  }
-  if (pConsumer->assignedTopics) {
   taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
 }
-}
 
 int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
   int32_t tlen = 0;
@@ -428,6 +423,13 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
 }
 
 void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
+  void *pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+    taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
+  }
   taosHashCleanup(pSub->consumerHash);
   taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp);
 }

source/dnode/mnode/impl/src/mndStb.c

@@ -1187,6 +1187,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName,
       if (pCol->colId > 0 && pCol->colId == colId) {
         sdbRelease(pSdb, pTopic);
         nodesDestroyNode(pAst);
+        nodesDestroyList(pNodeList);
         terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC;
         mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId);
         return -1;
@@ -1197,6 +1198,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName,
   NEXT:
     sdbRelease(pSdb, pTopic);
     nodesDestroyNode(pAst);
+    nodesDestroyList(pNodeList);
   }
   return 0;
 }
@@ -1228,6 +1230,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName
       if (pCol->colId > 0 && pCol->colId == colId) {
         sdbRelease(pSdb, pStream);
         nodesDestroyNode(pAst);
+        nodesDestroyList(pNodeList);
         terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED;
         mError("stream:%s, check colId:%d conflicted", pStream->name, pCol->colId);
         return -1;
@@ -1238,6 +1241,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName
   NEXT:
     sdbRelease(pSdb, pStream);
     nodesDestroyNode(pAst);
+    nodesDestroyList(pNodeList);
   }
   return 0;
 }
@@ -1275,6 +1279,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName,
       if ((pCol->colId) > 0 && (pCol->colId == colId)) {
         sdbRelease(pSdb, pSma);
         nodesDestroyNode(pAst);
+        nodesDestroyList(pNodeList);
         terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA;
         mError("tsma:%s, check colId:%d conflicted", pSma->name, pCol->colId);
         return -1;
@@ -1285,6 +1290,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName,
   NEXT:
     sdbRelease(pSdb, pSma);
     nodesDestroyNode(pAst);
+    nodesDestroyList(pNodeList);
   }
   return 0;
 }
@@ -1834,7 +1840,6 @@ _OVER:
   return ret;
 }
 
-
 static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp,
                               void *alterOriData, int32_t alterOriDataLen) {
   int32_t code = -1;
@@ -2091,6 +2096,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName,
       if (pCol->tableId == suid) {
         sdbRelease(pSdb, pTopic);
         nodesDestroyNode(pAst);
+        nodesDestroyList(pNodeList);
         return -1;
       } else {
         goto NEXT;
@@ -2099,6 +2105,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName,
   NEXT:
     sdbRelease(pSdb, pTopic);
     nodesDestroyNode(pAst);
+    nodesDestroyList(pNodeList);
   }
   return 0;
 }
@@ -2136,6 +2143,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
       if (pCol->tableId == suid) {
         sdbRelease(pSdb, pStream);
         nodesDestroyNode(pAst);
+        nodesDestroyList(pNodeList);
         return -1;
       } else {
         goto NEXT;
@@ -2144,6 +2152,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
   NEXT:
     sdbRelease(pSdb, pStream);
     nodesDestroyNode(pAst);
+    nodesDestroyList(pNodeList);
   }
   return 0;
 }

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -490,8 +490,12 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
     mndReleaseConsumer(pMnode, pConsumerOld);
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
       ASSERT(0);
+      tDeleteSMqConsumerObj(pConsumerNew);
+      taosMemoryFree(pConsumerNew);
       goto REB_FAIL;
     }
+    tDeleteSMqConsumerObj(pConsumerNew);
+    taosMemoryFree(pConsumerNew);
   }
 
   // 3.3 set removed consumer
@@ -509,8 +513,12 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
     mndReleaseConsumer(pMnode, pConsumerOld);
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
       ASSERT(0);
+      tDeleteSMqConsumerObj(pConsumerNew);
+      taosMemoryFree(pConsumerNew);
       goto REB_FAIL;
     }
+    tDeleteSMqConsumerObj(pConsumerNew);
+    taosMemoryFree(pConsumerNew);
   }
 #if 0
   if (consumerNum) {
@@ -900,6 +908,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName)
     // iter all vnode to delete handle
     if (taosHashGetSize(pSub->consumerHash) != 0) {
       sdbRelease(pSdb, pSub);
+      terrno = TSDB_CODE_MND_IN_REBALANCE;
       return -1;
     }
     int32_t sz = taosArrayGetSize(pSub->unassignedVgs);

source/dnode/mnode/impl/src/mndTopic.c

@@ -224,6 +224,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
     if (taosDecodeSSchemaWrapper(buf, &pTopic->schema) == NULL) {
       goto TOPIC_DECODE_OVER;
     }
+    taosMemoryFree(buf);
   } else {
     pTopic->schema.nCols = 0;
     pTopic->schema.version = 0;
@@ -266,6 +267,11 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) {
 
 static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
   mTrace("topic:%s, perform delete action", pTopic->name);
+  taosMemoryFreeClear(pTopic->sql);
+  taosMemoryFreeClear(pTopic->ast);
+  taosMemoryFreeClear(pTopic->physicalPlan);
+  if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
+  taosArrayDestroy(pTopic->ntbColIds);
   return 0;
 }
 
@@ -347,6 +353,7 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
       }
     }
   }
+  nodesDestroyList(pNodeList);
   return 0;
 }
 
@@ -416,6 +423,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
       taosMemoryFree(topicObj.sql);
       return -1;
     }
+    nodesDestroyNode(pAst);
+    nodesDestroyNode((SNode *)pPlan);
   } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
     SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
     if (pStb == NULL) {
@@ -512,6 +521,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
   }
 
   taosMemoryFreeClear(topicObj.physicalPlan);
+  taosMemoryFreeClear(topicObj.sql);
+  taosMemoryFreeClear(topicObj.ast);
+  taosArrayDestroy(topicObj.ntbColIds);
+  if (topicObj.schema.nCols) taosMemoryFreeClear(topicObj.schema.pSchema);
   mndTransDrop(pTrans);
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
@@ -713,7 +726,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
   mndReleaseTopic(pMnode, pTopic);
 
   if (code != 0) {
-    terrno = code;
     mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
     return -1;
   }

source/dnode/mnode/impl/src/mndTrans.c

@@ -790,10 +790,20 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
         if (mndCheckDbConflict(pNew->dbname1, pTrans)) conflict = true;
         if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true;
       }
+      if (pTrans->conflict == TRN_CONFLICT_DB_INSIDE) {
+        if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; // for stb
+      }
     }
 
-    mError("trans:%d, can't execute since conflict with trans:%d, db1:%s db2:%s", pNew->id, pTrans->id, pTrans->dbname1,
+    if (conflict) {
-           pTrans->dbname2);
+      mError("trans:%d, db1:%s db2:%s type:%d, can't execute since conflict with trans:%d db1:%s db2:%s type:%d",
+             pNew->id, pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2,
+             pTrans->conflict);
+    } else {
+      mDebug("trans:%d, db1:%s db2:%s type:%d, not conflict with trans:%d db1:%s db2:%s type:%d", pNew->id,
+             pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2,
+             pTrans->conflict);
+    }
     sdbRelease(pMnode->pSdb, pTrans);
   }
 

source/dnode/vnode/src/inc/tsdb.h

@@ -44,7 +44,6 @@ typedef struct SMapData      SMapData;
 typedef struct SBlockIdx     SBlockIdx;
 typedef struct SDataBlk      SDataBlk;
 typedef struct SSttBlk       SSttBlk;
-typedef struct SColData      SColData;
 typedef struct SDiskDataHdr  SDiskDataHdr;
 typedef struct SBlockData    SBlockData;
 typedef struct SDelFile      SDelFile;
@@ -71,10 +70,6 @@ typedef struct SLDataIter    SLDataIter;
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
 
-#define HAS_NONE  ((int8_t)0x1)
-#define HAS_NULL  ((int8_t)0x2)
-#define HAS_VALUE ((int8_t)0x4)
-
 #define VERSION_MIN 0
 #define VERSION_MAX INT64_MAX
 
@@ -148,15 +143,6 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
 int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
 int32_t tCmprBlockL(void const *lhs, void const *rhs);
-// SColdata
-void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
-void    tColDataReset(SColData *pColData);
-void    tColDataClear(void *ph);
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
-int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
-int32_t tPutColData(uint8_t *p, SColData *pColData);
-int32_t tGetColData(uint8_t *p, SColData *pColData);
 // SBlockData
 #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
@@ -299,8 +285,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
 int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData);
 int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
 // tsdbRead.c ==============================================================================================
-int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap);
+int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id);
-void    tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap);
+void    tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id);
 // tsdbMerge.c ==============================================================================================
 int32_t tsdbMerge(STsdb *pTsdb);
 
@@ -470,18 +456,6 @@ struct SSttBlk {
   SBlockInfo bInfo;
 };
 
-struct SColData {
-  int16_t  cid;
-  int8_t   type;
-  int8_t   smaOn;
-  int32_t  nVal;
-  uint8_t  flag;
-  uint8_t *pBitMap;
-  int32_t *aOffset;
-  int32_t  nData;
-  uint8_t *pData;
-};
-
 // (SBlockData){.suid = 0, .uid = 0}: block data not initialized
 // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
 // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file

source/dnode/vnode/src/tq/tq.c

@@ -832,7 +832,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   tDecoderClear(pCoder);
 
   int32_t sz = taosArrayGetSize(pRes->uidList);
-  if (sz == 0) {
+  if (sz == 0 || pRes->affectedRows == 0) {
     taosArrayDestroy(pRes->uidList);
     return 0;
   }

source/dnode/vnode/src/tq/tqOffset.c

@@ -147,6 +147,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
       taosHashCancelIterate(pStore->pHash, pIter);
       return -1;
     }
+    taosMemoryFree(buf);
   }
   // close and rename file
   taosCloseFile(&pFile);

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -847,7 +847,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
 
-  tsdbTakeReadSnap(pTsdb, &pIter->pReadSnap);
+  tsdbTakeReadSnap(pTsdb, &pIter->pReadSnap, NULL);
 
   STbData *pMem = NULL;
   if (pIter->pReadSnap->pMem) {
@@ -941,7 +941,7 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
     taosArrayDestroy(pIter->pSkyline);
   }
 
-  tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap);
+  tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap, NULL);
 
 _err:
   return code;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -58,7 +58,7 @@ static int32_t tsdbGnrtCurrent(STsdb *pTsdb, STsdbFS *pFS, char *fname) {
   taosCalcChecksumAppend(0, pData, size);
 
   // create and write
-  pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
+  pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -898,7 +898,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
           // null value exists, check one-by-one
           if (pData->flag != HAS_VALUE) {
             for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) {
-              uint8_t v = GET_BIT2(pData->pBitMap, j);
+              uint8_t v = tColDataGetBitValue(pData, j);
               if (v == 0 || v == 1) {
                 colDataSetNull_f(pColData->nullbitmap, rowIndex);
               }
@@ -1357,15 +1357,10 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
   pInfo->hasDupTs = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true;
   pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order);
 
-  // todo here we need to each key in the last files to identify if it is really overlapped with last block
+  if (hasDataInLastBlock(pLastBlockReader)) {
-  // todo
+    int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
-  bool overlapWithlastBlock = false;
+    pInfo->overlapWithLastBlock = !(pBlock->maxKey.ts < tsLast || pBlock->minKey.ts > tsLast);
-#if 0
-  if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
-    SSttBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
-    overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey);
   }
-#endif
 
   pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity;
   pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock);
@@ -1891,151 +1886,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
   return code;
 }
 
-#if 0
-static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) {
-  SRowMerger merge = {0};
-  STSRow*    pTSRow = NULL;
-
-  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-  SArray*             pDelList = pBlockScanInfo->delSkyline;
-
-  TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader);
-  TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader);
-  ASSERT(pRow != NULL && piRow != NULL);
-
-  int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex];
-  bool    freeTSRow = false;
-
-  uint64_t uid = pBlockScanInfo->uid;
-
-  TSDBKEY k = TSDBROW_KEY(pRow);
-  TSDBKEY ik = TSDBROW_KEY(piRow);
-  if (ASCENDING_TRAVERSE(pReader->order)) {
-    // [1&2] key <= [k.ts && ik.ts]
-    if (key <= k.ts && key <= ik.ts) {
-      TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-      tRowMergerInit(&merge, &fRow, pReader->pSchema);
-
-      doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
-
-      if (ik.ts == key) {
-        tRowMerge(&merge, piRow);
-        doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader);
-      }
-
-      if (k.ts == key) {
-        tRowMerge(&merge, pRow);
-        doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader);
-      }
-
-      tRowMergerGetRow(&merge, &pTSRow);
-      doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-      return TSDB_CODE_SUCCESS;
-    } else {  // key > ik.ts || key > k.ts
-      ASSERT(key != ik.ts);
-
-      // [3] ik.ts < key <= k.ts
-      // [4] ik.ts < k.ts <= key
-      if (ik.ts < k.ts) {
-        doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-        if (freeTSRow) {
-          taosMemoryFree(pTSRow);
-        }
-        return TSDB_CODE_SUCCESS;
-      }
-
-      // [5] k.ts < key   <= ik.ts
-      // [6] k.ts < ik.ts <= key
-      if (k.ts < ik.ts) {
-        doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-        if (freeTSRow) {
-          taosMemoryFree(pTSRow);
-        }
-        return TSDB_CODE_SUCCESS;
-      }
-
-      // [7] k.ts == ik.ts < key
-      if (k.ts == ik.ts) {
-        ASSERT(key > ik.ts && key > k.ts);
-
-        doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-        taosMemoryFree(pTSRow);
-        return TSDB_CODE_SUCCESS;
-      }
-    }
-  } else {  // descending order scan
-    // [1/2] k.ts >= ik.ts && k.ts >= key
-    if (k.ts >= ik.ts && k.ts >= key) {
-      STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
-
-      tRowMergerInit(&merge, pRow, pSchema);
-      doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader);
-
-      if (ik.ts == k.ts) {
-        tRowMerge(&merge, piRow);
-        doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader);
-      }
-
-      if (k.ts == key) {
-        TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-        tRowMerge(&merge, &fRow);
-        doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
-      }
-
-      tRowMergerGetRow(&merge, &pTSRow);
-      doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-      return TSDB_CODE_SUCCESS;
-    } else {
-      ASSERT(ik.ts != k.ts);  // this case has been included in the previous if branch
-
-      // [3] ik.ts > k.ts >= Key
-      // [4] ik.ts > key >= k.ts
-      if (ik.ts > key) {
-        doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-        if (freeTSRow) {
-          taosMemoryFree(pTSRow);
-        }
-        return TSDB_CODE_SUCCESS;
-      }
-
-      // [5] key > ik.ts > k.ts
-      // [6] key > k.ts > ik.ts
-      if (key > ik.ts) {
-        TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-        tRowMergerInit(&merge, &fRow, pReader->pSchema);
-
-        doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
-        tRowMergerGetRow(&merge, &pTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-        taosMemoryFree(pTSRow);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      //[7] key = ik.ts > k.ts
-      if (key == ik.ts) {
-        doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow);
-
-        TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-        tRowMerge(&merge, &fRow);
-        doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
-        tRowMergerGetRow(&merge, &pTSRow);
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid);
-
-        taosMemoryFree(pTSRow);
-        return TSDB_CODE_SUCCESS;
-      }
-    }
-  }
-
-  ASSERT(0);
-  return -1;
-}
-#endif
-
 static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
   if (pBlockScanInfo->iterInit) {
     return TSDB_CODE_SUCCESS;
@@ -2241,9 +2091,11 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
     // it is a clean block, load it directly
     if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
+      if (pReader->order == TSDB_ORDER_ASC || (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) {
         copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
         goto _end;
       }
+    }
   } else {  // file blocks not exist
     pBlockScanInfo = pReader->status.pTableIter;
   }
@@ -2252,9 +2104,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
   SBlockData*         pBlockData = &pReader->status.fileBlockData;
   int32_t             step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
 
-
   while (1) {
-    // todo check the validate of row in file block
     bool hasBlockData = false;
     {
       while (pBlockData->nRow > 0) {  // find the first qualified row in data block
@@ -2615,7 +2465,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
     code = buildComposedDataBlock(pReader);
   } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) {
     // data in memory that are earlier than current file block
-    // todo rows in buffer should be less than the file block in asc, greater than file block in desc
+    // rows in buffer should be less than the file block in asc, greater than file block in desc
     int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts;
     code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
   } else {
@@ -3508,7 +3358,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
   }
 
   if (numOfTables > 0) {
-    code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap);
+    code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -3532,7 +3382,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
       STsdbReader*    pPrevReader = pReader->innerReader[0];
       SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter;
 
-      code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap);
+      code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap, pReader->idStr);
       if (code != TSDB_CODE_SUCCESS) {
         goto _err;
       }
@@ -3590,7 +3440,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
     tsdbDataFReaderClose(&pReader->pFileReader);
   }
 
-  tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
+  tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr);
 
   taosMemoryFree(pReader->status.uidCheckInfo.tableUidList);
   SIOCostSummary* pCost = &pReader->cost;
@@ -3607,7 +3457,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
   }
 
   if (pReader->innerReader[0] != 0) {
-    tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap);
+    tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr);
   }
 
   tsdbDebug(
@@ -3870,8 +3720,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
     }
   }
 
-  tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s",
+  tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%"PRId64", query range:%" PRId64 " - %" PRId64 " in query %s",
-            pReader, pReader->suid, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr);
+            pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr);
 
   return code;
 }
@@ -4016,7 +3866,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap) {
+int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr) {
   int32_t code = 0;
 
   // alloc
@@ -4059,12 +3909,12 @@ int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap) {
     goto _exit;
   }
 
-  tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode));
+  tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
 _exit:
   return code;
 }
 
-void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
+void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) {
   if (pSnap) {
     if (pSnap->pMem) {
       tsdbUnrefMemTable(pSnap->pMem);
@@ -4077,6 +3927,5 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
     tsdbFSUnref(pTsdb, &pSnap->fs);
     taosMemoryFree(pSnap);
   }
-
+  tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
-  tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
 }

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -53,7 +53,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
   if (code) goto _err;
 
   for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) {
-    SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
+    SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet);
     int32_t    expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
     SDiskID    did;
 
@@ -65,6 +65,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
       taosArrayRemove(fs.aDFileSet, iSet);
       iSet--;
     } else {
+      if (expLevel == 0) continue;
       if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
         code = terrno;
         goto _exit;

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -909,292 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
   return code;
 }
 
-// SColData ========================================
-void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
-  pColData->cid = cid;
-  pColData->type = type;
-  pColData->smaOn = smaOn;
-  tColDataReset(pColData);
-}
-
-void tColDataReset(SColData *pColData) {
-  pColData->nVal = 0;
-  pColData->flag = 0;
-  pColData->nData = 0;
-}
-
-void tColDataClear(void *ph) {
-  SColData *pColData = (SColData *)ph;
-
-  tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
-  tFree(pColData->pData);
-}
-
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
-  int32_t code = 0;
-  int64_t size;
-  SValue  value = {0};
-  SValue *pValue = &value;
-
-  ASSERT(pColVal->cid == pColData->cid);
-  ASSERT(pColVal->type == pColData->type);
-
-  // realloc bitmap
-  size = BIT2_SIZE(pColData->nVal + 1);
-  code = tRealloc(&pColData->pBitMap, size);
-  if (code) goto _exit;
-  if ((pColData->nVal & 3) == 0) {
-    pColData->pBitMap[pColData->nVal >> 2] = 0;
-  }
-
-  // put value
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
-  } else {
-    pColData->flag |= HAS_VALUE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
-    pValue = &pColVal->value;
-  }
-
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
-    if (code) goto _exit;
-    pColData->aOffset[pColData->nVal] = pColData->nData;
-
-    // value
-    if ((!pColVal->isNone) && (!pColVal->isNull)) {
-      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
-      if (code) goto _exit;
-      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
-      pColData->nData += pColVal->value.nData;
-    }
-  } else {
-    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
-    if (code) goto _exit;
-    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
-  }
-
-  pColData->nVal++;
-
-_exit:
-  return code;
-}
-
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
-  int32_t code = 0;
-  int32_t size;
-
-  ASSERT(pColDataSrc->nVal > 0);
-  ASSERT(pColDataDest->cid = pColDataSrc->cid);
-  ASSERT(pColDataDest->type = pColDataSrc->type);
-
-  pColDataDest->smaOn = pColDataSrc->smaOn;
-  pColDataDest->nVal = pColDataSrc->nVal;
-  pColDataDest->flag = pColDataSrc->flag;
-
-  // bitmap
-  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
-    size = BIT2_SIZE(pColDataSrc->nVal);
-    code = tRealloc(&pColDataDest->pBitMap, size);
-    if (code) goto _exit;
-    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
-  }
-
-  // offset
-  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
-    size = sizeof(int32_t) * pColDataSrc->nVal;
-
-    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
-    if (code) goto _exit;
-
-    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
-  }
-
-  // value
-  pColDataDest->nData = pColDataSrc->nData;
-  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
-  if (code) goto _exit;
-  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
-
-_exit:
-  return code;
-}
-
-int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
-  int32_t code = 0;
-
-  ASSERT(iVal < pColData->nVal);
-  ASSERT(pColData->flag);
-
-  if (pColData->flag == HAS_NONE) {
-    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag == HAS_NULL) {
-    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag != HAS_VALUE) {
-    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-    if (v == 0) {
-      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-      goto _exit;
-    } else if (v == 1) {
-      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-      goto _exit;
-    }
-  }
-
-  // get value
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    if (iVal + 1 < pColData->nVal) {
-      pColVal->value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
-    } else {
-      pColVal->value.nData = pColData->nData - pColData->aOffset[iVal];
-    }
-
-    pColVal->value.pData = pColData->pData + pColData->aOffset[iVal];
-  } else {
-    int32_t n = 0;
-    uint8_t* p = pColData->pData + tDataTypes[pColData->type].bytes * iVal;
-
-//    if (IS_VAR_DATA_TYPE(pColData->type)) {
-//      n = tGetBinary(p, &pColVal->value.pData, pColVal->value.pData ? &pColVal->value.nData : NULL);
-//    } else {
-      switch (pColData->type) {
-        case TSDB_DATA_TYPE_BOOL:
-        case TSDB_DATA_TYPE_TINYINT:
-          pColVal->value.i8 = *(int8_t*)p;
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-          pColVal->value.i16 = *(int16_t*)p;
-          break;
-        case TSDB_DATA_TYPE_INT:
-          pColVal->value.i32 = *(int32_t*)p;
-          break;
-        case TSDB_DATA_TYPE_TIMESTAMP:
-        case TSDB_DATA_TYPE_BIGINT:
-          pColVal->value.i64 = *(int64_t*)p;
-          break;
-        case TSDB_DATA_TYPE_FLOAT:
-          n += tGetFloat((uint8_t*)p, &pColVal->value.f);
-          break;
-        case TSDB_DATA_TYPE_DOUBLE:
-          n += tGetDouble((uint8_t*)p, &pColVal->value.d);
-          break;
-        case TSDB_DATA_TYPE_UTINYINT:
-          pColVal->value.u8 = *(uint8_t*)p;
-          break;
-        case TSDB_DATA_TYPE_USMALLINT:
-          pColVal->value.u16 = *(uint16_t*)p;
-          break;
-        case TSDB_DATA_TYPE_UINT:
-          pColVal->value.u32 = *(uint32_t*)p;
-          break;
-        case TSDB_DATA_TYPE_UBIGINT:
-          pColVal->value.u64 = *(uint64_t*)p;
-          break;
-        default:
-          ASSERT(0);
-      }
-//    }
-//    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &pColVal->value, pColData->type);
-  }
-
-  pColVal->cid = pColData->cid;
-  pColVal->type = pColData->type;
-
-_exit:
-  return code;
-}
-
-int32_t tPutColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tPutI16v(p ? p + n : p, pColData->cid);
-  n += tPutI8(p ? p + n : p, pColData->type);
-  n += tPutI8(p ? p + n : p, pColData->smaOn);
-  n += tPutI32v(p ? p + n : p, pColData->nVal);
-  n += tPutU8(p ? p + n : p, pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    if (p) {
-      memcpy(p + n, pColData->pBitMap, size);
-    }
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    if (p) {
-      memcpy(p + n, pColData->aOffset, size);
-    }
-    n += size;
-  }
-  n += tPutI32v(p ? p + n : p, pColData->nData);
-  if (p) {
-    memcpy(p + n, pColData->pData, pColData->nData);
-  }
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-int32_t tGetColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tGetI16v(p + n, &pColData->cid);
-  n += tGetI8(p + n, &pColData->type);
-  n += tGetI8(p + n, &pColData->smaOn);
-  n += tGetI32v(p + n, &pColData->nVal);
-  n += tGetU8(p + n, &pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    pColData->pBitMap = p + n;
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    pColData->aOffset = (int32_t *)(p + n);
-    n += size;
-  }
-  n += tGetI32v(p + n, &pColData->nData);
-  pColData->pData = p + n;
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
-  SColData *pColData1 = (SColData *)p1;
-  SColData *pColData2 = (SColData *)p2;
-
-  if (pColData1->cid < pColData2->cid) {
-    return -1;
-  } else if (pColData1->cid > pColData2->cid) {
-    return 1;
-  }
-
-  return 0;
-}
-
 // SBlockData ======================================================
 int32_t tBlockDataCreate(SBlockData *pBlockData) {
   int32_t code = 0;
@@ -1226,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
   tFree((uint8_t *)pBlockData->aVersion);
   tFree((uint8_t *)pBlockData->aTSKEY);
   taosArrayDestroy(pBlockData->aIdx);
-  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
+  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
   pBlockData->aUid = NULL;
   pBlockData->aVersion = NULL;
   pBlockData->aTSKEY = NULL;
@@ -1295,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
   for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-    tColDataReset(pColData);
+    tColDataClear(pColData);
   }
 }
 
@@ -1545,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
   while (lidx <= ridx) {
     int32_t   midx = (lidx + ridx) / 2;
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
-    int32_t   c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
+    int32_t   c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
 
     if (c == 0) {
       *ppColData = pColData;
@@ -2030,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   int32_t size = 0;
   // bitmap
   if (pColData->flag != HAS_VALUE) {
-    uint8_t *pBitMap = pColData->pBitMap;
+    int32_t szBitMap;
-    int32_t  szBitMap = BIT2_SIZE(pColData->nVal);
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-
+      szBitMap = BIT2_SIZE(pColData->nVal);
-    // BIT2 to BIT1
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      szBitMap = BIT1_SIZE(pColData->nVal);
-      pBitMap = taosMemoryCalloc(1, szBitMap);
-      if (pBitMap == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _exit;
-      }
-
-      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-        uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-        switch (pColData->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT1(pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT1(pBitMap, iVal, 1);
     } else {
-              SET_BIT1(pBitMap, iVal, 0);
+      szBitMap = BIT1_SIZE(pColData->nVal);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT1(pBitMap, iVal, v - 1);
-            break;
-          default:
-            ASSERT(0);
-        }
-      }
     }
 
-    code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap,
+    code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size,
-                        ppBuf);
+                        &pBlockCol->szBitmap, ppBuf);
     if (code) goto _exit;
-
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      taosMemoryFree(pBitMap);
-    }
   }
   size += pBlockCol->szBitmap;
 
@@ -2108,47 +1791,16 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
   uint8_t *p = pIn;
   // bitmap
   if (pBlockCol->szBitmap) {
-    if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+    int32_t szBitMap;
-      uint8_t *pBitMap = NULL;
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap,
+      szBitMap = BIT2_SIZE(pColData->nVal);
-                            BIT1_SIZE(pColData->nVal), ppBuf);
-      if (code) goto _exit;
-
-      code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal));
-      if (code) {
-        tFree(pBitMap);
-        goto _exit;
-      }
-
-      // BIT1 to BIT2
-      for (int32_t iVal = 0; iVal < nVal; iVal++) {
-        uint8_t v = GET_BIT1(pBitMap, iVal);
-        switch (pBlockCol->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT2(pColData->pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT2(pColData->pBitMap, iVal, 2);
     } else {
-              SET_BIT2(pColData->pBitMap, iVal, 0);
+      szBitMap = BIT1_SIZE(pColData->nVal);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT2(pColData->pBitMap, iVal, v + 1);
-            break;
-          default:
-            ASSERT(0);
-        }
     }
 
-      tFree(pBitMap);
+    code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf);
-    } else {
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap,
-                            BIT2_SIZE(pColData->nVal), ppBuf);
     if (code) goto _exit;
   }
-  }
   p += pBlockCol->szBitmap;
 
   // offset

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -533,6 +533,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR
     taosArrayPush(rsp.pArray, &cRsp);
   }
 
+  vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids));
   tqUpdateTbUidList(pVnode->pTq, tbUids, true);
   if (tdUpdateTbUidList(pVnode->pSma, pStore, true) < 0) {
     goto _exit;
@@ -885,8 +886,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
         if (NULL != submitBlkRsp.pMeta) {
           vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta);
         }
-      }
+
         taosArrayPush(newTbUids, &createTbReq.uid);
+      }
 
       submitBlkRsp.uid = createTbReq.uid;
       submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
@@ -917,6 +919,11 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
     submitRsp.affectedRows += submitBlkRsp.affectedRows;
     taosArrayPush(submitRsp.pArray, &submitBlkRsp);
   }
+
+  if (taosArrayGetSize(newTbUids) > 0) {
+    vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids));
+  }
+
   tqUpdateTbUidList(pVnode->pTq, newTbUids, true);
 
 _exit:

source/libs/catalog/src/ctgAsync.c

@@ -13,12 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "trpc.h"
-#include "query.h"
-#include "tname.h"
 #include "catalogInt.h"
+#include "query.h"
 #include "systable.h"
+#include "tname.h"
 #include "tref.h"
+#include "trpc.h"
 
 int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   SName*   name = (SName*)param;
@@ -45,7 +45,8 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -69,8 +70,8 @@ int32_t ctgInitGetTbMetasTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d",
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx,
-         pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum);
+         ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -94,7 +95,8 @@ int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -118,7 +120,8 @@ int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -142,12 +145,12 @@ int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   SName*   name = (SName*)param;
   SCtgTask task = {0};
@@ -173,7 +176,8 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -197,13 +201,12 @@ int32_t ctgInitGetTbHashsTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d",
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx,
-         pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum);
+         ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum);
 
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   SCtgTask task = {0};
 
@@ -253,7 +256,8 @@ int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -277,7 +281,8 @@ int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -301,7 +306,8 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), user->user);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -344,7 +350,8 @@ int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -373,12 +380,12 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
   SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@@ -455,7 +462,6 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con
 
   taosHashCleanup(pTb);
 
-
   for (int32_t i = 0; i < pJob->tbIndexNum; ++i) {
     SName* name = taosArrayGet(pReq->pTableIndex, i);
     ctgDropTbIndexEnqueue(pCtg, name, true);
@@ -478,7 +484,8 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param) {
+int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp,
+                   void* param) {
   int32_t code = 0;
   int32_t tbMetaNum = (int32_t)ctgGetTablesReqNum(pReq->pTableMeta);
   int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup);
@@ -494,7 +501,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
   int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
   int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
 
-  int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum;
+  int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
+                    userNum + dbInfoNum + tbIndexNum + tbCfgNum;
 
   *job = taosMemoryCalloc(1, sizeof(SCtgJob));
   if (NULL == *job) {
@@ -526,7 +534,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
   pJob->svrVerNum = svrVerNum;
 
 #if CTG_BATCH_FETCH
-  pJob->pBatchs = taosHashInit(CTG_DEFAULT_BATCH_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  pJob->pBatchs =
+      taosHashInit(CTG_DEFAULT_BATCH_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
   if (NULL == pJob->pBatchs) {
     ctgError("taosHashInit %d batch failed", CTG_DEFAULT_BATCH_NUM);
     CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
@@ -625,10 +634,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const
 
   taosAcquireRef(gCtgMgmt.jobPool, pJob->refId);
 
-  qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate);
+  qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId,
+         taskNum, pReq->forceUpdate);
   return TSDB_CODE_SUCCESS;
 
-
 _return:
 
   ctgFreeJob(*job);
@@ -658,7 +667,6 @@ int32_t ctgDumpTbMetasRes(SCtgTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbVgroup) {
@@ -772,7 +780,6 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbCfg) {
@@ -910,7 +917,8 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
 
   int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
   if (taskDone < taosArrayGetSize(pJob->pTasks)) {
-    qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
+    qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone,
+           (int32_t)taosArrayGetSize(pJob->pTasks));
     return TSDB_CODE_SUCCESS;
   }
 
@@ -1092,7 +1100,6 @@ _return:
   CTG_RET(code);
 }
 
-
 int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
   int32_t           code = 0;
   SCtgDBCache*      dbCache = NULL;
@@ -1277,7 +1284,6 @@ _return:
   CTG_RET(code);
 }
 
-
 int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
   int32_t      code = 0;
   SCtgTask*    pTask = tReq->pTask;
@@ -1304,7 +1310,6 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *
       break;
   }
 
-
 _return:
 
   ctgHandleTaskEnd(pTask, code);
@@ -1342,7 +1347,6 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf
       break;
   }
 
-
 _return:
 
   ctgHandleTaskEnd(pTask, code);
@@ -1408,7 +1412,6 @@ _return:
   CTG_RET(code);
 }
 
-
 int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
   int32_t   code = 0;
   SCtgTask* pTask = tReq->pTask;
@@ -1464,7 +1467,6 @@ int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf
   CTG_RET(TSDB_CODE_APP_ERROR);
 }
 
-
 int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
   int32_t   code = 0;
   SCtgTask* pTask = tReq->pTask;
@@ -1541,9 +1543,11 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *
     goto _return;
   }
 
-  if (ctx->user.type == AUTH_TYPE_READ && pOut->readDbs && taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
+  if (ctx->user.type == AUTH_TYPE_READ && pOut->readDbs &&
+      taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
     pass = true;
-  } else if (ctx->user.type == AUTH_TYPE_WRITE && pOut->writeDbs && taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
+  } else if (ctx->user.type == AUTH_TYPE_WRITE && pOut->writeDbs &&
+             taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) {
     pass = true;
   }
 
@@ -1805,7 +1809,8 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) {
       SCtgTaskReq tReq;
       tReq.pTask = pTask;
       tReq.msgIdx = -1;
-      CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false));
+      CTG_ERR_JRET(
+          ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false));
 
       ctgReleaseVgInfoToCache(pCtg, dbCache);
       dbCache = NULL;
@@ -1858,7 +1863,6 @@ _return:
   return code;
 }
 
-
 int32_t ctgLaunchGetTbIndexTask(SCtgTask* pTask) {
   int32_t           code = 0;
   SCatalog*         pCtg = pTask->pJob->pCtg;
@@ -1930,7 +1934,6 @@ _return:
   CTG_RET(code);
 }
 
-
 int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) {
   SCatalog*         pCtg = pTask->pJob->pCtg;
   SRequestConnInfo* pConn = &pTask->pJob->conn;
@@ -1957,7 +1960,6 @@ int32_t ctgLaunchGetDnodeTask(SCtgTask *pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) {
   SCatalog*         pCtg = pTask->pJob->pCtg;
   SRequestConnInfo* pConn = &pTask->pJob->conn;
@@ -2124,7 +2126,6 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) {
   SCtgTbMetaCtx* ctx = pTask->taskCtx;
 
@@ -2145,14 +2146,14 @@ int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) {
   CTG_RET(cloneDbVgInfo(pOut->dbVgroup, (SDBVgInfo**)pRes));
 }
 
-
 SCtgAsyncFps gCtgAsyncFps[] = {
     {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL},
     {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL},
     {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg},
     {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL},
     {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL},
-  {ctgInitGetTbMetaTask,   ctgLaunchGetTbMetaTask,   ctgHandleGetTbMetaRsp,  ctgDumpTbMetaRes,  ctgCompTbMetaTasks, ctgCloneTbMeta},
+    {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks,
+     ctgCloneTbMeta},
     {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL},
     {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL},
     {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL},
@@ -2233,7 +2234,6 @@ _return:
   CTG_RET(code);
 }
 
-
 int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) {
   SCtgJob* pJob = pTask->pJob;
   int32_t  subTaskId = -1;
@@ -2289,6 +2289,3 @@ int32_t ctgLaunchJob(SCtgJob *pJob) {
 
   return TSDB_CODE_SUCCESS;
 }
-
-
-

source/libs/catalog/src/ctgRemote.c

@@ -75,7 +75,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
     SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx);
     pMsgCtx->pBatchs = pBatchs;
 
-    ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
+    ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId,
+             rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
 
     (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode));
   }
@@ -351,6 +352,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
     SCtgTaskReq tReq;
     tReq.pTask = pTask;
     tReq.msgIdx = -1;
+
     CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, cbParam->reqType, pMsg, rspCode));
 
 #if CTG_BATCH_FETCH
@@ -361,6 +363,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
 _return:
 
   taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
 
   if (pJob) {
     taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId);
@@ -442,8 +445,8 @@ _return:
   CTG_RET(code);
 }
 
-int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTaskReq* tReq, int32_t msgType, void* msg,
+int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTaskReq* tReq, int32_t msgType,
-                    uint32_t msgSize) {
+                    void* msg, uint32_t msgSize) {
   int32_t     code = 0;
   SCtgTask*   pTask = tReq->pTask;
   SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
@@ -1243,7 +1246,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
     }
     taosArrayPush(pTaskId, &pTask->taskId);
 
-    CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen));
+    CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg,
+                            msgLen));
 #endif
   }
 

source/libs/catalog/src/ctgUtil.c

@@ -99,7 +99,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
 }
 
 void ctgFreeQNode(SCtgQNode *node) {
-  //TODO
+  if (NULL == node) {
+    return;
+  }
+
+  if (node->op) {
+    taosMemoryFree(node->op->data);
+    taosMemoryFree(node->op);
+  }
+  
+  taosMemoryFree(node);
 }
 
 void ctgFreeSTableIndex(void *info) {

source/libs/executor/src/dataDeleter.c

@@ -79,21 +79,32 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
   pEntry->dataLen = sizeof(SDeleterRes);
 
   ASSERT(1 == pEntry->numOfRows);
-  ASSERT(1 == pEntry->numOfCols);
+  ASSERT(3 == pEntry->numOfCols);
 
   pBuf->useSize = sizeof(SDataCacheEntry);
 
   SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
+  SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1);
+  SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2);
 
   SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
   pRes->suid = pHandle->pParam->suid;
   pRes->uidList = pHandle->pParam->pUidList;
-  pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
-  pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;
   strcpy(pRes->tableName, pHandle->pDeleter->tableFName);
   strcpy(pRes->tsColName, pHandle->pDeleter->tsColName);
   pRes->affectedRows = *(int64_t*)pColRes->pData;
 
+  if (pRes->affectedRows) {
+    pRes->skey = *(int64_t*)pColSKey->pData;
+    pRes->ekey = *(int64_t*)pColEKey->pData;
+    ASSERT(pRes->skey <= pRes->ekey);
+  } else {
+    pRes->skey = pHandle->pDeleter->deleteTimeRange.skey;
+    pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey;
+  }
+
+  qDebug("delete %ld rows, from %ld to %ld", pRes->affectedRows, pRes->skey, pRes->ekey);
+
   pBuf->useSize += pEntry->dataLen;
 
   atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
@@ -172,7 +183,8 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
   SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData;
   *pLen = pEntry->dataLen;
   *pQueryEnd = pDeleter->queryEnd;
-  qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
+  qDebug("got data len %" PRId64 ", row num %d in sink", *pLen,
+         ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
 }
 
 static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
@@ -230,7 +242,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam) {
+int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle,
+                          void* pParam) {
   SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle));
   if (NULL == deleter) {
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;

source/libs/executor/src/executor.c

@@ -265,6 +265,15 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
 
 int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
   SExecTaskInfo*  pTaskInfo = (SExecTaskInfo*)tinfo;
+  STableListInfo* pListInfo = &pTaskInfo->tableqinfoList;
+
+  if (isAdd) {
+    qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
+  }
+
+  if (pListInfo->map == NULL) {
+    pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  }
 
   // traverse to the stream scanner node to add this table id
   SOperatorInfo* pInfo = pTaskInfo->pRoot;
@@ -311,14 +320,22 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
         }
       }
 
-      taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo);
+      bool exists = false;
-      if (pTaskInfo->tableqinfoList.map == NULL) {
+#if 0
-        pTaskInfo->tableqinfoList.map =
+      for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) {
-            taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+        STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k);
+        if (pKeyInfo->uid == keyInfo.uid) {
+          qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str);
+          exists = true;
         }
+      }
+#endif
 
+      if (!exists) {
+        taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo);
         taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId));
       }
+    }
 
     if (keyBuf != NULL) {
       taosMemoryFree(keyBuf);
@@ -480,6 +497,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) {
   if (ret != TSDB_CODE_SUCCESS) {
     pTaskInfo->code = ret;
     cleanUpUdfs();
+
     qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
     atomic_store_64(&pTaskInfo->owner, 0);
 
@@ -512,8 +530,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) {
   }
 
   cleanUpUdfs();
-  uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
 
+  uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
   qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
          GET_TASKID(pTaskInfo), current, (int32_t)taosArrayGetSize(pResList), total, 0, el / 1000.0);
 

source/libs/executor/src/scanoperator.c

@@ -617,19 +617,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
   // if scan table by table
   if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
-    if (pInfo->noTable) return NULL;
+    if (pInfo->noTable) {
+      return NULL;
+    }
+
+    int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+
     while (1) {
       SSDataBlock* result = doTableScanGroup(pOperator);
       if (result) {
         return result;
       }
+
       // if no data, switch to next table and continue scan
       pInfo->currentTable++;
-      if (pInfo->currentTable >= taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList)) {
+      if (pInfo->currentTable >= numOfTables) {
         return NULL;
       }
+
       STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
       tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
+      qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables,
+             pInfo->currentTable, pTaskInfo->id.str);
+
       tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
       pInfo->scanTimes = 0;
     }
@@ -1323,6 +1333,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
     bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
                      isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup);
     if ((update || closedWin) && out) {
+      qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
       uint64_t gpId = closedWin && pInfo->partitionSup.needCalc
                           ? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId)
                           : 0;

source/libs/function/CMakeLists.txt

@@ -14,7 +14,7 @@ target_include_directories(
 
 target_link_libraries(
     function
-    PRIVATE os util common nodes scalar qcom transport
+    PRIVATE os util common nodes scalar qcom transport stream
     PUBLIC uv_a
 )
 

source/libs/function/src/builtins.c

@@ -311,22 +311,6 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
-    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
-  }
-
-  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
-  if (!IS_TIMESTAMP_TYPE(paraType) && !IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
-    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  } else if (IS_NULL_TYPE(paraType)) {
-    paraType = TSDB_DATA_TYPE_BIGINT;
-  }
-
-  pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isLtrim) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -2076,7 +2060,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "min",
     .type = FUNCTION_TYPE_MIN,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
-    .translateFunc = translateMinMax,
+    .translateFunc = translateInOutNum,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
     .initFunc     = minmaxFunctionSetup,
@@ -2091,7 +2075,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "max",
     .type = FUNCTION_TYPE_MAX,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
-    .translateFunc = translateMinMax,
+    .translateFunc = translateInOutNum,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
     .initFunc     = minmaxFunctionSetup,

source/libs/function/src/builtinsimpl.c

@@ -18,6 +18,7 @@
 #include "function.h"
 #include "query.h"
 #include "querynodes.h"
+#include "streamState.h"
 #include "tcompare.h"
 #include "tdatablock.h"
 #include "tdigest.h"
@@ -56,8 +57,13 @@ typedef struct SAvgRes {
 } SAvgRes;
 
 typedef struct STuplePos {
+  union {
+    struct {
       int32_t pageId;
       int32_t offset;
+    };
+    STupleKey streamTupleKey;
+  };
 } STuplePos;
 
 typedef struct SMinmaxResInfo {
@@ -1146,7 +1152,8 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   return true;
 }
 
-static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock);
+static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock,
+                               const STupleKey* pKey);
 static int32_t   updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
 static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);
 
@@ -1201,10 +1208,10 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
       pBuf->v = *(int64_t*)tval;
       if (pCtx->subsidiaries.num > 0) {
         index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
-        pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+        pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
       }
     } else {
-      if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) {
+      if (IS_SIGNED_NUMERIC_TYPE(type)) {
         int64_t prev = 0;
         GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
 
@@ -1213,7 +1220,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
           *(int64_t*)&pBuf->v = val;
           if (pCtx->subsidiaries.num > 0) {
             index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
-            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
           }
         }
       } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
@@ -1225,7 +1232,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
           *(uint64_t*)&pBuf->v = val;
           if (pCtx->subsidiaries.num > 0) {
             index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
-            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
           }
         }
       } else if (type == TSDB_DATA_TYPE_DOUBLE) {
@@ -1237,7 +1244,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
           *(double*)&pBuf->v = val;
           if (pCtx->subsidiaries.num > 0) {
             index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
-            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
           }
         }
       } else if (type == TSDB_DATA_TYPE_FLOAT) {
@@ -1251,7 +1258,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
 
         if (pCtx->subsidiaries.num > 0) {
           index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
-          pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+          pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
         }
       }
     }
@@ -1263,7 +1270,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
   int32_t start = pInput->startRowIndex;
   int32_t numOfRows = pInput->numOfRows;
 
-  if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
+  if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
     if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
       int8_t* pData = (int8_t*)pCol->pData;
       int8_t* val = (int8_t*)&pBuf->v;
@@ -1276,7 +1283,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1307,7 +1314,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1338,7 +1345,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1357,7 +1364,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
 
         numOfElems += 1;
       }
-    } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) {
+    } else if (type == TSDB_DATA_TYPE_BIGINT) {
       int64_t* pData = (int64_t*)pCol->pData;
       int64_t* val = (int64_t*)&pBuf->v;
 
@@ -1369,7 +1376,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1402,7 +1409,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1433,7 +1440,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1464,7 +1471,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1495,7 +1502,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         if (!pBuf->assign) {
           *val = pData[i];
           if (pCtx->subsidiaries.num > 0) {
-            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+            pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
           }
           pBuf->assign = true;
         } else {
@@ -1527,7 +1534,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
       if (!pBuf->assign) {
         *val = pData[i];
         if (pCtx->subsidiaries.num > 0) {
-          pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+          pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
         }
         pBuf->assign = true;
       } else {
@@ -1558,7 +1565,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
       if (!pBuf->assign) {
         *val = pData[i];
         if (pCtx->subsidiaries.num > 0) {
-          pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
+          pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
         }
         pBuf->assign = true;
       } else {
@@ -1581,7 +1588,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
 
 _min_max_over:
   if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
-    pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
+    pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
     pBuf->nullTupleSaved = true;
   }
   return numOfElems;
@@ -2758,7 +2765,7 @@ static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowInde
   }
 
   if (!pInfo->hasResult) {
-    pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock);
+    pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
   } else {
     updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos);
   }
@@ -3426,7 +3433,7 @@ int32_t topFunction(SqlFunctionCtx* pCtx) {
   }
 
   if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
-    pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
+    pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
     pRes->nullTupleSaved = true;
   }
   return TSDB_CODE_SUCCESS;
@@ -3454,7 +3461,7 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) {
   }
 
   if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
-    pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
+    pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
     pRes->nullTupleSaved = true;
   }
 
@@ -3506,7 +3513,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
 
     // save the data of this tuple
     if (pCtx->subsidiaries.num > 0) {
-      pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock);
+      pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
     }
 #ifdef BUF_PAGE_DEBUG
     qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId,
@@ -3578,7 +3585,8 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
   return buf;
 }
 
-static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length) {
+static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length,
+                                 const STupleKey* pKey) {
   STuplePos p = {0};
   if (pHandle->pBuf != NULL) {
     SFilePage* pPage = NULL;
@@ -3604,12 +3612,16 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf
     releaseBufPage(pHandle->pBuf, pPage);
   } else {
     // other tuple save policy
+    if (streamStateFuncPut(pHandle->pState, pKey, pBuf, length) < 0) {
+      ASSERT(0);
+    }
+    p.streamTupleKey = *pKey;
   }
 
   return p;
 }
 
-STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock) {
+STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey) {
   if (pCtx->subsidiaries.rowLen == 0) {
     int32_t rowLen = 0;
     for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
@@ -3622,7 +3634,7 @@ STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc
   }
 
   char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
-  return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen);
+  return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pKey);
 }
 
 static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) {
@@ -3632,6 +3644,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
     setBufPageDirty(pPage, true);
     releaseBufPage(pHandle->pBuf, pPage);
   } else {
+    streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -3650,7 +3663,10 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo
     releaseBufPage(pHandle->pBuf, pPage);
     return p;
   } else {
-    return NULL;
+    void*   value = NULL;
+    int32_t vLen;
+    streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
+    return (char*)value;
   }
 }
 
@@ -4764,7 +4780,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
       colDataAppendNULL(pOutput, i);
       // handle selectivity
       if (pCtx->subsidiaries.num > 0) {
-        appendSelectivityValue(pCtx, i, i);
+        appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
       }
       continue;
     }
@@ -4781,11 +4797,11 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
     } else {
       pInfo->durationStart = 0;
     }
-    colDataAppend(pOutput, i, (char*)&output, false);
+    colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false);
 
     // handle selectivity
     if (pCtx->subsidiaries.num > 0) {
-      appendSelectivityValue(pCtx, i, i);
+      appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
     }
   }
 
@@ -4981,7 +4997,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da
   if (pInfo->numSampled < pInfo->samples) {
     sampleAssignResult(pInfo, data, pInfo->numSampled);
     if (pCtx->subsidiaries.num > 0) {
-      pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock);
+      pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
     }
     pInfo->numSampled++;
   } else {
@@ -5012,7 +5028,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
   }
 
   if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) {
-    pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
+    pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
     pInfo->nullTupleSaved = true;
   }
 
@@ -5398,8 +5414,8 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
 
   int32_t i = pInput->startRowIndex;
   if (pCtx->start.key != INT64_MIN) {
-    ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) ||
+    //ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) ||
-           (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC));
+    //       (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC));
 
     ASSERT(last->key == INT64_MIN);
     for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
@@ -5447,6 +5463,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5462,6 +5482,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5476,6 +5500,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5490,6 +5518,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5504,6 +5536,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5518,6 +5554,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5532,6 +5572,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5546,6 +5590,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5560,6 +5608,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }
@@ -5574,6 +5626,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
         numOfElems++;
 
         INIT_INTP_POINT(st, tsList[i], val[i]);
+        if (pInfo->p.key == st.key) {
+          return TSDB_CODE_FUNC_DUP_TIMESTAMP;
+        }
+
         pInfo->dOutput += twa_get_area(pInfo->p, st);
         pInfo->p = st;
       }

source/libs/index/src/indexTfile.c

@@ -323,10 +323,6 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
   while ((rt = stmStNextWith(st, NULL)) != NULL) {
     FstSlice* s = &rt->data;
     char*     ch = (char*)fstSliceData(s, NULL);
-    // if (0 != strncmp(ch, tem->colName, tem->nColName)) {
-    //  swsResultDestroy(rt);
-    //  break;
-    //}
 
     TExeCond cond = cmpFn(ch, p, tem->colType);
     if (MATCH == cond) {

source/libs/nodes/src/nodesCloneFuncs.c

@@ -399,6 +399,8 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi
   COPY_SCALAR_FIELD(modifyType);
   COPY_SCALAR_FIELD(msgType);
   CLONE_NODE_FIELD(pAffectedRows);
+  CLONE_NODE_FIELD(pStartTs);
+  CLONE_NODE_FIELD(pEndTs);
   COPY_SCALAR_FIELD(tableId);
   COPY_SCALAR_FIELD(stableId);
   COPY_SCALAR_FIELD(tableType);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -2431,6 +2431,8 @@ static const char* jkDeletePhysiPlanTsColName = "TsColName";
 static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey";
 static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey";
 static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows";
+static const char* jkDeletePhysiPlanStartTs = "StartTs";
+static const char* jkDeletePhysiPlanEndTs = "EndTs";
 
 static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
   const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj;
@@ -2457,6 +2459,12 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeletePhysiPlanStartTs, nodeToJson, pNode->pStartTs);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeletePhysiPlanEndTs, nodeToJson, pNode->pEndTs);
+  }
 
   return code;
 }
@@ -2486,6 +2494,12 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeletePhysiPlanStartTs, &pNode->pStartTs);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeletePhysiPlanEndTs, &pNode->pEndTs);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -727,6 +727,8 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyNode(pStmt->pFromTable);
       nodesDestroyNode(pStmt->pWhere);
       nodesDestroyNode(pStmt->pCountFunc);
+      nodesDestroyNode(pStmt->pFirstFunc);
+      nodesDestroyNode(pStmt->pLastFunc);
       nodesDestroyNode(pStmt->pTagCond);
       break;
     }
@@ -791,6 +793,8 @@ void nodesDestroyNode(SNode* pNode) {
       destroyVgDataBlockArray(pLogicNode->pDataBlocks);
       // pVgDataBlocks is weak reference
       nodesDestroyNode(pLogicNode->pAffectedRows);
+      nodesDestroyNode(pLogicNode->pStartTs);
+      nodesDestroyNode(pLogicNode->pEndTs);
       taosMemoryFreeClear(pLogicNode->pVgroupList);
       nodesDestroyList(pLogicNode->pInsertCols);
       break;
@@ -997,6 +1001,8 @@ void nodesDestroyNode(SNode* pNode) {
       SDataDeleterNode* pSink = (SDataDeleterNode*)pNode;
       destroyDataSinkNode((SDataSinkNode*)pSink);
       nodesDestroyNode(pSink->pAffectedRows);
+      nodesDestroyNode(pSink->pStartTs);
+      nodesDestroyNode(pSink->pEndTs);
       break;
     }
     case QUERY_NODE_PHYSICAL_SUBPLAN: {

source/libs/parser/inc/sql.y

@@ -452,7 +452,7 @@ sma_stream_opt(A) ::= stream_options(B) WATERMARK duration_literal(C).
 sma_stream_opt(A) ::= stream_options(B) MAX_DELAY duration_literal(C).            { ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; }
 
 /************************************************ create/drop topic ***************************************************/
-cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C).      { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_or_subquery(C).     { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); }
 cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C).      { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); }
 cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) 
   WITH META AS DATABASE db_name(C).                                               { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); }
@@ -471,7 +471,7 @@ cmd ::= DESCRIBE full_table_name(A).
 cmd ::= RESET QUERY CACHE.                                                        { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); }
 
 /************************************************ explain *************************************************************/
-cmd ::= EXPLAIN analyze_opt(A) explain_options(B) query_expression(C).            { pCxt->pRootNode = createExplainStmt(pCxt, A, B, C); }
+cmd ::= EXPLAIN analyze_opt(A) explain_options(B) query_or_subquery(C).           { pCxt->pRootNode = createExplainStmt(pCxt, A, B, C); }
 
 %type analyze_opt                                                                 { bool }
 %destructor analyze_opt                                                           { }
@@ -502,7 +502,7 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B).
 
 /************************************************ create/drop stream **************************************************/
 cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A)
-  stream_options(B) INTO full_table_name(C) AS query_expression(D).               { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, D); }
+  stream_options(B) INTO full_table_name(C) AS query_or_subquery(D).              { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, D); }
 cmd ::= DROP STREAM exists_opt(A) stream_name(B).                                 { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); }
 
 stream_options(A) ::= .                                                           { A = createStreamOptions(pCxt); }
@@ -535,12 +535,12 @@ dnode_list(A) ::= dnode_list(B) DNODE NK_INTEGER(C).
 cmd ::= DELETE FROM full_table_name(A) where_clause_opt(B).                       { pCxt->pRootNode = createDeleteStmt(pCxt, A, B); }
 
 /************************************************ select **************************************************************/
-cmd ::= query_expression(A).                                                      { pCxt->pRootNode = A; }
+cmd ::= query_or_subquery(A).                                                     { pCxt->pRootNode = A; }
 
 /************************************************ insert **************************************************************/
 cmd ::= INSERT INTO full_table_name(A) 
-  NK_LP col_name_list(B) NK_RP query_expression(C).                               { pCxt->pRootNode = createInsertStmt(pCxt, A, B, C); }
+  NK_LP col_name_list(B) NK_RP query_or_subquery(C).                              { pCxt->pRootNode = createInsertStmt(pCxt, A, B, C); }
-cmd ::= INSERT INTO full_table_name(A) query_expression(B).                       { pCxt->pRootNode = createInsertStmt(pCxt, A, NULL, B); }
+cmd ::= INSERT INTO full_table_name(A) query_or_subquery(B).                      { pCxt->pRootNode = createInsertStmt(pCxt, A, NULL, B); }
 
 /************************************************ literal *************************************************************/
 literal(A) ::= NK_INTEGER(B).                                                     { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B)); }
@@ -936,28 +936,26 @@ every_opt(A) ::= .
 every_opt(A) ::= EVERY NK_LP duration_literal(B) NK_RP.                           { A = releaseRawExprNode(pCxt, B); }
 
 /************************************************ query_expression ****************************************************/
-query_expression(A) ::= 
+query_expression(A) ::= query_simple(B) 
-  query_expression_body(B) 
   order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E).                {
                                                                                     A = addOrderByClause(pCxt, B, C);
                                                                                     A = addSlimitClause(pCxt, A, D);
                                                                                     A = addLimitClause(pCxt, A, E);
                                                                                   }
 
-query_expression_body(A) ::= query_primary(B).                                    { A = B; }
+query_simple(A) ::= query_specification(B).                                       { A = B; }
-query_expression_body(A) ::=
+query_simple(A) ::= union_query_expression(B).                                    { A = B; }
-  query_expression_body(B) UNION ALL query_expression_body(D).                    { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); }
-query_expression_body(A) ::=
-  query_expression_body(B) UNION query_expression_body(D).                        { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, D); }
 
-query_primary(A) ::= query_specification(B).                                      { A = B; }
+union_query_expression(A) ::=
-query_primary(A) ::=
+  query_simple_or_subquery(B) UNION ALL query_simple_or_subquery(C).              { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, C); }
-  NK_LP query_expression_body(B) 
+union_query_expression(A) ::=
-    order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E) NK_RP.        { 
+  query_simple_or_subquery(B) UNION query_simple_or_subquery(C).                  { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, C); }
-                                                                                    A = addOrderByClause(pCxt, B, C);
+
-                                                                                    A = addSlimitClause(pCxt, A, D);
+query_simple_or_subquery(A) ::= query_simple(B).                                  { A = B; }
-                                                                                    A = addLimitClause(pCxt, A, E);
+query_simple_or_subquery(A) ::= subquery(B).                                      { A = releaseRawExprNode(pCxt, B); }
-                                                                                  }
+
+query_or_subquery(A) ::= query_expression(B).                                     { A = B; }
+query_or_subquery(A) ::= subquery(B).                                             { A = releaseRawExprNode(pCxt, B); }
 
 %type order_by_clause_opt                                                         { SNodeList* }
 %destructor order_by_clause_opt                                                   { nodesDestroyList($$); }

source/libs/parser/src/parAstCreater.c

@@ -1787,10 +1787,10 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
   return (SNode*)pStmt;
 }
 
-SNode* createCountFuncForDelete(SAstCreateContext* pCxt) {
+SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) {
   SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
   CHECK_OUT_OF_MEM(pFunc);
-  strcpy(pFunc->functionName, "count");
+  strcpy(pFunc->functionName, pFuncName);
   if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) {
     nodesDestroyNode((SNode*)pFunc);
     CHECK_OUT_OF_MEM(NULL);
@@ -1804,8 +1804,10 @@ SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) {
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->pFromTable = pTable;
   pStmt->pWhere = pWhere;
-  pStmt->pCountFunc = createCountFuncForDelete(pCxt);
+  pStmt->pCountFunc = createFuncForDelete(pCxt, "count");
-  if (NULL == pStmt->pCountFunc) {
+  pStmt->pFirstFunc = createFuncForDelete(pCxt, "first");
+  pStmt->pLastFunc = createFuncForDelete(pCxt, "last");
+  if (NULL == pStmt->pCountFunc || NULL == pStmt->pFirstFunc || NULL == pStmt->pLastFunc) {
     nodesDestroyNode((SNode*)pStmt);
     CHECK_OUT_OF_MEM(NULL);
   }

source/libs/parser/src/parInsert.c

@@ -1423,9 +1423,7 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa
 }
 
 static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
-  if (!pCxt->pComCxt->async) {
   taosMemoryFreeClear(pCxt->pTableMeta);
-  }
   destroyBoundColumnInfo(&pCxt->tags);
   tdDestroySVCreateTbReq(&pCxt->createTblReq);
 }

source/libs/parser/src/parTranslater.c

@@ -3347,10 +3347,16 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
   if (TSDB_CODE_SUCCESS == code) {
     code = translateDeleteWhere(pCxt, pDelete);
   }
-  if (TSDB_CODE_SUCCESS == code) {
   pCxt->currClause = SQL_CLAUSE_SELECT;
+  if (TSDB_CODE_SUCCESS == code) {
     code = translateExpr(pCxt, &pDelete->pCountFunc);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateExpr(pCxt, &pDelete->pFirstFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateExpr(pCxt, &pDelete->pLastFunc);
+  }
   return code;
 }
 
@@ -5960,12 +5966,6 @@ typedef struct SVgroupCreateTableBatch {
   char               dbName[TSDB_DB_NAME_LEN];
 } SVgroupCreateTableBatch;
 
-static void destroyCreateTbReq(SVCreateTbReq* pReq) {
-  taosMemoryFreeClear(pReq->name);
-  taosMemoryFreeClear(pReq->comment);
-  taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema);
-}
-
 static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo,
                                         SVgroupCreateTableBatch* pBatch) {
   char  dbFName[TSDB_DB_FNAME_LEN] = {0};
@@ -5980,7 +5980,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
   if (pStmt->pOptions->commentNull == false) {
     req.comment = strdup(pStmt->pOptions->comment);
     if (NULL == req.comment) {
-      destroyCreateTbReq(&req);
+      tdDestroySVCreateTbReq(&req);
       return TSDB_CODE_OUT_OF_MEMORY;
     }
     req.commentLen = strlen(pStmt->pOptions->comment);
@@ -5991,7 +5991,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
   req.ntb.schemaRow.version = 1;
   req.ntb.schemaRow.pSchema = taosMemoryCalloc(req.ntb.schemaRow.nCols, sizeof(SSchema));
   if (NULL == req.name || NULL == req.ntb.schemaRow.pSchema) {
-    destroyCreateTbReq(&req);
+    tdDestroySVCreateTbReq(&req);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   if (pStmt->ignoreExists) {
@@ -6007,7 +6007,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
   strcpy(pBatch->dbName, pStmt->dbName);
   pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));
   if (NULL == pBatch->req.pArray) {
-    destroyCreateTbReq(&req);
+    tdDestroySVCreateTbReq(&req);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   taosArrayPush(pBatch->req.pArray, &req);
@@ -6052,16 +6052,7 @@ static void destroyCreateTbReqBatch(void* data) {
   size_t                   size = taosArrayGetSize(pTbBatch->req.pArray);
   for (int32_t i = 0; i < size; ++i) {
     SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
-    taosMemoryFreeClear(pTableReq->name);
+    tdDestroySVCreateTbReq(pTableReq);
-    taosMemoryFreeClear(pTableReq->comment);
-
-    if (pTableReq->type == TSDB_NORMAL_TABLE) {
-      taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema);
-    } else if (pTableReq->type == TSDB_CHILD_TABLE) {
-      taosMemoryFreeClear(pTableReq->ctb.pTag);
-      taosMemoryFreeClear(pTableReq->ctb.name);
-      taosArrayDestroy(pTableReq->ctb.tagName);
-    }
   }
 
   taosArrayDestroy(pTbBatch->req.pArray);
@@ -6422,6 +6413,8 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
   if (TSDB_CODE_SUCCESS == code) {
     addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid,
                              pStmt->useTableName, &info, tagName, pSuperTableMeta->tableInfo.numOfTags);
+  } else {
+    taosMemoryFree(pTag);
   }
 
   taosArrayDestroy(tagName);

source/libs/parser/src/parUtil.c

@@ -1124,7 +1124,7 @@ int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* p
   int32_t   reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo);
   SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex);
   if (TSDB_CODE_SUCCESS == pRes->code) {
-    *pMeta = pRes->pRes;
+    *pMeta = tableMetaDup(pRes->pRes);
     if (NULL == *pMeta) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }

source/libs/planner/src/planLogicCreater.c

@@ -1372,9 +1372,21 @@ static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pD
   }
 
   int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc));
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pFirstFunc));
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pLastFunc));
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pFirstFunc);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pLastFunc);
+  }
   // set the output
   if (TSDB_CODE_SUCCESS == code) {
     code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets);
@@ -1405,7 +1417,9 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet
   strcpy(pModify->tsColName, pRealTable->pMeta->schema->name);
   pModify->deleteTimeRange = pDelete->timeRange;
   pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc);
-  if (NULL == pModify->pAffectedRows) {
+  pModify->pStartTs = nodesCloneNode(pDelete->pFirstFunc);
+  pModify->pEndTs = nodesCloneNode(pDelete->pLastFunc);
+  if (NULL == pModify->pAffectedRows || NULL == pModify->pStartTs || NULL == pModify->pEndTs) {
     nodesDestroyNode((SNode*)pModify);
     return TSDB_CODE_OUT_OF_MEMORY;
   }

source/libs/planner/src/planPhysiCreater.c

@@ -1323,8 +1323,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
 
 static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                         SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
-  SPartitionPhysiNode* pPart =
+  SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode(
-      (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode,
+      pCxt, (SLogicNode*)pPartLogicNode,
       pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION);
   if (NULL == pPart) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -1670,6 +1670,12 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode*
 
   int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows,
                                &pDeleter->pAffectedRows);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pStartTs, &pDeleter->pStartTs);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pEndTs, &pDeleter->pEndTs);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     pDeleter->sink.pInputDataBlockDesc = (SDataBlockDescNode*)nodesCloneNode((SNode*)pRoot->pOutputDataBlockDesc);
     if (NULL == pDeleter->sink.pInputDataBlockDesc) {

source/libs/planner/test/planSetOpTest.cpp

@@ -44,6 +44,8 @@ TEST_F(PlanSetOpTest, unionAllWithSubquery) {
   run("SELECT ts FROM (SELECT ts FROM st1s1) UNION ALL SELECT ts FROM (SELECT ts FROM st1s2)");
   // super table
   run("SELECT ts FROM (SELECT ts FROM st1) UNION ALL SELECT ts FROM (SELECT ts FROM st1)");
+
+  run("(SELECT SERVER_STATUS()) UNION ALL (SELECT SERVER_STATUS())");
 }
 
 TEST_F(PlanSetOpTest, unionAllWithOrderBy) {

source/libs/qworker/src/qwUtil.c

@@ -275,6 +275,7 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) {
   qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
   if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
     qDestroyTask(otaskHandle);
+    qDebug("task handle destryed");
   }
 }
 
@@ -305,6 +306,7 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) {
   if (ctx->sinkHandle) {
     dsDestroyDataSinker(ctx->sinkHandle);
     ctx->sinkHandle = NULL;
+    qDebug("sink handle destryed");
   }
 }
 
@@ -452,6 +454,10 @@ void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksH
 
 void qwDestroyImpl(void *pMgmt) {
   SQWorker *mgmt = (SQWorker *)pMgmt;
+  int8_t nodeType = mgmt->nodeType;
+  int32_t nodeId = mgmt->nodeId;
+  
+  qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);  
 
   taosTmrStop(mgmt->hbTimer);
   mgmt->hbTimer = NULL;
@@ -484,6 +490,8 @@ void qwDestroyImpl(void *pMgmt) {
   atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
 
   qwCloseRef();
+
+  qDebug("qworker destroyed, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);  
 }
 
 int32_t qwOpenRef(void) {

source/libs/qworker/src/qworker.c

@@ -9,6 +9,7 @@
 #include "tcommon.h"
 #include "tmsg.h"
 #include "tname.h"
+#include "tdatablock.h"
 
 SQWorkerMgmt gQwMgmt = {
     .lock = 0,
@@ -16,6 +17,11 @@ SQWorkerMgmt gQwMgmt = {
     .qwNum = 0,
 };
 
+static void freeBlock(void* param) {
+  SSDataBlock* pBlock = *(SSDataBlock**)param;
+  blockDataDestroy(pBlock);
+}
+
 int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
   int32_t         code = 0;
   SSchedulerHbRsp rsp = {0};
@@ -88,6 +94,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
     // if *taskHandle is NULL, it's killed right now
     if (taskHandle) {
       qwDbgSimulateSleep();
+
       code = qExecTaskOpt(taskHandle, pResList, &useconds);
       if (code) {
         if (code != TSDB_CODE_OPS_NOT_SUPPORT) {
@@ -150,8 +157,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
   }
 
 _return:
-
+  taosArrayDestroyEx(pResList, freeBlock);
-  taosArrayDestroy(pResList);
   QW_RET(code);
 }
 
@@ -954,13 +960,13 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
 
   void *pIter = taosHashIterate(mgmt->schHash, NULL);
   while (pIter) {
-    SQWSchStatus *sch = (SQWSchStatus *)pIter;
+    SQWSchStatus *sch1 = (SQWSchStatus *)pIter;
-    if (NULL == sch->hbConnInfo.handle) {
+    if (NULL == sch1->hbConnInfo.handle) {
       uint64_t *sId = taosHashGetKey(pIter, NULL);
       QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId);
 
-      if (sch->hbBrokenTs > 0 && ((currentMs - sch->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
+      if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
-          taosHashGetSize(sch->tasksHash) <= 0) {
+          taosHashGetSize(sch1->tasksHash) <= 0) {
         taosArrayPush(pExpiredSch, sId);
       }
 

source/libs/scheduler/inc/schInt.h

@@ -285,6 +285,7 @@ typedef struct SSchJob {
 typedef struct SSchTaskCtx {
   int64_t   jobRid;
   SSchTask *pTask;
+  bool      asyncLaunch;
 } SSchTaskCtx;
 
 extern SSchedulerMgmt schMgmt;

source/libs/scheduler/src/schRemote.c

@@ -385,7 +385,6 @@ _return:
 
   SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
 } 
-
 int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
   int32_t                code = 0;
   SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;

source/libs/scheduler/src/schTask.c

@@ -138,12 +138,6 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3
     return TSDB_CODE_SUCCESS;
   }
 
-  if ((execId != pTask->execId) || pTask->waitRetry) {  // ignore it
-    SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId,
-                  pTask->execId, pTask->waitRetry);
-    return TSDB_CODE_SUCCESS;
-  }
-
   SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId));
   if (NULL == nodeInfo) {  // ignore it
     SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId,
@@ -163,10 +157,15 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v
     SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId));
   }
   
-  SCH_SET_TASK_HANDLE(pTask, handle);
-
   schUpdateTaskExecNode(pJob, pTask, handle, execId);
 
+  if ((execId != pTask->execId) || pTask->waitRetry) {  // ignore it
+    SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+    SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
+  }
+  
+  SCH_SET_TASK_HANDLE(pTask, handle);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -352,7 +351,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
   pTask->waitRetry = true;
   schDropTaskOnExecNode(pJob, pTask);
   taosHashClear(pTask->execNodes);
-  SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask));
+  schRemoveTaskFromExecList(pJob, pTask);
   schDeregisterTaskHb(pJob, pTask);
   atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
   taosMemoryFreeClear(pTask->msg);
@@ -430,12 +429,14 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
 
   code = schDoTaskRedirect(pJob, pTask, pData, rspCode);
   taosMemoryFree(pData->pData);
+  taosMemoryFree(pData->pEpSet);
 
   SCH_RET(code);
 
 _return:
 
   taosMemoryFree(pData->pData);
+  taosMemoryFree(pData->pEpSet);
 
   SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
 }
@@ -597,7 +598,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
 int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
   atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
 
-  SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask));
+  schRemoveTaskFromExecList(pJob, pTask);
   SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
 
   if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
@@ -744,8 +745,7 @@ _return:
 int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
   int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));
   if (code) {
-    SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code);
+    SCH_TASK_WLOG("task already not in execTask list, code:%x", code);
-    SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -834,6 +834,11 @@ int32_t schLaunchTaskImpl(void *param) {
   }
 
   SSchTask *pTask = pCtx->pTask;
+
+  if (pCtx->asyncLaunch) {
+    SCH_LOCK_TASK(pTask);
+  }
+  
   int8_t    status = 0;
   int32_t   code = 0;
 
@@ -882,8 +887,6 @@ int32_t schLaunchTaskImpl(void *param) {
 
 _return:
 
-  taosMemoryFree(param);
-
   if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
     if (code) {
       code = schProcessOnTaskFailure(pJob, pTask, code);
@@ -893,8 +896,14 @@ _return:
     }
   }
 
+  if (pCtx->asyncLaunch) {
+    SCH_UNLOCK_TASK(pTask);
+  }
+
   schReleaseJob(pJob->refId);
 
+  taosMemoryFree(param);
+
   SCH_RET(code);
 }
 
@@ -908,6 +917,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
   param->pTask = pTask;
 
   if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
+    param->asyncLaunch = true;
     taosAsyncExec(schLaunchTaskImpl, param, NULL);
   } else {
     SCH_ERR_RET(schLaunchTaskImpl(param));

source/libs/stream/src/streamState.c

@@ -35,6 +35,10 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) {
     goto _err;
   }
 
+  if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) {
+    goto _err;
+  }
+
   if (streamStateBegin(pState) < 0) {
     goto _err;
   }
@@ -44,8 +48,9 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) {
   return pState;
 
 _err:
-  if (pState->pStateDb) tdbTbClose(pState->pStateDb);
+  tdbTbClose(pState->pStateDb);
-  if (pState->db) tdbClose(pState->db);
+  tdbTbClose(pState->pFuncStateDb);
+  tdbClose(pState->db);
   taosMemoryFree(pState);
   return NULL;
 }
@@ -53,6 +58,7 @@ _err:
 void streamStateClose(SStreamState* pState) {
   tdbCommit(pState->db, &pState->txn);
   tdbTbClose(pState->pStateDb);
+  tdbTbClose(pState->pFuncStateDb);
   tdbClose(pState->db);
 
   taosMemoryFree(pState);
@@ -101,6 +107,17 @@ int32_t streamStateAbort(SStreamState* pState) {
   return 0;
 }
 
+int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) {
+  return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->txn);
+}
+int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) {
+  return tdbTbGet(pState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen);
+}
+
+int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) {
+  return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn);
+}
+
 int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
   return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn);
 }

source/libs/tdb/src/db/tdbBtree.c

@@ -489,7 +489,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
   }
 
   // Copy the root page content to the child page
-  tdbPageCopy(pRoot, pChild);
+  tdbPageCopy(pRoot, pChild, 0);
 
   // Reinitialize the root page
   zArg.flags = TDB_BTREE_ROOT;
@@ -742,7 +742,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
     for (int i = 0; i < nOlds; i++) {
       tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL);
       tdbBtreeInitPage(pOldsCopy[i], &iarg, 0);
-      tdbPageCopy(pOlds[i], pOldsCopy[i]);
+      tdbPageCopy(pOlds[i], pOldsCopy[i], 0);
     }
     iNew = 0;
     nNewCells = 0;
@@ -840,7 +840,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
     i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]);
     // copy content to the parent page
     tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0);
-    tdbPageCopy(pNews[0], pParent);
+    tdbPageCopy(pNews[0], pParent, 1);
   }
 
   for (int i = 0; i < 3; i++) {

source/libs/tdb/src/db/tdbPage.c

@@ -229,7 +229,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
   return 0;
 }
 
-void tdbPageCopy(SPage *pFromPage, SPage *pToPage) {
+void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
   int delta, nFree;
 
   pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr);
@@ -250,8 +250,15 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) {
 
   // Copy the overflow cells
   for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) {
+    SCell *pNewCell = pFromPage->apOvfl[iOvfl];
+    if (deepCopyOvfl) {
+      int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
+      pNewCell = (SCell *)tdbOsMalloc(szCell);
+      memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
+    }
+
+    pToPage->apOvfl[iOvfl] = pNewCell;
     pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl];
-    pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl];
   }
   pToPage->nOverflow = pFromPage->nOverflow;
 }

source/libs/tdb/src/inc/tdbInt.h

@@ -333,7 +333,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
 int  tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
 int  tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
 int  tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
-void tdbPageCopy(SPage *pFromPage, SPage *pToPage);
+void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells);
 int  tdbPageCapacity(int pageSize, int amHdrSize);
 
 static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {

source/libs/tfs/src/tfs.c

@@ -113,6 +113,8 @@ SDiskSize tfsGetSize(STfs *pTfs) {
   return size;
 }
 
+int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; }
+
 int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
   pDiskId->level = expLevel;
   pDiskId->id = -1;

source/libs/transport/src/transCli.c

@@ -1498,9 +1498,7 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran
   pCtx->ahandle = pReq->info.ahandle;
   pCtx->msgType = pReq->msgType;
 
-  if (ctx != NULL) {
+  if (ctx != NULL) pCtx->appCtx = *ctx;
-    pCtx->appCtx = *ctx;
-  }
 
   SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg));
   cliMsg->ctx = pCtx;

source/libs/transport/src/transSvr.c

@@ -1148,6 +1148,7 @@ int transReleaseSrvHandle(void* handle) {
 
   tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle);
   transAsyncSend(pThrd->asyncPool, &m->q);
+
   transReleaseExHandle(transGetRefMgt(), refId);
   return 0;
 _return1:
@@ -1177,8 +1178,10 @@ int transSendResponse(const STransMsg* msg) {
   STraceId* trace = (STraceId*)&msg->info.traceId;
   tGTrace("conn %p start to send resp (1/2)", exh->handle);
   transAsyncSend(pThrd->asyncPool, &m->q);
+
   transReleaseExHandle(transGetRefMgt(), refId);
   return 0;
+
 _return1:
   tTrace("handle %p failed to send resp", exh);
   rpcFreeCont(msg->pCont);
@@ -1207,6 +1210,7 @@ int transRegisterMsg(const STransMsg* msg) {
   STrans* pTransInst = pThrd->pTransInst;
   tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle);
   transAsyncSend(pThrd->asyncPool, &m->q);
+
   transReleaseExHandle(transGetRefMgt(), refId);
   return 0;
 

source/os/src/osDir.c

@@ -31,6 +31,15 @@ typedef struct TdDir {
   HANDLE     hFind;
 } TdDir;
 
+enum
+  {
+    WRDE_NOSPACE = 1,		/* Ran out of memory.  */
+    WRDE_BADCHAR,		/* A metachar appears in the wrong place.  */
+    WRDE_BADVAL,		/* Undefined var reference with WRDE_UNDEF.  */
+    WRDE_CMDSUB,		/* Command substitution with WRDE_NOCMD.  */
+    WRDE_SYNTAX			/* Shell syntax error.  */
+  };
+
 int wordexp(char *words, wordexp_t *pwordexp, int flags) {
   pwordexp->we_offs = 0;
   pwordexp->we_wordc = 1;
@@ -265,9 +274,21 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
 
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
   wordexp_t full_path;
-  if (0 != wordexp(dirname, &full_path, 0)) {
+  switch (wordexp (dirname, &full_path, 0)) {
-    printf("failed to expand path:%s since %s", dirname, strerror(errno));
+  case 0:
+    break;
+  case WRDE_NOSPACE:
     wordfree (&full_path);
+    // printf("failed to expand path:%s since Out of memory\n", dirname);
+    return -1;
+  case WRDE_BADCHAR:
+    // printf("failed to expand path:%s since illegal occurrence of newline or one of |, &, ;, <, >, (, ), {, }\n", dirname);
+    return -1;
+  case WRDE_SYNTAX:
+    // printf("failed to expand path:%s since Shell syntax error, such as unbalanced parentheses or unmatched quotes\n", dirname);
+    return -1;
+  default:
+    // printf("failed to expand path:%s since %s\n", dirname, strerror(errno));
     return -1;
   }
 

source/os/src/osSemaphore.c

@@ -400,6 +400,9 @@ int tsem_init(tsem_t *psem, int flags, unsigned int count) {
 }
 
 int tsem_destroy(tsem_t *psem) {
+  if (psem == NULL || *psem == NULL) return -1;
+  dispatch_release(*psem);
+  *psem = NULL;
   return 0;
 }
 
@@ -421,13 +424,7 @@ int tsem_timewait(tsem_t *psem, int64_t nanosecs) {
   return 0;
 }
 
-bool taosCheckPthreadValid(TdThread thread) {
+bool taosCheckPthreadValid(TdThread thread) { return thread != 0; }
-  int32_t ret = taosThreadKill(thread, 0);
-  if (ret == ESRCH) return false;
-  if (ret == EINVAL) return false;
-  // alive
-  return true;
-}
 
 int64_t taosGetSelfPthreadId() {
   TdThread thread = taosThreadSelf();

source/os/src/osSysinfo.c

@@ -344,30 +344,27 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
   *numOfCores = si.dwNumberOfProcessors;
   return 0;
 #elif defined(_TD_DARWIN_64)
-  char   *line = NULL;
+  char    buf[16];
-  size_t  size = 0;
   int32_t done = 0;
   int32_t code = -1;
 
-  TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
+  TdCmdPtr pCmd = taosOpenCmd("sysctl -n machdep.cpu.brand_string");
-  if (pFile == NULL) return false;
+  if (pCmd == NULL) return code;
-
+  if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
-  while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
-    line[size - 1] = '\0';
-    if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
-      const char *v = strchr(line, ':') + 2;
-      tstrncpy(cpuModel, v, maxLen);
     code = 0;
     done |= 1;
-    } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
-      const char *v = strchr(line, ':') + 2;
-      *numOfCores = atof(v);
-      done |= 2;
-    }
   }
+  taosCloseCmd(&pCmd);
 
-  if (line != NULL) taosMemoryFree(line);
+  pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count");
-  taosCloseFile(&pFile);
+  if (pCmd == NULL) return code;
+  memset(buf, 0, sizeof(buf));
+  if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
+    code = 0;
+    done |= 2;
+    *numOfCores = atof(buf);
+  }
+  taosCloseCmd(&pCmd);
 
   return code;
 #else

source/util/src/tarray.c

@@ -386,12 +386,13 @@ void* taosArrayDestroy(SArray* pArray) {
 }
 
 void taosArrayDestroyP(SArray* pArray, FDelete fp) {
-  if(!pArray) return;
+  if (pArray) {
     for (int32_t i = 0; i < pArray->size; i++) {
       fp(*(void**)TARRAY_GET_ELEM(pArray, i));
     }
     taosArrayDestroy(pArray);
   }
+}
 
 void taosArrayDestroyEx(SArray* pArray, FDelete fp) {
   if (pArray == NULL) {

source/util/src/terror.c

@@ -288,6 +288,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY,       "Consumer not ready")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED,         "Topic subscribed cannot be dropped")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED,    "Topic must be dropped first")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED,              "Consumer group being used by some consumer")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_IN_REBALANCE,             "Topic being rebalanced")
 
 // mnode-stream
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST,     "Stream already exists")
@@ -579,6 +580,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_NUM,         "Invalid function par
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE,        "Invalid function para type")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE,       "Invalid function para value")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION,      "Not buildin function")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP,            "Duplicate timestamps not allowed in function")
 
 //udf
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                   "udf is stopping")

source/util/test/trefTest.c

@@ -94,7 +94,7 @@ void *openRefSpace(void *param) {
   pSpace->rsetId = taosOpenRef(50, myfree);
 
   if (pSpace->rsetId < 0) {
-    printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
+    printf("failed to open ref, reason:%s\n", tstrerror(pSpace->rsetId));
     return NULL;
   }