Merge branch 'main' into fix/TD-23161

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG ad1a32b
+        GIT_TAG d11f210
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

include/util/taoserror.h

@@ -688,6 +688,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_NOT_SUPPORT_JOIN          TAOS_DEF_ERROR_CODE(0, 0x2664)
 #define TSDB_CODE_PAR_INVALID_TAGS_PC           TAOS_DEF_ERROR_CODE(0, 0x2665)
 #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY    TAOS_DEF_ERROR_CODE(0, 0x2666)
+#define TSDB_CODE_PAR_INVALID_OPTR_USAGE        TAOS_DEF_ERROR_CODE(0, 0x2667)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner

source/client/inc/clientInt.h

@@ -287,22 +287,25 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
 }
 
 static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
-  SMqRspObj* msg = (SMqRspObj*)res;
+  SMqRspObj* pRspObj = (SMqRspObj*)res;
-  msg->resIter++;
+  pRspObj->resIter++;
-  if (msg->resIter < msg->rsp.blockNum) {
+
-    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+  if (pRspObj->resIter < pRspObj->rsp.blockNum) {
-    if (msg->rsp.withSchema) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
-      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
+    if (pRspObj->rsp.withSchema) {
-      setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
-      taosMemoryFreeClear(msg->resInfo.row);
+      setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
-      taosMemoryFreeClear(msg->resInfo.pCol);
+      taosMemoryFreeClear(pRspObj->resInfo.row);
-      taosMemoryFreeClear(msg->resInfo.length);
+      taosMemoryFreeClear(pRspObj->resInfo.pCol);
-      taosMemoryFreeClear(msg->resInfo.convertBuf);
+      taosMemoryFreeClear(pRspObj->resInfo.length);
-      taosMemoryFreeClear(msg->resInfo.convertJson);
+      taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
+      taosMemoryFreeClear(pRspObj->resInfo.convertJson);
     }
-    setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false);
+
-    return &msg->resInfo;
+    setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false);
+    return &pRspObj->resInfo;
   }
+
   return NULL;
 }
 

source/client/src/clientJniConnector.c

@@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
     return JNI_RESULT_SET_NULL;
   }
 
-  void   *data;
+  void   *data = NULL;
-  int32_t numOfRows;
+  int32_t numOfRows = 0;
   int     error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
   if (numOfRows == 0) {
     if (error_code == JNI_SUCCESS) {

source/client/src/clientMain.c

@@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
 }
 
 int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
+  *numOfRows = 0;
+  *pData = NULL;
+
   if (res == NULL || TD_RES_TMQ_META(res)) {
     return 0;
   }

source/client/src/clientRawBlockWrite.c

@@ -776,12 +776,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
 
+  SCatalog*      pCatalog = NULL;
+  code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+  SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
+  SName       pName = {0};
+  toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
+  STableMeta *pTableMeta = NULL;
+  code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
+  if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
+    code = TSDB_CODE_SUCCESS;
+    taosMemoryFreeClear(pTableMeta);
+    goto end;
+  }
+  if (code != TSDB_CODE_SUCCESS) {
+    uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name);
+    goto end;
+  }
+  pReq.suid = pTableMeta->uid;
+  taosMemoryFreeClear(pTableMeta);
+
   // build drop stable
   pReq.igNotExists = true;
   pReq.source = TD_REQ_FROM_TAOX;
-  pReq.suid = processSuid(req.suid, pRequest->pDb);
+//  pReq.suid = processSuid(req.suid, pRequest->pDb);
 
-  uDebug("taosDropStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
+  uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
   STscObj* pTscObj = pRequest->pTscObj;
   SName    tableName = {0};
   tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@@ -806,7 +831,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
   launchQueryImpl(pRequest, &pQuery, true, NULL);
 
   if (pRequest->code == TSDB_CODE_SUCCESS) {
-    SCatalog* pCatalog = NULL;
+//    SCatalog* pCatalog = NULL;
     catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
     catalogRemoveTableMeta(pCatalog, &tableName);
   }
@@ -900,15 +925,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
       STableMeta* pTableMeta = NULL;
       SName       sName = {0};
       tb_uid_t oldSuid = pCreateReq->ctb.suid;
-      pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
+//      pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
-      uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" processSuid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, pCreateReq->ctb.suid, oldSuid);
-
       toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
       code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
       if (code != TSDB_CODE_SUCCESS) {
         uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName);
         goto end;
       }
+      pCreateReq->ctb.suid = pTableMeta->uid;
+      uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
 
       for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
         char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
@@ -1041,8 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
   for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
     pDropReq = req.pReqs + iReq;
     pDropReq->igNotExists = true;
-    pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
+//    pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
-    uDebug("taosDropTable name:%s suid:%"PRId64" processSuid:%"PRId64, pDropReq->name, pDropReq->suid, pDropReq->suid);
 
     SVgroupInfo pInfo = {0};
     SName       pName = {0};
@@ -1052,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
       goto end;
     }
 
+    STableMeta *pTableMeta = NULL;
+    code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
+    if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
+      code = TSDB_CODE_SUCCESS;
+      taosMemoryFreeClear(pTableMeta);
+      continue;
+    }
+    if (code != TSDB_CODE_SUCCESS) {
+      uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name);
+      goto end;
+    }
+    tb_uid_t oldSuid = pDropReq->suid;
+    pDropReq->suid = pTableMeta->suid;
+    taosMemoryFreeClear(pTableMeta);
+    uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
+
     taosArrayPush(pRequest->tableList, &pName);
     SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
     if (pTableBatch == NULL) {
@@ -1066,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
     }
   }
 
+  if (taosHashGetSize(pVgroupHashmap) == 0){
+    goto end;
+  }
   SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
   if (NULL == pBufArray) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1673,7 +1716,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
       }
       if (strcmp(tbName, pCreateReq.name) == 0) {
         cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
-        pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
+//        pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
         tDecoderClear(&decoderTmp);
         break;
       }
@@ -1706,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     if (pCreateReqDst) {
       pTableMeta->vgId = vg.vgId;
       pTableMeta->uid = pCreateReqDst->uid;
+      pCreateReqDst->ctb.suid = pTableMeta->suid;
     }
     void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
     if (hData == NULL) {

source/client/src/clientTmq.c

@@ -24,6 +24,9 @@
 #include "tref.h"
 #include "ttimer.h"
 
+#define EMPTY_BLOCK_POLL_IDLE_DURATION  100
+#define DEFAULT_AUTO_COMMIT_INTERVAL    5000
+
 struct SMqMgmt {
   int8_t  inited;
   tmr_h   timer;
@@ -68,18 +71,16 @@ struct tmq_conf_t {
 };
 
 struct tmq_t {
-  int64_t refId;
+  int64_t        refId;
-  // conf
+  char           groupId[TSDB_CGROUP_LEN];
-  char     groupId[TSDB_CGROUP_LEN];
+  char           clientId[256];
-  char     clientId[256];
+  int8_t         withTbName;
-  int8_t   withTbName;
+  int8_t         useSnapshot;
-  int8_t   useSnapshot;
+  int8_t         autoCommit;
-  int8_t   autoCommit;
+  int32_t        autoCommitInterval;
-  int32_t  autoCommitInterval;
+  int32_t        resetOffsetCfg;
-  int32_t  resetOffsetCfg;
+  uint64_t       consumerId;
-  uint64_t consumerId;
+  bool           hbBgEnable;
-  bool     hbBgEnable;
-
   tmq_commit_cb* commitCb;
   void*          commitCbUserParam;
 
@@ -90,14 +91,15 @@ struct tmq_t {
   int8_t  epStatus;
   int32_t epSkipCnt;
 #endif
-  int64_t pollCnt;
+  // poll info
+  int64_t       pollCnt;
+  int64_t       totalRows;
 
   // timer
-  tmr_h hbLiveTimer;
+  tmr_h         hbLiveTimer;
-  tmr_h epTimer;
+  tmr_h         epTimer;
-  tmr_h reportTimer;
+  tmr_h         reportTimer;
-  tmr_h commitTimer;
+  tmr_h         commitTimer;
-
   STscObj*      pTscObj;       // connection
   SArray*       clientTopics;  // SArray<SMqClientTopic>
   STaosQueue*   mqueue;        // queue of rsp
@@ -127,6 +129,7 @@ enum {
 
 typedef struct {
   int64_t      pollCnt;
+  int64_t      numOfRows;
   STqOffsetVal committedOffset;
   STqOffsetVal currentOffset;
   int32_t      vgId;
@@ -149,6 +152,7 @@ typedef struct {
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
   uint64_t        reqId;
+  SEpSet*         pEpset;
   union {
     SMqDataRsp dataRsp;
     SMqMetaRsp metaRsp;
@@ -220,7 +224,7 @@ tmq_conf_t* tmq_conf_new() {
 
   conf->withTbName = false;
   conf->autoCommit = true;
-  conf->autoCommitInterval = 5000;
+  conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL;
   conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST;
   conf->hbBgEnable = true;
 
@@ -266,7 +270,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
   }
 
   if (strcasecmp(key, "auto.commit.interval.ms") == 0) {
-    conf->autoCommitInterval = atoi(value);
+    conf->autoCommitInterval = taosStr2int64(value);
     return TMQ_CONF_OK;
   }
 
@@ -310,7 +314,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
   }
 
   if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) {
-    conf->snapBatchSize = atoi(value);
+    conf->snapBatchSize = taosStr2int64(value);
     return TMQ_CONF_OK;
   }
 
@@ -330,18 +334,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     conf->ip = taosStrdup(value);
     return TMQ_CONF_OK;
   }
+
   if (strcasecmp(key, "td.connect.user") == 0) {
     conf->user = taosStrdup(value);
     return TMQ_CONF_OK;
   }
+
   if (strcasecmp(key, "td.connect.pass") == 0) {
     conf->pass = taosStrdup(value);
     return TMQ_CONF_OK;
   }
+
   if (strcasecmp(key, "td.connect.port") == 0) {
-    conf->port = atoi(value);
+    conf->port = taosStr2int64(value);
     return TMQ_CONF_OK;
   }
+
   if (strcasecmp(key, "td.connect.db") == 0) {
     return TMQ_CONF_OK;
   }
@@ -403,65 +411,42 @@ static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t
   return NULL;
 }
 
+// Two problems do not need to be addressed here
+// 1. update to of epset. the response of poll request will automatically handle this problem
+// 2. commit failure. This one needs to be resolved.
 static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
   SMqCommitCbParam*    pParam = (SMqCommitCbParam*)param;
   SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params;
 
-  if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
+//  if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
-    taosThreadMutexLock(&pParam->pTmq->lock);
+//    taosThreadMutexLock(&pParam->pTmq->lock);
-    int32_t numOfVgroups, index;
+//    int32_t numOfVgroups, index;
-    SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
+//    SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
-
+//    if (pVg == NULL) {
-    if (pVg == NULL) {
+//      tscDebug("consumer:0x%" PRIx64
-      tscDebug("consumer:0x%" PRIx64
+//               " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d",
-               " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d",
+//               pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups);
-               pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups);
+//    } else { // let's retry the commit
-    } else { // let's retry the commit
+//      int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
-      int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
+//      if (code1 != TSDB_CODE_SUCCESS) {  // retry failed.
-      if (code1 != TSDB_CODE_SUCCESS) {  // retry failed.
+//        tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
-        tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
+//                 " retry failed, ignore this commit. code:%s ordinal:%d/%d",
-                 " retry failed, ignore this commit. code:%s ordinal:%d/%d",
+//                 pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
-                 pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
+//                 tstrerror(terrno), index + 1, numOfVgroups);
-                 tstrerror(terrno), index + 1, numOfVgroups);
+//      }
-      }
+//    }
-    }
+//
-
+//    taosThreadMutexUnlock(&pParam->pTmq->lock);
-    taosThreadMutexUnlock(&pParam->pTmq->lock);
+//
-
+//    taosMemoryFree(pParam->pOffset);
-    taosMemoryFree(pParam->pOffset);
+//    taosMemoryFree(pBuf->pData);
-    taosMemoryFree(pBuf->pData);
+//    taosMemoryFree(pBuf->pEpSet);
-    taosMemoryFree(pBuf->pEpSet);
+//
-
+//    tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
-    tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
+//    return 0;
-    return 0;
+//  }
-  }
+//
-
+//  // todo replace the pTmq with refId
-  // todo replace the pTmq with refId
-  taosThreadMutexLock(&pParam->pTmq->lock);
-  tmq_t* pTmq = pParam->pTmq;
-  int32_t index = 0, numOfVgroups = 0;
-
-  SMqClientVg* pVg = foundClientVg(pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
-  if (pVg == NULL) {
-    tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d has been transferred to other consumer, ordinal:%d/%d",
-             pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups);
-  } else { // update the epset if needed
-    if (pBuf->pEpSet != NULL) {
-      SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet);
-      SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet));
-
-      tscDebug("consumer:0x%" PRIx64 " subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d, ordinal:%d/%d",
-               pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port,
-               index + 1, numOfVgroups);
-
-      pVg->epSet = *pBuf->pEpSet;
-    }
-
-    tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d, commit offset success. ordinal:%d/%d", pTmq->consumerId,
-             pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups);
-  }
-
-  taosThreadMutexUnlock(&pParam->pTmq->lock);
 
   taosMemoryFree(pParam->pOffset);
   taosMemoryFree(pBuf->pData);
@@ -486,8 +471,8 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
   pOffset->subKey[groupLen] = TMQ_SEPARATOR;
   strcpy(pOffset->subKey + groupLen + 1, pTopicName);
 
-  int32_t len;
+  int32_t len = 0;
-  int32_t code;
+  int32_t code = 0;
   tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
   if (code < 0) {
     return -1;
@@ -647,8 +632,7 @@ FAIL:
   return 0;
 }
 
-static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
+static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) {
-                                     void* userParam) {
   int32_t code = -1;
 
   SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
@@ -740,31 +724,30 @@ static int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic,
   if (msg) { // user invoked commit
     return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam);
   } else {  // this for auto commit
-    return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam);
+    return doAutoCommit(tmq, automatic, async, userCb, userParam);
   }
 }
 
+static void generateTimedTask(int64_t refId, int32_t type) {
+  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+  if (tmq != NULL) {
+    int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
+    *pTaskType = type;
+    taosWriteQitem(tmq->delayedTask, pTaskType);
+    tsem_post(&tmq->rspSem);
+  }
+  taosReleaseRef(tmqMgmt.rsetId, refId);
+}
+
 void tmqAssignAskEpTask(void* param, void* tmrId) {
   int64_t refId = *(int64_t*)param;
-  tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+  generateTimedTask(refId, TMQ_DELAYED_TASK__ASK_EP);
-  if (tmq != NULL) {
-    int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
-    *pTaskType = TMQ_DELAYED_TASK__ASK_EP;
-    taosWriteQitem(tmq->delayedTask, pTaskType);
-    tsem_post(&tmq->rspSem);
-  }
   taosMemoryFree(param);
 }
 
 void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
   int64_t refId = *(int64_t*)param;
-  tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+  generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT);
-  if (tmq != NULL) {
-    int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
-    *pTaskType = TMQ_DELAYED_TASK__COMMIT;
-    taosWriteQitem(tmq->delayedTask, pTaskType);
-    tsem_post(&tmq->rspSem);
-  }
   taosMemoryFree(param);
 }
 
@@ -777,6 +760,8 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) {
     taosWriteQitem(tmq->delayedTask, pTaskType);
     tsem_post(&tmq->rspSem);
   }
+
+  taosReleaseRef(tmqMgmt.rsetId, refId);
   taosMemoryFree(param);
 }
 
@@ -790,6 +775,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
 
 void tmqSendHbReq(void* param, void* tmrId) {
   int64_t refId = *(int64_t*)param;
+
   tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
   if (tmq == NULL) {
     taosMemoryFree(param);
@@ -803,17 +789,19 @@ void tmqSendHbReq(void* param, void* tmrId) {
   int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req);
   if (tlen < 0) {
     tscError("tSerializeSMqHbReq failed");
-    return;
+    goto OVER;
   }
+
   void* pReq = taosMemoryCalloc(1, tlen);
   if (tlen < 0) {
     tscError("failed to malloc MqHbReq msg, size:%d", tlen);
-    return;
+    goto OVER;
   }
+
   if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) {
     tscError("tSerializeSMqHbReq %d failed", tlen);
     taosMemoryFree(pReq);
-    return;
+    goto OVER;
   }
 
   SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
@@ -821,6 +809,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
     taosMemoryFree(pReq);
     goto OVER;
   }
+
   sendInfo->msgInfo = (SDataBuf){
       .pData = pReq,
       .len = tlen,
@@ -840,6 +829,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
 
 OVER:
   taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer);
+  taosReleaseRef(tmqMgmt.rsetId, refId);
 }
 
 int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
@@ -892,15 +882,21 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
     tDeleteSMqAskEpRsp(&pEpRspWrapper->msg);
   } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
     SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
+    taosMemoryFreeClear(pRsp->pEpset);
+
     taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree);
     taosArrayDestroy(pRsp->dataRsp.blockDataLen);
     taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree);
     taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
   } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
     SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
+    taosMemoryFreeClear(pRsp->pEpset);
+
     taosMemoryFree(pRsp->metaRsp.metaRsp);
   } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) {
     SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
+    taosMemoryFreeClear(pRsp->pEpset);
+
     taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree);
     taosArrayDestroy(pRsp->taosxRsp.blockDataLen);
     taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree);
@@ -974,8 +970,15 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
   return rsp;
 }
 
+static void freeClientVgImpl(void* param) {
+  SMqClientTopic* pTopic = param;
+  taosMemoryFreeClear(pTopic->schema.pSchema);
+  taosArrayDestroy(pTopic->vgs);
+}
+
 void tmqFreeImpl(void* handle) {
-  tmq_t* tmq = (tmq_t*)handle;
+  tmq_t*  tmq = (tmq_t*)handle;
+  int64_t id = tmq->consumerId;
 
   // TODO stop timer
   if (tmq->mqueue) {
@@ -991,16 +994,11 @@ void tmqFreeImpl(void* handle) {
   tsem_destroy(&tmq->rspSem);
   taosThreadMutexDestroy(&tmq->lock);
 
-  int32_t sz = taosArrayGetSize(tmq->clientTopics);
+  taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    taosMemoryFreeClear(pTopic->schema.pSchema);
-    taosArrayDestroy(pTopic->vgs);
-  }
-
-  taosArrayDestroy(tmq->clientTopics);
   taos_close_internal(tmq->pTscObj);
   taosMemoryFree(tmq);
+
+  tscDebug("consumer:0x%" PRIx64 " closed", id);
 }
 
 static void tmqMgmtInit(void) {
@@ -1100,8 +1098,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   char         buf[80] = {0};
   STqOffsetVal offset = {.type = pTmq->resetOffsetCfg};
   tFormatOffset(buf, tListLen(buf), &offset);
-  tscInfo("consumer:0x%" PRIx64 " is setup, groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
+  tscInfo("consumer:0x%" PRIx64 " is setup, refId:%"PRId64", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
-          pTmq->consumerId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf,
+          pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf,
           pTmq->hbBgEnable);
 
   return pTmq;
@@ -1242,10 +1240,12 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para
 
 int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
+
+  int64_t         refId = pParam->refId;
   SMqClientVg*    pVg = pParam->pVg;
   SMqClientTopic* pTopic = pParam->pTopic;
 
-  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
+  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
   if (tmq == NULL) {
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
@@ -1296,6 +1296,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
             tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId);
 
     tsem_post(&tmq->rspSem);
+    taosReleaseRef(tmqMgmt.rsetId, refId);
+
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
     return 0;
@@ -1321,7 +1323,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   pRspWrapper->vgHandle = pVg;
   pRspWrapper->topicHandle = pTopic;
   pRspWrapper->reqId = requestId;
+  pRspWrapper->pEpset = pMsg->pEpSet;
 
+  pMsg->pEpSet = NULL;
   if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
     SDecoder decoder;
     tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
@@ -1350,13 +1354,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   taosMemoryFree(pMsg->pData);
-  taosMemoryFree(pMsg->pEpSet);
   taosWriteQitem(tmq->mqueue, pRspWrapper);
 
   tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64,
            tmq->consumerId, rspType, vgId, tmq->mqueue->numOfItems, requestId);
 
   tsem_post(&tmq->rspSem);
+  taosReleaseRef(tmqMgmt.rsetId, refId);
+
   return 0;
 
 CREATE_MSG_FAIL:
@@ -1365,6 +1370,8 @@ CREATE_MSG_FAIL:
   }
 
   tsem_post(&tmq->rspSem);
+  taosReleaseRef(tmqMgmt.rsetId, refId);
+
   return -1;
 }
 
@@ -1402,6 +1409,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
         .vgStatus = TMQ_VG_STATUS__IDLE,
         .vgSkipCnt = 0,
         .emptyBlockReceiveTs = 0,
+        .numOfRows = 0,
     };
 
     taosArrayPush(pTopic->vgs, &clientVg);
@@ -1516,6 +1524,14 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
   if (head->epoch <= epoch) {
     tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep",
              tmq->consumerId, head->epoch, epoch);
+    if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) {
+      SMqAskEpRsp rsp;
+      tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
+      int8_t flag = (taosArrayGetSize(rsp.topics) == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY;
+      atomic_store_8(&tmq->status, flag);
+      tDeleteSMqAskEpRsp(&rsp);
+    }
+
     goto END;
   }
 
@@ -1545,6 +1561,8 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
 END:
+  taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
+
   if (!async) {
     tsem_post(&pParam->rspSem);
   } else {
@@ -1584,21 +1602,32 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
-SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg) {
   SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
   pRspObj->resType = RES_TYPE__TMQ;
+
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
+
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
   memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
+
   if (!pWrapper->dataRsp.withSchema) {
     setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
   }
 
+  // extract the rows in this data packet
+  for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i);
+    int64_t rows = htobe64(pRetrieve->numOfRows);
+    pRspObj->resInfo.totalRows += rows;
+    pVg->numOfRows += rows;
+  }
+
   return pRspObj;
 }
 
@@ -1702,13 +1731,13 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 
     for (int j = 0; j < numOfVg; j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-      if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < 100) { // less than 100ms
+      if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms
         tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 100ms before start next poll", tmq->consumerId, tmq->epoch,
                  pVg->vgId);
         continue;
       }
 
-      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
+      int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus == TMQ_VG_STATUS__WAIT) {
         int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
         tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch,
@@ -1731,6 +1760,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
     }
   }
 
+  tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId);
   return 0;
 }
 
@@ -1782,31 +1812,44 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
 
       /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
-      if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
+      SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
+
+      if (pDataRsp->head.epoch == consumerEpoch) {
+        // todo fix it: race condition
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
-        pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
+
+        // update the epset
+        if (pollRspWrapper->pEpset != NULL) {
+          SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset);
+          SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet));
+          tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId,
+                   pVg->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port);
+          pVg->epSet = *pollRspWrapper->pEpset;
+        }
+
+        pVg->currentOffset = pDataRsp->rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
 
-        if (pollRspWrapper->dataRsp.blockNum == 0) {
+        char buf[80];
-          tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId,
+        tFormatOffset(buf, 80, &pDataRsp->rspOffset);
-                   pollRspWrapper->reqId);
+        if (pDataRsp->blockNum == 0) {
+          tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, reqId:0x%" PRIx64, tmq->consumerId,
+                   pVg->vgId, buf, pollRspWrapper->reqId);
           taosFreeQitem(pollRspWrapper);
           rspWrapper = NULL;
           continue;
+        } else {  // build rsp
+          SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg);
+          tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%"PRId64" reqId:0x%" PRIx64,
+                   tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pRsp->resInfo.totalRows, pollRspWrapper->reqId);
+
+          tmq->totalRows += pRsp->resInfo.totalRows;
+          taosFreeQitem(pollRspWrapper);
+          return pRsp;
         }
-
-        // build rsp
-        char buf[80];
-        tFormatOffset(buf, 80, &pVg->currentOffset);
-        SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
-        tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId,
-                     pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, pollRspWrapper->reqId);
-
-        taosFreeQitem(pollRspWrapper);
-        return pRsp;
       } else {
         tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
-                 tmq->consumerId, pollRspWrapper->dataRsp.head.epoch, consumerEpoch);
+                 tmq->consumerId, pDataRsp->head.epoch, consumerEpoch);
         tmqFreeRspWrapper(rspWrapper);
         taosFreeQitem(pollRspWrapper);
       }
@@ -1853,12 +1896,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
         // build rsp
         void* pRsp = NULL;
         if (pollRspWrapper->taosxRsp.createTableNum == 0) {
-          pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+          pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg);
         } else {
           pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
         }
 
-
         char buf[80];
         tFormatOffset(buf, 80, &pVg->currentOffset);
         tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId, pVg->vgId,
@@ -1938,7 +1980,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
       return NULL;
     }
 
-    if (timeout != -1) {
+    if (timeout >= 0) {
       int64_t currentTime = taosGetTimestampMs();
       int64_t elapsedTime = currentTime - startTime;
       if (elapsedTime > timeout) {
@@ -1946,9 +1988,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
                  tmq->consumerId, tmq->epoch, startTime, currentTime);
         return NULL;
       }
-      /*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/
-      /*", left time %" PRId64,*/
-      /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/
       tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
     } else {
       // use tsem_timewait instead of tsem_wait to avoid unexpected stuck
@@ -1964,6 +2003,24 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
       return rsp;
     }
 
+    int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
+    tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d",
+             tmq->consumerId, tmq->pollCnt, tmq->totalRows, numOfTopics, tmq->epoch);
+
+    tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", tmq->consumerId);
+    for (int32_t i = 0; i < numOfTopics; ++i) {
+      SMqClientTopic* pTopics = taosArrayGet(tmq->clientTopics, i);
+
+      tscDebug("consumer:0x%" PRIx64 " topic:%d", tmq->consumerId, i);
+      int32_t numOfVgs = taosArrayGetSize(pTopics->vgs);
+      for (int32_t j = 0; j < numOfVgs; ++j) {
+        SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j);
+        tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows);
+      }
+    }
+
+    tscDebug("consumer:0x%" PRIx64 " rows dist end", tmq->consumerId);
+
     int32_t     retryCnt = 0;
     tmq_list_t* lst = tmq_list_new();
     while (1) {
@@ -2166,7 +2223,9 @@ int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) {
 }
 
 int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
-  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParamSet->refId);
+  int64_t refId = pParamSet->refId;
+
+  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
   if (tmq == NULL) {
     if (!pParamSet->async) {
       tsem_destroy(&pParamSet->rspSem);
@@ -2194,6 +2253,8 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
   taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree);
     taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree);
 #endif
+
+  taosReleaseRef(tmqMgmt.rsetId, refId);
   return 0;
 }
 

source/client/src/clientTmqConnector.c

@@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
 
   TAOS_RES *tres = (TAOS_RES *)res;
 
-  void   *data;
+  void   *data = NULL;
-  int32_t numOfRows;
+  int32_t numOfRows = 0;
   int     error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
   if (numOfRows == 0) {
     if (error_code == JNI_SUCCESS) {

source/dnode/mnode/impl/src/mndConsumer.c

@@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {}
 
 bool mndRebTryStart() {
   int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
-  mInfo("tq timer, rebalance counter old val:%d", old);
+  mDebug("tq timer, rebalance counter old val:%d", old);
   return old == 0;
 }
 
@@ -101,7 +101,7 @@ void mndRebCntDec() {
     int32_t newVal = val - 1;
     int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal);
     if (oldVal == val) {
-      mInfo("rebalance trans end, rebalance counter:%d", newVal);
+      mDebug("rebalance trans end, rebalance counter:%d", newVal);
       break;
     }
   }
@@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
   SMqConsumerObj *pConsumer;
   void           *pIter = NULL;
 
-  mTrace("start to process mq timer");
+  mDebug("start to process mq timer");
 
   // rebalance cannot be parallel
   if (!mndRebTryStart()) {
-    mInfo("mq rebalance already in progress, do nothing");
+    mDebug("mq rebalance already in progress, do nothing");
     return 0;
   }
 
@@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
   } else {
     taosHashCleanup(pRebMsg->rebSubHash);
     rpcFreeCont(pRebMsg);
-    mInfo("mq rebalance finished, no modification");
+    mDebug("mq rebalance finished, no modification");
     mndRebEnd();
   }
   return 0;

source/dnode/mnode/impl/src/mndDb.c

@@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
   pOld->cfg.cacheLast = pNew->cfg.cacheLast;
   pOld->cfg.replications = pNew->cfg.replications;
   pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
+  pOld->cfg.minRows = pNew->cfg.minRows;
+  pOld->cfg.maxRows = pNew->cfg.maxRows;
   pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
   pOld->compactStartTime = pNew->compactStartTime;
   taosWUnLockLatch(&pOld->lock);
@@ -616,13 +618,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
     }
   } else {
     if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
-      if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
+      code = terrno;
-        mInfo("db:%s, is creating and createdb response after trans finished", createReq.db);
+      goto _OVER;
-        code = TSDB_CODE_ACTION_IN_PROGRESS;
-        goto _OVER;
-      } else {
-        goto _OVER;
-      }
     } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) {
       goto _OVER;
     } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) {
@@ -1270,14 +1267,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) {
       usedbRsp.errCode = terrno;
 
       if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
-        if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) {
+        code = terrno;
-          mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db);
+        goto _OVER;
-          code = TSDB_CODE_ACTION_IN_PROGRESS;
-          goto _OVER;
-        }
       }
-
-      mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
     } else {
       if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
         goto _OVER;

source/dnode/mnode/impl/src/mndStb.c

@@ -3114,6 +3114,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
   char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
   STR_TO_VARSTR(typeName, "SUPER_TABLE");
   while (numOfRows < rows) {
+    void *prevIter = pShow->pIter;
     pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
     if (pShow->pIter == NULL) break;
 
@@ -3122,6 +3123,12 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
       continue;
     }
 
+    if ((numOfRows + pStb->numOfColumns) > rows) {
+      pShow->pIter = prevIter;
+      sdbRelease(pSdb, pStb);
+      break;
+    }
+
     SName name = {0};
     char  stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN);

source/dnode/vnode/src/meta/metaCache.c

@@ -14,6 +14,7 @@
  */
 #include "meta.h"
 
+#define TAG_FILTER_RES_KEY_LEN  32
 #define META_CACHE_BASE_BUCKET  1024
 #define META_CACHE_STATS_BUCKET 16
 
@@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry {
 typedef struct STagFilterResEntry {
   SList    list;      // the linked list of md5 digest, extracted from the serialized tag query condition
   uint32_t hitTimes;  // queried times for current super table
-  uint32_t accTime;
 } STagFilterResEntry;
 
 struct SMetaCache {
@@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
   return 0;
 }
 
+static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) {
+//  ASSERT(keyLen == sizeof(int64_t) * 2);
+  memcpy(&pBuf[2], key, keyLen);
+}
+
+// the format of key:
+// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
+static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) {
+  buf[0] = (uint64_t) pHashMap;
+  buf[1] = suid;
+  setMD5DigestInKey(buf, key, keyLen);
+  ASSERT(keyLen == sizeof(uint64_t) * 2);
+}
+
 int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1,
                                   bool* acquireRes) {
+  int32_t vgId = TD_VID(pMeta->pVnode);
+
   // generate the composed key for LRU cache
   SLRUCache*     pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
   SHashObj*      pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
   TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
 
-  uint64_t buf[4];
-
   *acquireRes = 0;
-
+  uint64_t key[4];
-  buf[0] = (uint64_t)pTableMap;
+  initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
-  buf[1] = suid;
-  memcpy(&buf[2], pKey, keyLen);
 
   taosThreadMutexLock(pLock);
   pMeta->pCache->sTagFilterResCache.accTimes += 1;
 
-  int32_t    len = keyLen + sizeof(uint64_t) * 2;
+  LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
-  LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len);
   if (pHandle == NULL) {
     taosThreadMutexUnlock(pLock);
     return TSDB_CODE_SUCCESS;
@@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK
 
   uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes;
   if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) {
-    metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
+    metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
   }
 
   taosLRUCacheRelease(pCache, pHandle, false);
@@ -563,10 +574,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
 // check both the payload size and selectivity ratio
 int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
                               int32_t payloadLen, double selectivityRatio) {
+  int32_t code = 0;
+  int32_t vgId = TD_VID(pMeta->pVnode);
+
   if (selectivityRatio > tsSelectivityRatio) {
     metaDebug("vgId:%d, suid:%" PRIu64
               " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
-              TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio);
+              vgId, suid, selectivityRatio, tsSelectivityRatio);
     taosMemoryFree(pPayload);
     return TSDB_CODE_SUCCESS;
   }
@@ -574,7 +588,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
   if (payloadLen > tsTagFilterResCacheSize) {
     metaDebug("vgId:%d, suid:%" PRIu64
               " failed to add to uid list cache, due to payload length %d greater than threshold %d",
-              TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize);
+              vgId, suid, payloadLen, tsTagFilterResCacheSize);
     taosMemoryFree(pPayload);
     return TSDB_CODE_SUCCESS;
   }
@@ -583,26 +597,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
   SHashObj*      pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
   TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
 
-  // the format of key:
+  uint64_t key[4] = {0};
-  // hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
+  initCacheKey(key, pTableEntry, suid, pKey, keyLen);
 
-  uint64_t buf[4] = {0};
-  buf[0] = (uint64_t)pTableEntry;
-  buf[1] = suid;
-  memcpy(&buf[2], pKey, keyLen);
-  ASSERT(keyLen == 16);
-
-  int32_t code = 0;
   taosThreadMutexLock(pLock);
-
   STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
   if (pEntry == NULL) {
     code = addNewEntry(pTableEntry, pKey, keyLen, suid);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
-  } else {
+  } else { // check if it exists or not
-    // check if it exists or not
     size_t size = listNEles(&(*pEntry)->list);
     if (size == 0) {
       tdListAppend(&(*pEntry)->list, pKey);
@@ -620,12 +625,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
   }
 
   // add to cache.
-  taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL,
+  taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL,
                      TAOS_LRU_PRIORITY_LOW);
 _end:
   taosThreadMutexUnlock(pLock);
-
+  metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
-  metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid,
             (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
 
   return code;
@@ -633,33 +637,36 @@ _end:
 
 // remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
 int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
-  int32_t  keyLen = sizeof(uint64_t) * 3;
+  uint64_t  p[4] = {0};
-  uint64_t p[4] = {0};
+  int32_t   vgId = TD_VID(pMeta->pVnode);
+  SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
 
-  p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry;
+  uint64_t dummy[2] = {0};
-  p[1] = suid;
+  initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16);
 
   TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
-
   taosThreadMutexLock(pLock);
-  STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
+
+  STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
   if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
     taosThreadMutexUnlock(pLock);
     return TSDB_CODE_SUCCESS;
   }
 
+  (*pEntry)->hitTimes = 0;
+
   SListIter iter = {0};
   tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
 
   SListNode* pNode = NULL;
   while ((pNode = tdListNext(&iter)) != NULL) {
-    memcpy(&p[2], pNode->data, 16);
+    setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t));
-    taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen);
+    taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN);
   }
 
-  (*pEntry)->hitTimes = 0;
   tdListEmpty(&(*pEntry)->list);
-
   taosThreadMutexUnlock(pLock);
+
+  metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid);
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/tq/tqRead.c

@@ -296,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
   // todo set the correct vgId
   tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
   if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
-    tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
+    tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
     return -1;
   } else {
     tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);

source/dnode/vnode/src/tq/tqSink.c

@@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
                   TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid);
           metaReaderClear(&mr);
           taosMemoryFree(ctbName);
+          continue;
         }
 
         tbData.uid = mr.me.uid;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -596,9 +596,11 @@ typedef struct {
   int64_t            lastTs;
 } SFSLastNextRowIter;
 
-static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
+static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
+                                    int nCols) {
   SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
   int32_t             code = 0;
+  bool                checkRemainingRow = true;
 
   switch (state->state) {
     case SFSLASTNEXTROW_FS:
@@ -631,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         if (code) goto _err;
       }
 
+      state->pLoadInfo->colIds = aCols;
+      state->pLoadInfo->numOfCols = nCols;
       tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
                      &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
@@ -646,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         goto _next_fileset;
       }
       state->state = SFSLASTNEXTROW_BLOCKROW;
+      checkRemainingRow = false;
     }
     case SFSLASTNEXTROW_BLOCKROW: {
-      bool hasVal = false;
+      bool skipRow = false;
-      state->row = tMergeTreeGetRow(&state->mergeTree);
+      do {
-      *ppRow = &state->row;
+        bool hasVal = false;
-      hasVal = tMergeTreeNext(&state->mergeTree);
+        state->row = tMergeTreeGetRow(&state->mergeTree);
-      if (TSDBROW_TS(&state->row) <= state->lastTs) {
+        *ppRow = &state->row;
-        *pIgnoreEarlierTs = true;
+        hasVal = tMergeTreeNext(&state->mergeTree);
-        *ppRow = NULL;
+        if (TSDBROW_TS(&state->row) <= state->lastTs) {
-        return code;
+          *pIgnoreEarlierTs = true;
-      }
+          *ppRow = NULL;
+          return code;
+        }
 
-      *pIgnoreEarlierTs = false;
+        *pIgnoreEarlierTs = false;
-      if (!hasVal) {
+        if (!hasVal) {
-        state->state = SFSLASTNEXTROW_FILESET;
+          state->state = SFSLASTNEXTROW_FILESET;
-      }
+          break;
+        }
+
+        if (checkRemainingRow) {
+          bool skipBlock = true;
+
+          SBlockData *pBlockData = state->row.pBlockData;
+
+          for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
+            for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
+              SColData *pColData = &pBlockData->aColData[colIndex];
+              int16_t   cid = pColData->cid;
+
+              if (cid == aCols[inputColIndex]) {
+                if (isLast && (pColData->flag & HAS_VALUE)) {
+                  skipBlock = false;
+                  break;
+                } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+                  skipBlock = false;
+                  break;
+                }
+              }
+            }
+          }
+          /*
+          for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
+            SColData *pColData = &pBlockData->aColData[colIndex];
+            int16_t   cid = pColData->cid;
+
+            if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
+              if (isLast && (pColData->flag & HAS_VALUE)) {
+                skipBlock = false;
+                break;
+              } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+                skipBlock = false;
+                break;
+              }
+
+              ++inputColIndex;
+            }
+          }
+          */
+          if (skipBlock) {
+            skipRow = true;
+          }
+        }
+      } while (skipRow);
 
       return code;
     }
@@ -740,9 +793,11 @@ typedef struct SFSNextRowIter {
   int64_t            lastTs;
 } SFSNextRowIter;
 
-static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
+static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
+                                int nCols) {
   SFSNextRowIter *state = (SFSNextRowIter *)iter;
   int32_t         code = 0;
+  bool            checkRemainingRow = true;
 
   switch (state->state) {
     case SFSNEXTROW_FS:
@@ -828,8 +883,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
       }
     }
     case SFSNEXTROW_BLOCKDATA:
+    _next_datablock:
       if (state->iBlock >= 0) {
         SDataBlk block = {0};
+        bool     skipBlock = true;
+        int      inputColIndex = 0;
 
         tDataBlkReset(&block);
         tBlockDataReset(state->pBlockData);
@@ -848,18 +906,94 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
         *pIgnoreEarlierTs = false;
         tBlockDataReset(state->pBlockData);
         TABLEID tid = {.suid = state->suid, .uid = state->uid};
-        code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0);
+        code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols);
         if (code) goto _err;
 
         code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData);
         if (code) goto _err;
 
+        for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
+          SColData *pColData = &state->pBlockData->aColData[colIndex];
+
+          if (isLast && (pColData->flag & HAS_VALUE)) {
+            skipBlock = false;
+            break;
+          } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+            skipBlock = false;
+            break;
+          }
+        }
+
+        if (skipBlock) {
+          if (--state->iBlock < 0) {
+            tsdbDataFReaderClose(state->pDataFReader);
+            *state->pDataFReader = NULL;
+            // resetLastBlockLoadInfo(state->pLoadInfo);
+
+            if (state->aBlockIdx) {
+              // taosArrayDestroy(state->aBlockIdx);
+              tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
+
+              state->aBlockIdxHandle = NULL;
+              state->aBlockIdx = NULL;
+            }
+
+            state->state = SFSNEXTROW_FILESET;
+            goto _next_fileset;
+          } else {
+            goto _next_datablock;
+          }
+        }
+
         state->nRow = state->blockData.nRow;
         state->iRow = state->nRow - 1;
 
         state->state = SFSNEXTROW_BLOCKROW;
+        checkRemainingRow = false;
       }
-    case SFSNEXTROW_BLOCKROW:
+    case SFSNEXTROW_BLOCKROW: {
+      if (checkRemainingRow) {
+        bool skipBlock = true;
+        int  inputColIndex = 0;
+        for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
+          SColData *pColData = &state->pBlockData->aColData[colIndex];
+          int16_t   cid = pColData->cid;
+
+          if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
+            if (isLast && (pColData->flag & HAS_VALUE)) {
+              skipBlock = false;
+              break;
+            } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+              skipBlock = false;
+              break;
+            }
+
+            ++inputColIndex;
+          }
+        }
+
+        if (skipBlock) {
+          if (--state->iBlock < 0) {
+            tsdbDataFReaderClose(state->pDataFReader);
+            *state->pDataFReader = NULL;
+            // resetLastBlockLoadInfo(state->pLoadInfo);
+
+            if (state->aBlockIdx) {
+              // taosArrayDestroy(state->aBlockIdx);
+              tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
+
+              state->aBlockIdxHandle = NULL;
+              state->aBlockIdx = NULL;
+            }
+
+            state->state = SFSNEXTROW_FILESET;
+            goto _next_fileset;
+          } else {
+            goto _next_datablock;
+          }
+        }
+      }
+
       if (state->iRow >= 0) {
         state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow);
         *ppRow = &state->row;
@@ -885,6 +1019,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
       }
 
       return code;
+    }
     default:
       ASSERT(0);
       break;
@@ -960,7 +1095,8 @@ typedef struct SMemNextRowIter {
   // TSDBROW          *curRow;
 } SMemNextRowIter;
 
-static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
+static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
+                                 int nCols) {
   SMemNextRowIter *state = (SMemNextRowIter *)iter;
   int32_t          code = 0;
   *pIgnoreEarlierTs = false;
@@ -1072,7 +1208,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
   return deleted;
 }
 
-typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs);
+typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
+                                  int nCols);
 typedef int32_t (*_next_row_clear_fn_t)(void *iter);
 
 typedef struct {
@@ -1222,12 +1359,14 @@ _err:
 }
 
 // iterate next row non deleted backward ts, version (from high to low)
-static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
+static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast,
+                              int16_t *aCols, int nCols) {
   int code = 0;
   for (;;) {
     for (int i = 0; i < 4; ++i) {
       if (pIter->input[i].next && !pIter->input[i].stop) {
-        code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs);
+        code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs,
+                                         isLast, aCols, nCols);
         if (code) goto _err;
 
         if (pIter->input[i].pRow == NULL) {
@@ -1358,7 +1497,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
 
   do {
     TSDBROW *pRow = NULL;
-    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
+    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0);
 
     if (!pRow) {
       break;
@@ -1488,11 +1627,21 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   bool      ignoreEarlierTs = false;
   SArray   *pColArray = NULL;
   SColVal  *pColVal = &(SColVal){0};
+  int16_t   nCols = nLastCol;
 
   int32_t code = initLastColArray(pTSchema, &pColArray);
   if (TSDB_CODE_SUCCESS != code) {
     return code;
   }
+  SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t));
+  if (NULL == aColArray) {
+    taosArrayDestroy(pColArray);
+
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  for (int i = 1; i < pTSchema->numOfCols; ++i) {
+    taosArrayPush(aColArray, &pTSchema->columns[i].colId);
+  }
 
   TSKEY lastRowTs = TSKEY_MAX;
 
@@ -1502,7 +1651,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
 
   do {
     TSDBROW *pRow = NULL;
-    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
+    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
 
     if (!pRow) {
       break;
@@ -1547,9 +1696,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
           memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
         }
 
-        if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
+        if (!COL_VAL_IS_VALUE(pColVal)) {
-          noneCol = iCol;
+          if (!setNoneCol) {
-          setNoneCol = true;
+            noneCol = iCol;
+            setNoneCol = true;
+          }
+        } else {
+          int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
+          taosArrayRemove(aColArray, aColIndex);
         }
       }
       if (!setNoneCol) {
@@ -1590,6 +1744,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
         }
 
         taosArraySet(pColArray, iCol, &lastCol);
+        int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ);
+        taosArrayRemove(aColArray, aColIndex);
       } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
         noneCol = iCol;
         setNoneCol = true;
@@ -1613,6 +1769,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   //}
 
   nextRowIterClose(&iter);
+  taosArrayDestroy(aColArray);
   // taosMemoryFreeClear(pTSchema);
   return code;
 
@@ -1621,6 +1778,7 @@ _err:
   // taosMemoryFreeClear(pTSchema);
   *ppLastArray = NULL;
   taosArrayDestroy(pColArray);
+  taosArrayDestroy(aColArray);
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
   if (suid != 0) {
     p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1);
     if (p->pSchema == NULL) {
-      taosMemoryFree(p);
       tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
       return TSDB_CODE_PAR_TABLE_NOT_EXIST;
     }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
 
   pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
   if (pUidList->tableUidList == NULL) {
+    taosHashCleanup(pTableMap);
     return NULL;
   }
+
   pUidList->currentIndex = 0;
 
   for (int32_t j = 0; j < numOfTables; ++j) {
@@ -4763,7 +4765,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
   pTableBlockInfo->defMinRows = pc->minRows;
   pTableBlockInfo->defMaxRows = pc->maxRows;
 
-  int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket);
+  int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket);
 
   pTableBlockInfo->numOfFiles += 1;
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -510,7 +510,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
   vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg);
   if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG ||
-       pMsg->msgType == TDMT_VND_BATCH_META) &&
+       pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) &&
       !syncIsReadyForRead(pVnode->sync)) {
     vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
     return 0;

source/libs/executor/src/executil.c

@@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
   int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
+    taosMemoryFree(pResBlock);
     return NULL;
   }
 

source/libs/executor/src/executorimpl.c

@@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else {
     terrno = TSDB_CODE_INVALID_PARA;
+    taosMemoryFree(ops);
     return NULL;
   }
 

source/libs/executor/src/groupoperator.c

@@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
   size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
 
   for (int32_t i = 0; i < numOfGroupCols; ++i) {
-    SColumn*         pCol = taosArrayGet(pGroupCols, i);
+    SColumn*         pCol = (SColumn*) taosArrayGet(pGroupCols, i);
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
 
+    // valid range check. todo: return error code.
+    if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
+      continue;
+    }
+
     if (pBlock->pBlockAgg != NULL) {
       pColAgg = pBlock->pBlockAgg[pCol->slotId];  // TODO is agg data matched?
     }
@@ -999,14 +1004,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
       memset(tbName, 0, TSDB_TABLE_NAME_LEN);
       int32_t len = 0;
       if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) {
-        len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1);
+        len = 1;
-        memcpy(tbName, TSDB_DATA_NULL_STR, len);
+        tbName[0] = 0;
       } else {
         void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
         len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
         memcpy(tbName, varDataVal(pData), len);
+        streamStatePutParName(pState, groupId, tbName);
       }
-      streamStatePutParName(pState, groupId, tbName);
       memcpy(pTmpBlock->info.parTbName, tbName, len);
       pDestBlock->info.rows--;
     } else {

source/libs/executor/src/sysscanoperator.c

@@ -562,15 +562,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
       continue;
     }
 
-    sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
+    if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) {
-
-    if (numOfRows >= pOperator->resultInfo.capacity) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
       numOfRows = 0;
 
+      metaTbCursorPrev(pInfo->pCur);
+
       if (pInfo->pRes->info.rows > 0) {
         break;
       }
+    } else {
+      sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
     }
   }
 

source/libs/executor/src/timewindowoperator.c

@@ -3911,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
   SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < rows; i += winRows) {
-    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
       i++;
       continue;
     }

source/libs/executor/src/tsort.c

@@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
     if (pPage == NULL) {
       taosArrayDestroy(pPageIdList);
       blockDataDestroy(p);
-      taosArrayDestroy(pPageIdList);
       return terrno;
     }
 

source/libs/parser/src/parTranslater.c

@@ -1331,6 +1331,32 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount)
   return code;
 }
 
+static bool isCountNotNullValue(SFunctionNode* pFunc) {
+  if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return false;
+  }
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull);
+}
+
+// count(1) is rewritten as count(ts) for scannning optimization
+static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) {
+  SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0);
+  STableNode*  pTable = NULL;
+  int32_t      code = findTable(pCxt, NULL, &pTable);
+  if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
+    SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+    if (NULL == pCol) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
+      setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol);
+      NODES_DESTORY_LIST(pCount->pParameterList);
+      code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol);
+    }
+  }
+  return code;
+}
+
 static bool isCountTbname(SFunctionNode* pFunc) {
   if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
     return false;
@@ -2041,7 +2067,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) {
   if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) {
     return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION;
   }
-  return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN;
+  return TSDB_CODE_PAR_INVALID_OPTR_USAGE;
 }
 
 static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) {
@@ -2114,13 +2140,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
   }
   if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
     if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) {
-      return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
+      return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
     } else {
       return rewriteColToSelectValFunc(pCxt, pNode);
     }
   }
   if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) {
-    return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
+    return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
   }
   return DEAL_RES_CONTINUE;
 }

source/libs/parser/src/parUtil.c

@@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid usage of RANGE clause, EVERY clause or FILL clause";
     case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN:
       return "No valid function in window query";
+    case TSDB_CODE_PAR_INVALID_OPTR_USAGE:
+      return "Invalid usage of expr: %s";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/parser/test/parSelectTest.cpp

@@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) {
 TEST_F(ParserSelectTest, intervalSemanticCheck) {
   useDb("root", "test");
 
-  run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN);
+  run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE);
   run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE);
   run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 "
       "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)",

source/libs/stream/src/streamExec.c

@@ -20,7 +20,7 @@
 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
   int32_t code;
   void*   exec = pTask->exec.executor;
-  while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
+  while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
     qError("stream task wait for the end of fill history");
     taosMsleep(2);
     continue;

source/libs/stream/src/streamState.c

@@ -130,21 +130,25 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
   char cfgPath[1030];
   sprintf(cfgPath, "%s/cfg", statePath);
 
+  szPage = szPage < 0 ? 4096 : szPage;
+  pages = pages < 0 ? 256 : pages;
   char cfg[1024];
   memset(cfg, 0, 1024);
   TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ);
   if (pCfgFile != NULL) {
-    int64_t size;
+    int64_t size = 0;
     taosFStatFile(pCfgFile, &size, NULL);
-    taosReadFile(pCfgFile, cfg, size);
+    if (size > 0) {
-    sscanf(cfg, "%d\n%d\n", &szPage, &pages);
+      taosReadFile(pCfgFile, cfg, size);
+      sscanf(cfg, "%d\n%d\n", &szPage, &pages);
+    }
   } else {
-    taosMulModeMkDir(statePath, 0755);
+    int32_t code = taosMulModeMkDir(statePath, 0755);
-    pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
+    if (code == 0) {
-    szPage = szPage < 0 ? 4096 : szPage;
+      pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
-    pages = pages < 0 ? 256 : pages;
+      sprintf(cfg, "%d\n%d\n", szPage, pages);
-    sprintf(cfg, "%d\n%d\n", szPage, pages);
+      taosWriteFile(pCfgFile, cfg, strlen(cfg));
-    taosWriteFile(pCfgFile, cfg, strlen(cfg));
+    }
   }
   taosCloseFile(&pCfgFile);
 

tests/parallel_test/cases.task

@@ -1097,7 +1097,7 @@
 ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py
 #,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
 ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R
-,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
+#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
 
 #docs-examples test
 ,,n,docs-examples-test,bash python.sh

tests/pytest/util/sql.py

@@ -23,6 +23,29 @@ import pandas as pd
 from util.log import *
 from util.constant import *
 
+# from datetime import timezone
+from tzlocal import get_localzone
+import pytz
+import time
+
+def _locaTzTimeStamp(utctimestamp):
+    tz = get_localzone()
+    temptz = str(tz)
+    localtz = pytz.timezone(temptz)
+    defUtctimestamp=1035640800
+    local_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz)
+    defLocaltimestamp = time.mktime(local_dt.timetuple())
+    deltaTzTime = int(defLocaltimestamp)-int(defUtctimestamp)
+    temp = int(str(utctimestamp)[0:10])
+    tempOther = str(utctimestamp)[10-len(str(utctimestamp)):]
+    localtemp = deltaTzTime + temp
+    localAll = str(localtemp) + str(tempOther)
+    localtimestamp =  int(localAll)
+    
+    print(f"local timezone is {localtimestamp}, Deltel time is {deltaTzTime}s")
+    return localtimestamp
+
+
 def _parse_datetime(timestr):
     try:
         return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f')
@@ -227,28 +250,80 @@ class TDSql:
         self.checkRowCol(row, col)
         return self.cursor.istype(col, dataType)
 
+
     def checkData(self, row, col, data):
+        if row >= self.queryRows:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno, self.sql, row+1, self.queryRows)
+            tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args)
+        if col >= self.queryCols:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno, self.sql, col+1, self.queryCols)
+            tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args)   
+      
         self.checkRowCol(row, col)
+
         if self.queryResult[row][col] != data:
             if self.cursor.istype(col, "TIMESTAMP"):
-                # suppose user want to check nanosecond timestamp if a longer data passed
+                # suppose user want to check nanosecond timestamp if a longer data passed`` 
-                if (len(data) >= 28):
+                if isinstance(data,str) :
-                    if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data):
+                    if (len(data) >= 28):
-                        tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                        resultData = _locaTzTimeStamp(self.queryResult[row][col])
+                        if pd.to_datetime(resultData) == pd.to_datetime(data):
+                            # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}")
+                            tdLog.info("check successfully")
+                        else:
+                            caller = inspect.getframeinfo(inspect.stack()[1][0])
+                            args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                            tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)                        
+                    else:
+                        if self.queryResult[row][col] == _parse_datetime(data):
+                            # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                            tdLog.info("check successfully")
+                        else:
+                            caller = inspect.getframeinfo(inspect.stack()[1][0])
+                            args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                            tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
+                    return
+                elif isinstance(data,int) :
+                    if len(str(data)) == 16 :
+                        unitTime = 'us'
+                    elif len(str(data)) == 13 :
+                        unitTime = 'ms'    
+                    elif len(str(data)) == 19 :
+                        unitTime = 'ns'   
+                    else:
+                        caller = inspect.getframeinfo(inspect.stack()[1][0])
+                        args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                        tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
+                    resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime) 
+                    if resultData == self.queryResult[row][col] :
+                        # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}")
+                        tdLog.info("check successfully")
+                    else:
+                        caller = inspect.getframeinfo(inspect.stack()[1][0])
+                        args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                        tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
+                    return
                 else:
-                    if self.queryResult[row][col] == _parse_datetime(data):
+                    caller = inspect.getframeinfo(inspect.stack()[1][0])
-                        tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                    args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
-                return
+                    tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
+                  
 
             if str(self.queryResult[row][col]) == str(data):
-                tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                tdLog.info("check successfully")
                 return
 
             elif isinstance(data, float):
                 if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001:
-                    tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                    tdLog.info("check successfully")
                 elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001:
-                    tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                    tdLog.info("check successfully")
+
                 else:
                     caller = inspect.getframeinfo(inspect.stack()[1][0])
                     args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
@@ -258,8 +333,7 @@ class TDSql:
                 caller = inspect.getframeinfo(inspect.stack()[1][0])
                 args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
                 tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
-
+        tdLog.info("check successfully")
-        tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
 
     # return true or false replace exit, no print out
     def checkRowColNoExit(self, row, col):

tests/requirements.txt

@@ -8,3 +8,4 @@ distro
 requests
 pexpect
 faker
+pyopenssl

tests/script/tsim/parser/col_arithmetic_operation.sim

@@ -9,7 +9,7 @@ $dbPrefix = ca_db
 $tbPrefix = ca_tb
 $stbPrefix = ca_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $ts0 = 1537146000000
 $delta = 600000
@@ -18,6 +18,7 @@ $i = 0
 $db = $dbPrefix . $i
 $stb = $stbPrefix . $i
 
+print  drop database $db -x step1
 sql drop database $db -x step1
 step1:
 sql create database $db

tests/script/tsim/parser/col_arithmetic_query.sim

@@ -4,7 +4,7 @@
 $dbPrefix = ca_db
 $tbPrefix = ca_tb
 $stbPrefix = ca_stb
-$rowNum = 10000
+$rowNum = 1000
 
 $i = 0
 $db = $dbPrefix . $i
@@ -33,7 +33,7 @@ endi
 
 # asc/desc order [d.2] ======================================================
 sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc;
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 if $data00 != 0.000000000 then
@@ -57,7 +57,7 @@ if $data91 != 1.000000000 then
 endi
 
 sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc;
-if $rows != 10000 then
+if $rows != 1000 then
  return -1
 endi
 if $data00 != 2.200000000 then
@@ -151,7 +151,7 @@ sql       select top(c1,1) - 88 from $tb
 
 # all data types [d.6] ================================================================
 sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb
-if $rows != 10000 then
+if $rows != 1000 then
  return -1
 endi
 if $data00 != 0.000000000 then
@@ -221,7 +221,7 @@ if $data90 != 76.000000000 then
   return -1
 endi
 
-sql select c4 / 99.123 from $tb limit 10 offset 9999;
+sql select c4 / 99.123 from $tb limit 10 offset 999;
 if $rows != 1 then
   return -1
 endi
@@ -237,7 +237,7 @@ sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12);
 
 # constant column. [d.13]==============================================================
 sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 if $data00 != 0 then
@@ -261,7 +261,7 @@ endi
 
 # column value filter [d.14]===========================================================
 sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2
-if $rows != 2000 then
+if $rows != 200 then
   return -1
 endi
 if $data00 != 0 then
@@ -338,13 +338,14 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb
 if $rows != 1 then
   return -1
 endi
-if $data00 != 1.800000000 then
+print  $data00 
+if $data00 != 6.500000000 then
   return -1
 endi
-if $data01 != 100000.000000000 then
+if $data01 != 10000.000000000 then
   return -1
 endi
-if $data02 != 200000.000000000 then
+if $data02 != 20000.000000000 then
   return -1
 endi
 
@@ -371,7 +372,7 @@ endi
 if $data00 != 0.000000000 then
   return -1
 endi
-if $data01 != 225000.000000000 then
+if $data01 != 22500.000000000 then
   return -1
 endi
 if $data02 != 8.077777778 then
@@ -383,7 +384,7 @@ endi
 if $data04 != 0.444444444 then
   return -1
 endi
-if $data05 != 450000.000000000 then
+if $data05 != 45000.000000000 then
   return -1
 endi
 
@@ -484,10 +485,10 @@ endi
 if $data10 != 0.000000000 then
   return -1
 endi
-if $data20 != 0.997600000 then
+if $data20 != 0.976000000 then
   return -1
 endi
-if $data90 != 7.980800000 then
+if $data90 != 7.808000000 then
   return -1
 endi
 
@@ -496,7 +497,7 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0;
 if $rows != 1 then
   return -1
 endi
-if $data00 != -449988.000000000 then
+if $data00 != -44988.000000000 then
   return -1
 endi
 
@@ -546,7 +547,7 @@ endi
 
 # interval query [d.17]===============================================================
 sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from ca_stb0 interval(1s)
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 

tests/script/tsim/parser/commit.sim

@@ -7,7 +7,7 @@ $dbPrefix = sc_db
 $tbPrefix = sc_tb
 $stbPrefix = sc_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $loops = 5
 $log = 1

tests/script/tsim/parser/groupby.sim

@@ -7,7 +7,7 @@ $dbPrefix = group_db
 $tbPrefix = group_tb
 $mtPrefix = group_mt
 $tbNum = 8
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 
 print =============== groupby.sim
@@ -88,7 +88,7 @@ if $row != 20 then
  return -1
 endi
 
-if $data00 != 100 then
+if $data00 != 10 then
  return -1
 endi
 
@@ -96,7 +96,7 @@ if $data01 != 0 then
   return -1
 endi
 
-if $data10 != 100 then
+if $data10 != 10 then
   return -1
 endi
 
@@ -186,7 +186,7 @@ if $data04 != 0.00000 then
   return -1
 endi
 
-if $data10 != 100 then
+if $data10 != 10 then
   return -1
 endi
 
@@ -227,7 +227,7 @@ if $row != 20 then
   return -1
 endi
 
-if $data00 != 800 then
+if $data00 != 80 then
   return -1
 endi
 
@@ -235,7 +235,7 @@ if $data01 != 0 then
   return -1
 endi
 
-if $data10 != 800 then
+if $data10 != 80 then
   return -1
 endi
 
@@ -243,7 +243,7 @@ if $data11 != 1 then
   return -1
 endi
 
-if $data90 != 800 then
+if $data90 != 80 then
   return -1
 endi
 
@@ -297,7 +297,7 @@ if $data00 != $data03 then
   return -1
 endi
 
-if $data01 != @70-01-01 08:01:49.900@ then
+if $data01 != @70-01-01 08:01:40.900@ then
   return -1
 endi
 
@@ -305,7 +305,7 @@ if $data02 != @70-01-01 08:01:40.000@ then
   return -1
 endi
 
-if $data07 != 800 then
+if $data07 != 80 then
   return -1
 endi
 
@@ -313,7 +313,7 @@ if $data10 != $data13 then
   return -1
 endi
 
-if $data11 != @70-01-01 08:01:49.901@ then
+if $data11 != @70-01-01 08:01:40.901@ then
   return -1
 endi
 
@@ -321,7 +321,7 @@ if $data12 != @70-01-01 08:01:40.001@ then
   return -1
 endi
 
-if $data17 != 800 then
+if $data17 != 80 then
   return -1
 endi
 
@@ -329,7 +329,7 @@ if $data90 != $data93 then
   return -1
 endi
 
-if $data91 != @70-01-01 08:01:49.909@ then
+if $data91 != @70-01-01 08:01:40.909@ then
   return -1
 endi
 
@@ -337,11 +337,11 @@ if $data92 != @70-01-01 08:01:40.009@ then
   return -1
 endi
 
-if $data97 != 800 then
+if $data97 != 80 then
   return -1
 endi
 
-if $data95 != 7200 then
+if $data95 != 720 then
   return -1
 endi
 
@@ -358,7 +358,7 @@ if $data00 != 0 then
   return -1
 endi
 
-if $data11 != 800 then
+if $data11 != 80 then
   return -1
 endi
 
@@ -372,7 +372,7 @@ if $data00 != 0 then
   return -1
 endi
 
-if $data01 != @70-01-01 08:01:49.900@ then
+if $data01 != @70-01-01 08:01:40.900@ then
   return -1
 endi
 
@@ -389,13 +389,13 @@ if $data04 != 0 then
   return -1
 endi
 
-if $data06 != 100 then
+if $data06 != 10 then
   return -1
 endi
 
 
 sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 order by c4;
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 
@@ -420,13 +420,13 @@ if $rows != 1 then
   return -1
 endi
 
-sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 9990;
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 990;
 if $rows != 10 then
   return -1
 endi
 
 sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 
@@ -441,32 +441,32 @@ if $rows != 100 then
   return -1
 endi
 
-if $data00 != 100 then
+if $data00 != 10 then
   return -1
 endi
 
-if $data01 != 495000 then
+if $data01 != 4500 then
   return -1
 endi
 
-if $data02 != 100 then
+if $data02 != 10 then
   return -1
 endi
 
-if $data03 != 4950.000000000 then
+if $data03 != 450.000000000 then
-  print expect 4950.000000000 , acutal $data03
+  print expect 450.000000000 , acutal $data03
   return -1
 endi
 
-if $data10 != 100 then
+if $data10 != 10 then
   return -1
 endi
 
-if $data11 != 495100 then
+if $data11 != 4510 then
   return -1
 endi
 
-if $data13 != 4951.000000000 then
+if $data13 != 451.000000000 then
   return -1
 endi
 
@@ -481,19 +481,19 @@ if $rows != 1 then
   return -1
 endi
 
-#if $data00 != 79200.000000000 then
+#if $data00 != 2160.000000000 then
 #  return -1
 #endi
 
-#if $data01 != @binary99@ then
+#if $data01 != @binary27@ then
 #  return -1
 #endi
 
-#if $data02 != 99.000000000 then
+#if $data02 != 27.000000000 then
 #  return -1
 #endi
 
-#if $data03 != 99.000000000 then
+#if $data03 != 27.000000000 then
 #  return -1
 #endi
 
@@ -503,7 +503,7 @@ if $rows != 100 then
   return -1
 endi
 
-if $data00 != 4851.000000000 then
+if $data00 != 441.000000000 then
   return -1
 endi
 
@@ -511,19 +511,19 @@ if $data01 != 0 then
   return -1
 endi
 
-if $data02 != 9900 then
+if $data02 != 900 then
   return -1
 endi
 
-if $data03 != 4950.000000000 then
+if $data03 != 450.000000000 then
   return -1
 endi
 
-if $data04 != 2886.607004772 then
+if $data04 != 287.228132327 then
   return -1
 endi
 
-if $data10 != 4852.000000000 then
+if $data10 != 442.000000000 then
   return -1
 endi
 
@@ -531,15 +531,15 @@ if $data11 != 1 then
   return -1
 endi
 
-if $data12 != 9901 then
+if $data12 != 901 then
   return -1
 endi
 
-if $data13 != 4951.000000000 then
+if $data13 != 451.000000000 then
   return -1
 endi
 
-if $data14 != 2886.607004772 then
+if $data14 != 287.228132327 then
   return -1
 endi
 

tests/script/tsim/parser/interp.sim

@@ -7,7 +7,7 @@ $dbPrefix = intp_db
 $tbPrefix = intp_tb
 $stbPrefix = intp_stb
 $tbNum = 4
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $ts0 = 1537146000000
 $delta = 600000

tests/script/tsim/parser/join_manyblocks.sim

@@ -7,7 +7,7 @@ $dbPrefix = join_m_db
 $tbPrefix = join_tb
 $mtPrefix = join_mt
 $tbNum = 3
-$rowNum = 20000
+$rowNum = 2000
 $totalNum = $tbNum * $rowNum
 
 print =============== join_manyBlocks.sim
@@ -78,8 +78,8 @@ print ==============> td-3313
 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1;
 
 print $row
-if $row != 60000 then
+if $row != 6000 then
-  print expect 60000, actual: $row
+  print expect 6000, actual: $row
   return -1
 endi
 

tests/script/tsim/parser/limit1.sim

@@ -7,7 +7,7 @@ $dbPrefix = lm1_db
 $tbPrefix = lm1_tb
 $stbPrefix = lm1_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $ts0 = 1537146000000
 $delta = 600000

tests/script/tsim/parser/limit1_stb.sim

@@ -4,7 +4,7 @@ $dbPrefix = lm1_db
 $tbPrefix = lm1_tb
 $stbPrefix = lm1_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $ts0 = 1537146000000
 $delta = 600000
@@ -453,13 +453,14 @@ endi
 ### [TBASE-361]
 $offset = $rowNum / 2
 $offset = $offset + 1
+print === select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
 sql select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
 $val = $rowNum - $offset
 if $rows != $val then
   print expect $val, actual:$rows
   return -1
 endi
-if $data00 != @18-10-22 02:30:00.000@ then
+if $data00 != @18-09-20 20:30:00.000@ then
   return -1
 endi
 if $data01 != 1 then

tests/script/tsim/parser/limit1_tb.sim

@@ -4,7 +4,7 @@ $dbPrefix = lm1_db
 $tbPrefix = lm1_tb
 $stbPrefix = lm1_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $ts0 = 1537146000000
 $delta = 600000
@@ -13,7 +13,7 @@ $i = 0
 $db = $dbPrefix . $i
 $stb = $stbPrefix . $i
 
-print ====== use db
+print ====== use $db
 sql use $db
 
 ##### select from table
@@ -664,9 +664,9 @@ endi
 if $data21 != 4.027681991 then
   return -1
 endi
-
+print select  _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
 sql select  _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
-if $rows != 3704 then
+if $rows != 371 then
   return -1
 endi
 if $data01 != 2 then
@@ -994,11 +994,14 @@ endi
 $offset = $rowNum / 10
 $offset = $offset * 3
 $offset = $offset - 1
+
+print === select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
 sql select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
 if $rows != 1 then
   return -1
 endi
-if $data00 != @18-11-25 18:40:00.000@ then
+print $data00
+if $data00 != @18-09-24 06:40:00.000@ then
   return -1
 endi
 if $data01 != 4 then
@@ -1012,19 +1015,19 @@ sql select * from $tb order by ts desc limit 5
 if $rows != 5 then
   return -1
 endi
-if $data00 != @18-11-25 19:30:00.000@ then
+if $data00 != @18-09-24 07:30:00.000@ then
   return -1
 endi
 if $data01 != 9 then
   return -1
 endi
-if $data10 != @18-11-25 19:20:00.000@ then
+if $data10 != @18-09-24 07:20:00.000@ then
   return -1
 endi
 if $data12 != 8 then
   return -1
 endi
-if $data20 != @18-11-25 19:10:00.000@ then
+if $data20 != @18-09-24 07:10:00.000@ then
   return -1
 endi
 if $data23 != 7.00000 then
@@ -1048,7 +1051,7 @@ endi
 if $data39 != nchar6 then
   return -1
 endi
-if $data40 != @18-11-25 18:50:00.000@ then
+if $data40 != @18-09-24 06:50:00.000@ then
   return -1
 endi
 
@@ -1056,19 +1059,19 @@ sql select * from $tb order by ts desc limit 5 offset 1
 if $rows != 5 then
   return -1
 endi
-if $data00 != @18-11-25 19:20:00.000@ then
+if $data00 != @18-09-24 07:20:00.000@ then
   return -1
 endi
 if $data01 != 8 then
   return -1
 endi
-if $data10 != @18-11-25 19:10:00.000@ then
+if $data10 != @18-09-24 07:10:00.000@ then
   return -1
 endi
 if $data12 != 7 then
   return -1
 endi
-if $data20 != @18-11-25 19:00:00.000@ then
+if $data20 != @18-09-24 07:00:00.000@ then
   return -1
 endi
 if $data23 != 6.00000 then
@@ -1092,12 +1095,13 @@ endi
 if $data39 != nchar5 then
   return -1
 endi
-if $data40 != @18-11-25 18:40:00.000@ then
+if $data40 != @18-09-24 06:40:00.000@ then
   return -1
 endi
 
 $offset = $rowNum
 $offset = $offset - 2
+print ==== select * from $tb order by ts desc limit 5 offset $offset
 sql select * from $tb order by ts desc limit 5 offset $offset
 if $rows != 2 then
   return -1
@@ -1119,16 +1123,16 @@ sql select * from $tb where c1 < 8 order by ts desc limit 3 offset 2
 if $rows != 3 then
   return -1
 endi
-if $data00 != @18-11-25 18:50:00.000@ then
+if $data00 != @18-09-24 06:50:00.000@ then
   return -1
 endi
 if $data01 != 5 then
   return -1
 endi
-if $data10 != @18-11-25 18:40:00.000@ then
+if $data10 != @18-09-24 06:40:00.000@ then
   return -1
 endi
-if $data20 != @18-11-25 18:30:00.000@ then
+if $data20 != @18-09-24 06:30:00.000@ then
   return -1
 endi
 if $data12 != 4 then

tests/script/tsim/parser/nestquery.sim

@@ -8,7 +8,7 @@ print ======================== dnode1 start
 $dbPrefix = nest_db
 $tbPrefix = nest_tb
 $mtPrefix = nest_mt
-$tbNum = 10
+$tbNum = 3
 $rowNum = 10000
 $totalNum = $tbNum * $rowNum
 
@@ -23,7 +23,7 @@ sql create database if not exists $db
 sql use $db
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
 
-$half = $tbNum / 2
+$half = 2
 
 $i = 0
 while $i < $half
@@ -69,7 +69,7 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname)
 if $rows != 1 then
   return -1
 endi
-if $data00 != 10 then
+if $data00 != 4 then
   return -1
 endi
 
@@ -77,7 +77,7 @@ sql select count(*) from (select count(*) from nest_mt0 partition by tbname inte
 if $rows != 1 then
   return -1
 endi
-if $data00 != 170 then
+if $data00 != 68 then
   return -1
 endi
 
@@ -85,7 +85,7 @@ sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname inte
 if $rows != 1 then
   return -1
 endi
-if $data00 != 100000 then
+if $data00 != 40000 then
   return -1
 endi
 

tests/script/tsim/parser/projection_limit_offset.sim

@@ -7,7 +7,7 @@ $dbPrefix = group_db
 $tbPrefix = group_tb
 $mtPrefix = group_mt
 $tbNum = 8
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 
 print =============== projection_limit_offset.sim
@@ -81,199 +81,199 @@ $ts2 = $tb2 . .ts
 sql select ts from group_mt0
 print $rows
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 0;
 if $rows != 4008 then
   print expect 4008, actual:$rows
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 1;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 1;
 if $rows != 4007 then
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 101;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 101;
 print $rows
 if $rows != 3907 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.101@ then
+if $data00 != @70-01-01 08:01:40.101@ then
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 902;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 902;
 if $rows != 3106 then
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 400;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 400;
 if $rows != 3608 then
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 4007;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 4007;
 if $rows != 1 then
   return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 2000 offset 4008;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 2000 offset 4008;
 if $rows != 0 then
   return -1
 endi
 
 #==================================order by desc, multi vnode, limit/offset===================================
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 0;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 0;
 if $rows != 4008 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.500@ then
+if $data00 != @70-01-01 08:01:40.500@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 1;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 1;
 if $rows != 4007 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.500@ then
+if $data00 != @70-01-01 08:01:40.500@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 101;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 101;
 print $rows
 if $rows != 3907 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.488@ then
+if $data00 != @70-01-01 08:01:40.488@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 902;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 902;
 if $rows != 3106 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.388@ then
+if $data00 != @70-01-01 08:01:40.388@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 400;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 400;
 if $rows != 3608 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.450@ then
+if $data00 != @70-01-01 08:01:40.450@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 4007;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 4007;
 if $rows != 1 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.000@ then
+if $data00 != @70-01-01 08:01:40.000@ then
    return -1
 endi
 
-sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 2000 offset 4008;
+sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 2000 offset 4008;
 if $rows != 0 then
   return -1
 endi
 
 #=================================single value filter======================================
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 0;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 0;
 if $row != 8 then
    return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.000@ then
+if $data00 != @70-01-01 08:01:40.000@ then
   return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 1;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 1;
 if $row != 7 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 2;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 2;
 if $row != 6 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 4;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 4;
 if $row != 4 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 7;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 7;
 if $row != 1 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 8;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 8;
 if $row != 0 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 9;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 9;
 if $row != 0 then
    return -1
 endi
 
 #===============================single value filter, order by desc============================
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 0;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 0;
 if $row != 8 then
    return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.000@ then
+if $data00 != @70-01-01 08:01:40.000@ then
   return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 1;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 1;
 if $row != 7 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 2;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 2;
 if $row != 6 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 4;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 4;
 if $row != 4 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 7;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 7;
 if $row != 1 then
    return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.000@ then
+if $data00 != @70-01-01 08:01:40.000@ then
   return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 8;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 8;
 if $row != 0 then
    return -1
 endi
 
-sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 9;
+sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 9;
 if $row != 0 then
    return -1
 endi
 
 #[tbase-695]
-sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000
+sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:40.500' and c1<99999999 limit 10000 offset 500
 print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
 print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
 print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
 print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
-if $row != 35000 then
+if $row != 3500 then
    return -1
 endi
 
@@ -301,7 +301,7 @@ print tbase-722
 sql select spread(ts) from group_tb0;
 print $data00
 
-if $data00 != 9999.000000000 then
+if $data00 != 999.000000000 then
    return -1
 endi
 

tests/script/tsim/parser/selectResNum.sim

@@ -7,10 +7,10 @@ $dbPrefix = sc_db
 $tbPrefix = sc_tb
 $stbPrefix = sc_stb
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 $loops = 200000
-$log = 10000
+$log = 1000
 $ts0 = 1537146000000
 $delta = 600000
 print ========== selectResNum.sim

tests/script/tsim/parser/sliding.sim

@@ -7,7 +7,7 @@ $dbPrefix = sliding_db
 $tbPrefix = sliding_tb
 $mtPrefix = sliding_mt
 $tbNum = 8
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 
 print =============== sliding.sim
@@ -88,25 +88,25 @@ $ts1 = $tb1 . .ts
 $ts2 = $tb2 . .ts
 
 print ===============================interval_sliding query
-sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s);
+sql select _wstart, count(*) from sliding_tb0 interval(3s) sliding(3s);
 if $row != 10 then
  return -1
 endi
 if $data00 != @00-01-01 00:00:00.000@ then
  return -1
 endi
-if $data01 != 1000 then
+if $data01 != 100 then
  return -1
 endi
-if $data10 != @00-01-01 00:00:30.000@ then
+if $data10 != @00-01-01 00:00:03.000@ then
  return -1
 endi
-if $data11 != 1000 then
+if $data11 != 100 then
  return -1
 endi
 
 sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a);
-if $row != 10000 then
+if $row != 1000 then
   return -1
 endi
 if $data00 != @00-01-01 00:00:00.000@ then
@@ -123,10 +123,10 @@ if $data91 != 0.000000000 then
 endi
 
 sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc;
-if $row != 10000 then
+if $row != 1000 then
   return -1
 endi
-if $data00 != @00-01-01 00:04:59.970@ then
+if $data00 != @00-01-01 00:00:29.970@ then
  return -1
 endi
 if $data01 != 0.000000000 then
@@ -141,7 +141,7 @@ endi
 if $data04 != 99 then
   return -1
 endi
-if $data90 != @00-01-01 00:04:59.700@ then
+if $data90 != @00-01-01 00:00:29.700@ then
   return -1
 endi
 if $data91 != 0.000000000 then
@@ -157,41 +157,41 @@ if $data94 != 90 then
   return -1
 endi
 
-sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc;
+sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(3s) sliding(1s) order by _wstart asc;
 if $row != 32 then
   return -1
 endi
-if $data00 != @99-12-31 23:59:40.000@ then
+if $data00 != @99-12-31 23:59:58.000@ then
-  print expect 12-31 23:59:40.000, actual: $data00
+  print expect 12-31 23:59:58.000, actual: $data00
   return -1
 endi
-if $data01 != 334 then
+if $data01 != 34 then
   return -1
 endi
 if $data02 != 33 then
   return -1
 endi
 
-sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc;
+sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc;
 if $row != 2 then
   return -1
 endi
 if $data04 != 99 then
   return -1
 endi
-if $data01 != 999 then
+if $data01 != 99 then
   return -1
 endi
-if $data02 != 28.837977152 then
+if $data02 != 28.577380332 then
   return -1
 endi
 
 #interval offset + limit
-sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990;
+sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 90;
 if $row != 10 then
   return -1
 endi
-if $data00 != @00-01-01 00:04:30.270@ then
+if $data00 != @00-01-01 00:00:27.270@ then
   return -1
 endi
 if $data01 != 1 then
@@ -203,7 +203,7 @@ endi
 if $data03 != 0.000000000 then
   return -1
 endi
-if $data90 != @00-01-01 00:04:30.000@ then
+if $data90 != @00-01-01 00:00:27.000@ then
   return -1
 endi
 if $data91 != 1 then
@@ -217,43 +217,43 @@ if $data93 != 0.000000000 then
 endi
 
 #interval offset test
-sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1;
+sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(3s) order by _wstart asc limit 100 offset 1;
 if $row != 9 then
   return -1
 endi
-if $data00 != @00-01-01 00:00:30.000@ then
+if $data00 != @00-01-01 00:00:03.000@ then
   return -1
 endi
-if $data01 != 1000 then
+if $data01 != 100 then
   return -1
 endi
 if $data02 != 99 then
   return -1
 endi
-if $data80 != @00-01-01 00:04:30.000@ then
+if $data80 != @00-01-01 00:00:27.000@ then
   return -1
 endi
-if $data81 != 1000 then
+if $data81 != 100 then
   return -1
 endi
 
-sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0;
+sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc limit 100 offset 0;
 if $row != 2 then
  return -1
 endi
 if $data00 != @00-01-01 00:00:00.000@ then
  return -1
 endi
-if $data01 != 999 then
+if $data01 != 99 then
  return -1
 endi
 if $data02 != 99 then
  return -1
 endi
-if $data03 != 28.837977152 then
+if $data03 != 28.577380332 then
  return -1
 endi
-if $data10 != @00-01-01 00:00:30.000@ then
+if $data10 != @00-01-01 00:00:03.000@ then
  return -1
 endi
 if $data11 != 34 then
@@ -266,14 +266,14 @@ if $data13 != 9.810708435 then
  return -1
 endi
 
-sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1;
+sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 1;
 if $row != 15 then
  return -1
 endi
 if $data00 != @00-01-01 00:00:00.000@ then
  return -1
 endi
-if $data01 != 1000 then
+if $data01 != 100 then
  return -1
 endi
 if $data02 != 99 then
@@ -282,85 +282,85 @@ endi
 if $data03 != 28.866070048 then
  return -1
 endi
-if $data90 != @00-01-01 00:03:00.000@ then
+if $data90 != @00-01-01 00:00:18.000@ then
  return -1
 endi
-if $data91 != 1000 then
+if $data91 != 100 then
  return -1
 endi
 if $data92 != 99 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 5;
 if $row != 11 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 6;
 if $row != 10 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 7;
 if $row != 9 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 8;
 if $row != 8 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 9;
 if $row != 7 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 10;
 if $row != 6 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 11;
 if $row != 5 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 12;
 if $row != 4 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 13;
 if $row != 3 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 14;
 if $row != 2 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 15;
 if $row != 1 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 16;
 if $row != 0 then
  return -1
 endi
 
-sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc;
+sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) order by _wstart desc;
 if $row != 10 then
  return -1
 endi
-#00-01-01 00:04:30.000|                   10|       0|              0.000000000|              0.000000000|
+#00-01-01 00:00:27.000 |                     1 |        0 |               0.000000000 |               0.000000000 |
-if $data00 != @00-01-01 00:04:30.000@ then
+if $data00 != @00-01-01 00:00:27.000@ then
  return -1
 endi
-if $data01 != 10 then
+if $data01 != 1 then
  return -1
 endi
 if $data02 != 0 then
@@ -370,18 +370,18 @@ if $data03 != 0.000000000 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15;
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 14;
 if $row != 1 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16;
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 15;
 if $row != 0 then
  return -1
 endi
 
 sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2;
-if $data00 != @00-01-01 00:04:59.910@ then
+if $data00 != @00-01-01 00:00:29.910@ then
   return -1
 endi
 

tests/script/tsim/parser/union.sim

@@ -8,7 +8,7 @@ $tbPrefix = union_tb
 $tbPrefix1 = union_tb_
 $mtPrefix = union_mt
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 
 print =============== union.sim
@@ -65,7 +65,7 @@ sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c
 
 $j = 0
 $t = 1578203484000
-$rowNum = 1000
+$rowNum = 100
 $tbNum = 5
 $i = 0
 
@@ -117,7 +117,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l
 
 # sql with parenthese
 sql (select c1 from union_tb0)
-if $rows != 10000 then
+if $rows != 1000 then
   return -1
 endi
 if $data00 != 0 then
@@ -187,7 +187,7 @@ endi
 if $data10 != 1 then
   return -1
 endi
-if $data20 != 4950000 then
+if $data20 != 495000 then
   return -1
 endi
 
@@ -202,7 +202,7 @@ endi
 if $data10 != 1 then
   return -1
 endi
-if $data20 != 495000 then
+if $data20 != 49500 then
   return -1
 endi
 
@@ -211,7 +211,7 @@ sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1
 if $rows != 11 then
   return -1
 endi
-if $data00 != 10000 then
+if $data00 != 1000 then
   return -1
 endi
 if $data10 != 9 then
@@ -227,7 +227,7 @@ endi
 print ===========================================tags union
 # two super table tag union, limit is not active during retrieve tags query
 sql (select t1 from union_mt0) union all (select t1 from union_mt0)
-if $rows != 200000 then
+if $rows != 20000 then
   return -1
 endi
 
@@ -248,7 +248,7 @@ sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (sele
 if $rows != 2 then
   return -1
 endi
-#if $data00 != 495000 then
+#if $data00 != 49500 then
 #  return -1
 #endi
 
@@ -272,7 +272,7 @@ endi
 
 # multi-vnode projection query
 sql (select c1 from union_mt0) union all select c1 from union_mt0;
-if $rows != 200000 then
+if $rows != 20000 then
  return -1
 endi
 
@@ -330,10 +330,10 @@ sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a
 if $rows != 2 then
   return -1
 endi
-if $data00 != 495000 then
+if $data00 != 49500 then
   return -1
 endi
-if $data10 != 495000 then
+if $data10 != 49500 then
   return -1
 endi
 

tests/script/tsim/parser/where.sim

@@ -7,7 +7,7 @@ $dbPrefix = wh_db
 $tbPrefix = wh_tb
 $mtPrefix = wh_mt
 $tbNum = 10
-$rowNum = 10000
+$rowNum = 1000
 $totalNum = $tbNum * $rowNum
 
 print =============== where.sim
@@ -62,7 +62,7 @@ sql select count(*) from $tb where c1<10 and c1<>2
 if $rows != 1 then
   return -1
 endi
-if $data00 != 900 then
+if $data00 != 90 then
   return -1
 endi
 
@@ -308,9 +308,9 @@ if $row != 0 then
 endi
 
 sql select * from wh_mt0 where c3 = 1;
-print $rows -> 1000
+print $rows -> 100
 print $data00 $data01 $data02
-if $row != 1000 then
+if $row != 100 then
   return -1
 endi
 

tests/script/tsim/query/tableCount.sim

@@ -0,0 +1,107 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql drop database if exists db1;
+sql create database db1 vgroups 3;
+sql create database db1;
+sql use db1;
+sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create table tba1 using sta tags(1, 1, 1);
+sql create table tba2 using sta tags(2, 2, 2);
+sql create table tba3 using sta tags(3, 3, 3);
+sql create table tba4 using sta tags(3, 3, 3);
+sql create table tba5 using sta tags(3, 3, 3);
+sql create table tba6 using sta tags(3, 3, 3);
+sql create table tba7 using sta tags(3, 3, 3);
+sql create table tba8 using sta tags(3, 3, 3);
+sql create table tbb1 using stb tags(4, 4, 4);
+sql create table tbb2 using stb tags(5, 5, 5);
+sql create table tbb3 using stb tags(6, 6, 6);
+sql create table tbb4 using stb tags(4, 4, 4);
+sql create table tbb5 using stb tags(5, 5, 5);
+sql create table tbb6 using stb tags(6, 6, 6);
+sql create table tbb7 using stb tags(7, 7, 7);
+sql create table tbb8 using stb tags(8, 8, 8);
+sql create table tbn1 (ts timestamp, f1 int);
+sql create database db2 vgroups 3;
+sql create database db2;
+sql use db2;
+sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
+sql create table tba1 using sta tags(1, 1, 1);
+sql create table tba2 using sta tags(2, 2, 2);
+sql create table tbb1 using stb tags(4, 4, 4);
+sql create table tbb2 using stb tags(5, 5, 5);
+sql create table tbb3 using stb tags(6, 6, 6);
+ 
+sql select count(table_name) from information_schema.ins_tables group by stable_name;
+if $rows != 3 then
+  return -1
+endi
+sql select count(table_name) from information_schema.ins_tables group by db_name;
+if $rows != 4 then
+  return -1
+endi
+sql select count(table_name) from information_schema.ins_tables group by db_name, stable_name;
+if $rows != 7 then
+  return -1
+endi
+sql select stable_name,count(table_name) from information_schema.ins_tables group by stable_name order by stable_name;
+if $rows != 3 then
+  return -1
+endi
+if $data01 != 30 then
+  return -1
+endi
+if $data11 != 10 then
+  return -1
+endi
+if $data21 != 11 then
+  return -1
+endi
+sql select db_name,count(table_name) from information_schema.ins_tables group by db_name order by db_name;
+if $rows != 4 then
+  return -1
+endi
+if $data01 != 17 then
+  return -1
+endi
+if $data11 != 5 then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data31 != 5 then
+  return -1
+endi
+sql select db_name,stable_name,count(table_name) from information_schema.ins_tables group by db_name, stable_name order by db_name, stable_name;
+if $rows != 7 then
+  return -1
+endi
+if $data02 != 1 then
+  return -1
+endi
+if $data12 != 8 then
+  return -1
+endi
+if $data22 != 8 then
+  return -1
+endi
+if $data32 != 2 then
+  return -1
+endi
+if $data42 != 3 then
+  return -1
+endi
+if $data52 != 24 then
+  return -1
+endi
+if $data62 != 5 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/deleteState.sim

@@ -51,13 +51,8 @@ if $loop_count == 10 then
   return -1
 endi
 
-if $data01 != 1 then
+if $rows != 0 then
-  print =====data01=$data01
+  print =====rows=$rows
-  goto loop1
-endi
-
-if $data02 != NULL then
-  print =====data02=$data02
   goto loop1
 endi
 

tests/script/tsim/stream/partitionbyColumnState.sim

@@ -27,13 +27,8 @@ if $loop_count == 10 then
   return -1
 endi
 
-if $data01 != 1 then
+if $rows != 0 then
-  print =====data01=$data01
+  print =====rows=$rows
-  goto loop0
-endi
-
-if $data02 != NULL then
-  print =====data02=$data02
   goto loop0
 endi
 

tests/script/tsim/stream/state1.sim

@@ -0,0 +1,101 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database
+sql create database test vgroups 4;
+sql select * from information_schema.ins_databases;
+if $rows != 3 then
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use test;
+
+sql create table t1(ts timestamp, a int, b int , c int, d double, id int);
+
+print create stream streams1 trigger at_once IGNORE EXPIRED 0  into streamt1 as select  _wstart, count(*) c1 from t1 state_window(a);
+
+sql create stream streams1 trigger at_once IGNORE EXPIRED 0  into streamt1 as select  _wstart, count(*) c1 from t1 state_window(a);
+
+sql insert into t1(ts) values(1648791213000);
+
+$loop_count = 0
+loop0:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sql select * from streamt1;
+print data00 data01
+print data10 data11
+
+if $rows != 0 then
+  print =====rows=$rows
+  goto loop0
+endi
+
+sql insert into t1 values(1648791214000,1,2,3,1.0,3);
+$loop_count = 0
+loop1:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sql select * from streamt1;
+print data00 data01
+print data10 data11
+
+if $rows != 1 then
+  print =====rows=$rows
+  goto loop1
+endi
+
+sql insert into t1 values(1648791215000,2,2,3,1.0,4);
+
+$loop_count = 0
+loop2:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sql select * from streamt1;
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop2
+endi
+
+sql insert into t1(ts) values(1648791216000);
+
+$loop_count = 0
+loop3:
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sql select * from streamt1;
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop2
+endi
+
+
+print state1 end
+
+system sh/stop_dnodes.sh

tests/script/tsim/valgrind/checkError7.sim

@@ -8,8 +8,9 @@ sql create database d1
 sql use d1
 
 $x = 0
-while $x < 128
+while $x < 5
   $tb = d1.s . $x
+  print create table $tb (ts timestamp, i int) tags (j int)
   sql create table $tb (ts timestamp, i int) tags (j int)
   $x = $x + 1
 endw
@@ -17,7 +18,7 @@ endw
 print ======================== describe stables
 # TODO : create stable error
 $m = 0
-while $m < 128
+while $m < 5
   $tb = s . $m
   $filter = ' . $tb
   $filter = $filter . '
@@ -36,15 +37,16 @@ print ======================== show stables
 sql show d1.stables
 
 print num of stables is $rows
-if $rows != 128 then
+if $rows != 5 then
   return -1
 endi
 
 print ======================== create table
 
 $x = 0
-while $x < 424
+while $x < 42
   $tb = d1.t . $x
+  print create table $tb using d1.s0 tags( $x )
   sql create table $tb using d1.s0 tags( $x )
   $x = $x + 1
 endw
@@ -54,7 +56,7 @@ print ======================== show stables
 sql show d1.tables
 
 print num of tables is $rows
-if $rows != 424 then
+if $rows != 42 then
   return -1
 endi
 

tests/system-test/0-others/information_schema.py

@@ -101,10 +101,18 @@ class TDTestCase:
                 tdSql.checkEqual(i[1],len(self.perf_list))
             elif i[0].lower() == self.dbname:
                 tdSql.checkEqual(i[1],self.tbnum+1)
+    def ins_columns_check(self):
+        tdSql.execute('create database db2 vgroups 2 replica 1')
+        tdSql.execute('create table db2.stb2 (ts timestamp,c0 int,c1 int, c2 double, c3 float, c4 binary(1000), c5 nchar(100),c7 bigint, c8 bool, c9 smallint) tags(t0 int)')
+        for i in range(2000):
+            tdSql.execute("create table db2.ctb%d using db2.stb2 tags(%d)" %(i,i))
+        tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type="CHILD_TABLE"')
+        tdSql.checkEqual(20000,len(tdSql.queryResult))
+        print("number of ins_columns of child table in db2 is %s" % len(tdSql.queryResult))
     def run(self):
         self.prepare_data()
         self.count_check()
-
+        self.ins_columns_check()
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/system-test/2-query/Today.py

@@ -16,6 +16,7 @@ class TDTestCase:
         tdSql.init(conn.cursor())
         self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
         self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp()
+        self.today_ts_ns = 0
         self.time_unit = ['b','u','a','s','m','h','d','w']
         self.error_param = ['1.5','abc','!@#','"abc"','today()']
         self.arithmetic_operators = ['+','-','*','/']
@@ -74,8 +75,10 @@ class TDTestCase:
             num_same = 0
             if v.lower() == 'timestamp':
                 tdSql.query(f'select {k} from {tbname}')
+                
                 for i in tdSql.queryResult:
                     if precision == 'ms':
+                        self.today_ts_trans = int(self.today_ts)*1000
                         if int(i[0].timestamp())*1000 > int(self.today_ts)*1000:
                             num_up += 1
                         elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000:
@@ -83,6 +86,7 @@ class TDTestCase:
                         elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000:
                             num_down += 1
                     elif precision == 'us':
+                        self.today_ts_trans = int(self.today_ts)*1000000
                         if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000:
                             num_up += 1
                         elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000:
@@ -90,6 +94,7 @@ class TDTestCase:
                         elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000:
                             num_down += 1
                     elif precision == 'ns':
+                        self.today_ts_trans = int(self.today_ts)*1000000000
                         if i[0] > int(self.today_ts)*1000000000:
                             num_up += 1
                         elif i[0] == int(self.today_ts)*1000000000:
@@ -97,8 +102,9 @@ class TDTestCase:
                         elif i[0] < int(self.today_ts)*1000000000:
                             num_down += 1
                 tdSql.query(f"select today() from {tbname}")
-                tdSql.checkRows(len(values_list)*tb_num)
+                tdSql.checkRows(len(values_list)*tb_num)    
-                tdSql.checkData(0, 0, str(self.today_date))
+                print(self.today_ts_trans,self.today_ts,precision,num_up,num_down,i[0])
+                tdSql.checkData(0, 0, self.today_ts_trans)
                 tdSql.query(f"select * from {tbname} where {k}=today()")
                 if tb == 'tb':
                     tdSql.checkRows(num_same*tb_num)
@@ -149,11 +155,12 @@ class TDTestCase:
                 if tb == 'tb':
                     tdSql.checkRows(num_same*tb_num)
                     for i in range(num_same*tb_num):
-                        tdSql.checkData(i, 0, str(self.today_date))
+                        print(self.today_ts_trans,precision,num_up,num_down)
+                        tdSql.checkData(i, 0, self.today_ts_trans)
                 elif tb == 'stb':
                     tdSql.checkRows(num_same)
                     for i in range(num_same):
-                        tdSql.checkData(i, 0, str(self.today_date))
+                        tdSql.checkData(i, 0, self.today_ts_trans)
     def today_check_ntb(self):
         for time_unit in self.db_percision:
             

tests/system-test/2-query/cast.py

@@ -65,7 +65,6 @@ class TDTestCase:
         data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
         tdSql.query(f"select c1  from {self.dbname}.t1")
         data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
-
         tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes")
 
         tdSql.query(f"select cast(c1 as bigint) as b from {self.dbname}.ct4")
@@ -108,12 +107,12 @@ class TDTestCase:
 
         tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1")
         for i in range(len(data_t1_c1)):
-            if data_ct4_c1[i] is None:
+            if data_t1_c1[i] is None:
                 tdSql.checkData( i, 0 , None )
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000)
+                date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c1[i]/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -225,12 +224,12 @@ class TDTestCase:
 
         tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1")
         for i in range(len(data_t1_c3)):
-            if data_ct4_c3[i] is None:
+            if data_t1_c3[i] is None:
                 tdSql.checkData( i, 0 , None )
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000)
+                date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c3[i]/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -282,12 +281,12 @@ class TDTestCase:
 
         tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1")
         for i in range(len(data_t1_c4)):
-            if data_ct4_c4[i] is None:
+            if data_t1_c4[i] is None:
                 tdSql.checkData( i, 0 , None )
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000)
+                date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c4[i]/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -329,7 +328,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
         tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1")
@@ -339,7 +338,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -385,7 +384,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -398,7 +397,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 
@@ -439,7 +438,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
         tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1")
@@ -449,7 +448,7 @@ class TDTestCase:
             else:
                 utc_zone = datetime.timezone.utc
                 utc_8 = datetime.timezone(datetime.timedelta(hours=8))
-                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i])/1000)
                 date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
                 tdSql.checkData( i, 0, date_data)
 

tests/system-test/2-query/json_tag.py

@@ -301,7 +301,7 @@ class TDTestCase:
 
         # where json value is not exist
         tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag1' is null")
-        tdSql.checkData(0, 0, 'jsons1_9')
+        # tdSql.checkData(0, 0, 'jsons1_9')
         tdSql.checkRows(2)
         tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag4' is null")
         tdSql.checkRows(9)

tests/system-test/2-query/limit.py

@@ -140,7 +140,7 @@ class TDTestCase:
         val2 = paraDict["ctbNum"] - 1
         # select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0
         sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(1)
         tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@@ -152,7 +152,7 @@ class TDTestCase:
                 
         # select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0        
         sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"])
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@@ -164,7 +164,7 @@ class TDTestCase:
         
         # select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1
         sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"])
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(1)
         
@@ -179,18 +179,18 @@ class TDTestCase:
         tb = paraDict["ctbPrefix"] + '0'
         # select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1
         sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(10)
-        tdSql.checkData(0, 0, "18-09-17 09:05:00.000")
+        tdSql.checkData(0, 0, "2018-09-17 09:05:00.000")
         tdSql.checkData(0, 1, -1)
         tdSql.checkData(1, 1, 1)
-        tdSql.checkData(9, 0, "18-09-17 09:50:00.000")
+        tdSql.checkData(9, 0, "2018-09-17 09:50:00.000")
         tdSql.checkData(9, 1, 5)
-  
+        
         tb5 = paraDict["ctbPrefix"] + '5'
         sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(10)
         tdSql.checkData(0, 0, -1)
@@ -206,22 +206,22 @@ class TDTestCase:
         limit = paraDict["rowsPerTbl"]
         offset = limit / 2
         sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(limit)
         tdSql.checkData(0, 1, 0)
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(8200)
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(10)
         tdSql.checkData(0, 0, 5)
@@ -231,7 +231,7 @@ class TDTestCase:
 
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(10)
         tdSql.checkData(0, 0, -1000)
@@ -240,13 +240,13 @@ class TDTestCase:
         tdSql.checkData(3, 0, 2)
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(9998)
 
 
         sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;"
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(0)
 
@@ -254,7 +254,7 @@ class TDTestCase:
         limit = paraDict["rowsPerTbl"]
         offset = limit / 2
         sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(limit)
         tdSql.checkData(0, 1, 0)
@@ -272,7 +272,7 @@ class TDTestCase:
         limit = paraDict["rowsPerTbl"]
         offset = limit / 2
         sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(limit)
 
@@ -280,7 +280,7 @@ class TDTestCase:
         limit = paraDict["rowsPerTbl"]
         offset = limit / 2 + 10
         sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(limit)
         tdSql.checkData(0, 1, 5)
@@ -298,7 +298,7 @@ class TDTestCase:
         limit = paraDict["rowsPerTbl"]
         offset = limit * 2 - 11
         sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(10)
         tdSql.checkData(0, 1, -1)
@@ -314,7 +314,7 @@ class TDTestCase:
         ### [TBASE-350]
         ## stb + interval + fill + group by + limit offset
         sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10"
-        print("====sql:%s"%(sqlStr))
+        tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(2)
 
@@ -322,7 +322,7 @@ class TDTestCase:
         offset = paraDict["rowsPerTbl"] * 2
         offset = offset - 2
         sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000  partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset)
-        print("====sql:%s"%(sqlStr))
+        # tdLog.info("====sql:%s"%(sqlStr))
         tdSql.query(sqlStr)
         tdSql.checkRows(1)
         tdSql.checkData(0, 0, 9)
@@ -339,7 +339,7 @@ class TDTestCase:
         tdLog.printNoPrefix("======== test case 1 end ...... ")
 
     def run(self):
-        tdSql.prepare()
+        # tdSql.prepare()
         self.prepareTestEnv()
         self.tmqCase1()
 

tests/system-test/2-query/timeerror.py

@@ -0,0 +1,69 @@
+import taos
+import sys
+from time import sleep
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        rows = 10
+
+        tdSql.execute("create database dbus PRECISION 'us' ")
+        tdSql.execute("create database dbns PRECISION 'ns' ")
+        tdSql.execute("create database dbms PRECISION 'ms' ")
+        dball = ["dbns","dbus","dbms"]
+        for i in range(len(dball)):
+            dbname = dball[i]
+            stb = f"{dbname}.stb1"
+            if  i == 0  :
+                qts = 1678883666951061471
+                qts1 = 1678883666951061471
+
+                qtime = "2023-03-15 20:34:26.951061471"
+            elif  i == 1 :
+                qts = 1678883666951061
+                qts1 = 1678883666951061
+
+                qtime = "2023-03-15 20:34:26.951061"
+            else:
+                qts = 1678883666951
+                qts1 = 1678883666953
+
+                qtime = "2023-03-15 20:34:26.951"
+
+            tdSql.execute(f"use {dbname}")
+            tdLog.printNoPrefix("==========step1:create table")
+            tdSql.execute(
+                f'''create table if not exists {stb}
+                (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
+                tags(location binary(64), type int, isused bool , family nchar(64))'''
+            )
+            tdSql.execute(f"create table {dbname}.t1 using {stb} tags('beijing', 1, 1, 'nchar1')")
+            tdSql.execute(f"create table {dbname}.t2 using {stb} tags('shanghai', 2, 0, 'nchar2')")
+            tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')")
+
+            tdLog.printNoPrefix("==========step2:insert data")
+            tdSql.execute(
+                f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)"
+            )
+        
+            tdLog.printNoPrefix("==========step3:query timestamp type")
+            tdSql.query(f"select ts from {dbname}.t3 limit 1")
+            tdSql.checkData(0,0,qtime)
+            tdSql.checkData(0,0,qts)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())