Merge remote-tracking branch 'origin/3.0' into feature/shm

include/common/tcommon.h

@@ -53,12 +53,12 @@ typedef struct {
 
 typedef struct SColumnDataAgg {
   int16_t colId;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
   int16_t maxIndex;
   int16_t minIndex;
   int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
 } SColumnDataAgg;
 
 typedef struct SDataBlockInfo {
@@ -193,20 +193,19 @@ typedef struct SColumn {
   uint8_t scale;
 } SColumn;
 
-typedef struct SLimit {
+typedef struct STableBlockDistInfo {
-  int64_t limit;
+  uint16_t  rowSize;
-  int64_t offset;
+  uint16_t  numOfFiles;
-} SLimit;
+  uint32_t  numOfTables;
-
+  uint64_t  totalSize;
-typedef struct SOrder {
+  uint64_t  totalRows;
-  uint32_t order;
+  int32_t   maxRows;
-  SColumn  col;
+  int32_t   minRows;
-} SOrder;
+  int32_t   firstSeekTimeUs;
-
+  uint32_t  numOfRowsInMemTable;
-typedef struct SGroupbyExpr {
+  uint32_t  numOfSmallBlocks;
-  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
+  SArray   *dataBlockInfos;
-  bool    groupbyTag;  // group by tag or column
+} STableBlockDistInfo;
-} SGroupbyExpr;
 
 enum {
   FUNC_PARAM_TYPE_VALUE = 0x1,
@@ -241,15 +240,6 @@ typedef struct SExprInfo {
   struct tExprNode*     pExpr;
 } SExprInfo;
 
-typedef struct SStateWindow {
-  SColumn col;
-} SStateWindow;
-
-typedef struct SSessionWindow {
-  int64_t gap;  // gap between two session window(in microseconds)
-  SColumn col;
-} SSessionWindow;
-
 #define QUERY_ASC_FORWARD_STEP 1
 #define QUERY_DESC_FORWARD_STEP -1
 

include/common/tdataformat.h

@@ -135,8 +135,8 @@ typedef struct {
 #define TD_VTYPE_PARTS 4  // 8 bits / TD_VTYPE_BITS = 4
 #define TD_VTYPE_OPTR  3  // TD_VTYPE_PARTS - 1, utilize to get remainder
 
-#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
+#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS))
-#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
+#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8))
 
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
@@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
 }
 #endif
 // ----------------- Data column structure
+// SDataCol arrangement: data => bitmap => dataOffset
 typedef struct SDataCol {
   int8_t          type;        // column type
   uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM

include/common/trow.h

@@ -147,20 +147,20 @@ typedef struct {
 
 typedef struct {
   // basic info
-  int8_t  rowType;
+  int8_t       rowType;
-  int16_t sver;
+  schema_ver_t sver;
-  STSRow *pBuf;
+  STSRow      *pBuf;
 
   // extended info
-  int32_t flen;
+  int32_t  flen;
-  int16_t nBoundCols;
+  col_id_t nBoundCols;
-  int16_t nCols;
+  col_id_t nCols;
-  int16_t nBitmaps;
+  col_id_t nBitmaps;
-  int16_t nBoundBitmaps;
+  col_id_t nBoundBitmaps;
-  int32_t offset;
+  int32_t  offset;
-  void   *pBitmap;
+  void    *pBitmap;
-  void   *pOffset;
+  void    *pOffset;
-  int32_t extendedRowSize;
+  int32_t  extendedRowSize;
 } SRowBuilder;
 
 #define TD_ROW_HEAD_LEN (sizeof(STSRow))
@@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t
   }
 #ifdef TD_SUPPORT_BITMAP
   // the primary TS key is stored separatedly
-  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+  pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
   if (nBoundCols > 0) {
-    pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
+    pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
   } else {
     pBuilder->nBoundBitmaps = 0;
   }

include/libs/function/function.h

@@ -52,7 +52,12 @@ typedef struct SFuncExecFuncs {
   FExecFinalize finalize;
 } SFuncExecFuncs;
 
-#define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
+typedef struct SFileBlockInfo {
+  int32_t numBlocksOfStep;
+} SFileBlockInfo;
+
+#define TSDB_BLOCK_DIST_STEP_ROWS 8
+#define MAX_INTERVAL_TIME_WINDOW  1000000  // maximum allowed time windows in final results
 
 #define FUNCTION_TYPE_SCALAR       1
 #define FUNCTION_TYPE_AGG          2
@@ -101,10 +106,6 @@ typedef struct SFuncExecFuncs {
 #define FUNCTION_DERIVATIVE   32
 #define FUNCTION_BLKINFO      33
 
-#define FUNCTION_HISTOGRAM    34
-#define FUNCTION_HLL          35
-#define FUNCTION_MODE         36
-#define FUNCTION_SAMPLE       37
 
 #define FUNCTION_COV          38
 

include/libs/function/functionMgt.h

@@ -72,10 +72,15 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ATAN,
 
   // string function
-  FUNCTION_TYPE_CHAR_LENGTH = 1500,
+  FUNCTION_TYPE_LENGTH = 1500,
+  FUNCTION_TYPE_CHAR_LENGTH,
   FUNCTION_TYPE_CONCAT,
   FUNCTION_TYPE_CONCAT_WS,
-  FUNCTION_TYPE_LENGTH,
+  FUNCTION_TYPE_LOWER,
+  FUNCTION_TYPE_UPPER,
+  FUNCTION_TYPE_LTRIM,
+  FUNCTION_TYPE_RTRIM,
+  FUNCTION_TYPE_SUBSTR,
 
   // conversion function
   FUNCTION_TYPE_CAST = 2000,
@@ -127,6 +132,7 @@ bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
+bool fmIsPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 

include/libs/nodes/nodes.h

@@ -213,16 +213,16 @@ typedef enum EDealRes {
 } EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
-void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext);
+void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext);
-void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext);
+void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext);
-void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
+void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
-void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
+void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
 
 typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext);
-void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 
 bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
 

include/libs/nodes/querynodes.h

@@ -79,6 +79,7 @@ typedef struct SValueNode {
   char* literal;
   bool isDuration;
   bool translate;
+  bool genByCalc;
   union {
     bool b;
     int64_t i;

include/libs/scalar/scalar.h

@@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
 int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
 
+/* Math functions */
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@@ -58,6 +59,17 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+/* String functions */
+int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 
 int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

source/client/src/clientMain.c

@@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) {
 }
 
 TAOS_ROW *taos_result_block(TAOS_RES *res) {
-  // TODO
+  SRequestObj* pRequest = (SRequestObj*) res;
-  return NULL;
+  if (pRequest == NULL) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
+
+  if (taos_is_update_query(res)) {
+    return NULL;
+  }
+
+  return &pRequest->body.resInfo.row;
 }
 
 // todo intergrate with tDataTypes
@@ -313,7 +322,9 @@ int taos_affected_rows(TAOS_RES *res) {
   return pResInfo->numOfRows;
 }
 
-int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }
+int taos_result_precision(TAOS_RES *res) {
+  return TSDB_TIME_PRECISION_MILLI;
+}
 
 int taos_select_db(TAOS *taos, const char *db) {
   STscObj *pObj = (STscObj *)taos;

source/client/src/tmq.c

@@ -79,6 +79,7 @@ struct tmq_t {
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
   int8_t         epStatus;
+  int32_t        epSkipCnt;
   int32_t        waitingRequest;
   int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
@@ -107,6 +108,7 @@ typedef struct {
   // connection info
   int32_t vgId;
   int32_t vgStatus;
+  int64_t skipCnt;
   SEpSet  epSet;
 } SMqClientVg;
 
@@ -138,6 +140,7 @@ typedef struct {
   tmq_t*          tmq;
   SMqClientVg*    pVg;
   int32_t         epoch;
+  int32_t         vgId;
   tsem_t          rspSem;
   tmq_message_t** msg;
   int32_t         sync;
@@ -313,6 +316,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
   pTmq->epStatus = 0;
+  pTmq->epSkipCnt = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -348,6 +352,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
+  pTmq->epSkipCnt = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -834,7 +840,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqClientVg*    pVg = pParam->pVg;
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
-    tscWarn("msg discard, code:%x", code);
+    tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code);
     goto CREATE_MSG_FAIL;
   }
 
@@ -842,13 +848,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
     /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
-    tsem_post(&tmq->rspSem);
+    /*tsem_post(&tmq->rspSem);*/
-    tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
+    tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
-    tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
+    tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
   } else {
     atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
@@ -892,29 +898,29 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
+  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset,
            pRsp->msg.rspOffset);
 
   pRsp->vg = pParam->pVg;
   taosWriteQitem(tmq->mqueue, pRsp);
   atomic_add_fetch_32(&tmq->readyRequest, 1);
-  tsem_post(&tmq->rspSem);
+  /*tsem_post(&tmq->rspSem);*/
   return 0;
 
 CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
-  tsem_post(&tmq->rspSem);
+  /*tsem_post(&tmq->rspSem);*/
   return code;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
-  tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch);
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
   char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
+  tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet);
   SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic));
   if (newTopics == NULL) {
     return false;
@@ -932,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
     taosHashClear(pHash);
     topic.topicName = strdup(pTopicEp->topic);
 
+    tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName);
     int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
     for (int32_t j = 0; j < topicNumCur; j++) {
       // find old topic
       SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
       if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) {
         int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs);
+        tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur);
         if (vgNumCur == 0) break;
         for (int32_t k = 0; k < vgNumCur; k++) {
           SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
           sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
-          tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);
+          tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey);
           taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
         }
         break;
@@ -956,18 +964,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
       int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
       int64_t  offset = pVgEp->offset;
-      tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);
+      tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset);
       if (pOffset != NULL) {
         offset = *pOffset;
-        tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);
+        tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey);
       }
-      tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);
+      tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset);
       SMqClientVg clientVg = {
           .pollCnt = 0,
           .currentOffset = offset,
           .vgId = pVgEp->vgId,
           .epSet = pVgEp->epSet,
           .vgStatus = TMQ_VG_STATUS__IDLE,
+          .skipCnt = 0,
       };
       taosArrayPush(topic.vgs, &clientVg);
       set = true;
@@ -984,9 +993,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
-  tscDebug("consumer %ld recv ep", tmq->consumerId);
   if (code != 0) {
-    tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
+    tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
     goto END;
   }
 
@@ -995,6 +1003,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // Epoch will only increase when received newer epoch ep msg
   SMqRspHead* head = pMsg->pData;
   int32_t     epoch = atomic_load_32(&tmq->epoch);
+  tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
   if (head->epoch <= epoch) {
     goto END;
   }
@@ -1019,7 +1028,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp);
 
     taosWriteQitem(tmq->mqueue, pRsp);
-    tsem_post(&tmq->rspSem);
+    /*tsem_post(&tmq->rspSem);*/
   }
 
 END:
@@ -1033,9 +1042,11 @@ END:
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
   if (epStatus == 1) {
-    tscDebug("consumer %ld skip ask ep", tmq->consumerId);
+    int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
-    return 0;
+    tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
+    if (epSkipCnt < 5000) return 0;
   }
+  atomic_store_32(&tmq->epSkipCnt, 0);
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
   SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
@@ -1221,24 +1232,34 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
-        tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId);
+        int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1);
+        tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt);
         continue;
+#if 0
+        if (skipCnt < 30000) {
+          continue;
+        } else {
+        tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId);
+        }
+#endif
       }
+      atomic_store_64(&pVg->skipCnt, 0);
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
       SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
       if (pParam == NULL) {
         taosMemoryFree(pReq);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
       pParam->tmq = tmq;
       pParam->pVg = pVg;
+      pParam->vgId = pVg->vgId;
       pParam->epoch = tmq->epoch;
       pParam->sync = 0;
 
@@ -1247,7 +1268,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
         taosMemoryFree(pReq);
         taosMemoryFree(pParam);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
 
@@ -1265,7 +1286,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset);
+      tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1379,7 +1400,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
     tmqAskEp(tmq, false);
     tmqPollImpl(tmq, blocking_time);
 
-    tsem_wait(&tmq->rspSem);
+    /*tsem_wait(&tmq->rspSem);*/
 
     rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
     if (rspMsg) {

source/common/src/tdataformat.c

@@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   spaceNeeded += (int)nBitmapBytes;
   // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
   // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
-  spaceNeeded += TYPE_BYTES[pCol->type];
+  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
+  // the additional space
 #endif
 
   if (pCol->spaceSize < spaceNeeded) {
@@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
     }
   }
 #ifdef TD_SUPPORT_BITMAP
+
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
     pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes);
@@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row)
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
   if (isAllRowsNull(pCol)) return true;
-  for (int i = 0; i < nEle; i++) {
+  for (int i = 0; i < nEle; ++i) {
     if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
   }
   return true;
@@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
 static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
-    for (int i = 0; i < nEle; i++) {
+    for (int i = 0; i < nEle; ++i) {
       dataColSetNullAt(pCol, i);
     }
   } else {
@@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) {
   // char *tptr = (char *)(pCol->pData);
 
   VarDataOffsetT offset = 0;
-  for (int i = 0; i < nEle; i++) {
+  for (int i = 0; i < nEle; ++i) {
     pCol->dataOff[i] = offset;
     offset += varDataTLen(tptr);
     tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
@@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
       tdFreeDataCols(pCols);
       return NULL;
     }
+#if 0  // no need as calloc used
     int i;
     for (i = 0; i < maxCols; i++) {
       pCols->cols[i].spaceSize = 0;
@@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
     }
+#endif
   }
 
   return pCols;
@@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
     void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
     if (ptr == NULL) return -1;
     pCols->cols = ptr;
-    for (i = oldMaxCols; i < pCols->maxCols; i++) {
+    for (i = oldMaxCols; i < pCols->maxCols; ++i) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
+      pCols->cols[i].pBitmap = NULL;
       pCols->cols[i].spaceSize = 0;
     }
   }
+#if 0
+  tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...)
+#endif
 
-  tdResetDataCols(pCols);
+  pCols->numOfRows = 0;
   pCols->numOfCols = schemaNCols(pSchema);
 
-  for (i = 0; i < schemaNCols(pSchema); i++) {
+  for (i = 0; i < schemaNCols(pSchema); ++i) {
     dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints);
   }
 
@@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     if (pCols->cols) {
       int maxCols = pCols->maxCols;
-      for (i = 0; i < maxCols; i++) {
+      for (i = 0; i < maxCols; ++i) {
         SDataCol *pCol = &pCols->cols[i];
         taosMemoryFreeClear(pCol->pData);
       }
@@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 void tdResetDataCols(SDataCols *pCols) {
   if (pCols != NULL) {
     pCols->numOfRows = 0;
-    for (int i = 0; i < pCols->maxCols; i++) {
+    for (int i = 0; i < pCols->maxCols; ++i) {
       dataColReset(pCols->cols + i);
     }
   }

source/common/src/tmsg.c

@@ -367,7 +367,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
       buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
       pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
-      for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
         buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
@@ -376,7 +376,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       }
       buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
@@ -408,7 +408,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
       buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
       pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
-      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
         buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);

source/common/src/trow.c

@@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
           memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
         }
         if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
-          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints);
+          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows);
           memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
         }
       }

source/dnode/mgmt/vm/vmHandle.c

@@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
   pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
+  pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
   pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
   pCfg->walCfg.level = pCreate->walLevel;
@@ -70,6 +71,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
   if (pVnode != NULL) {
+    tFreeSCreateVnodeReq(&createReq);
     dDebug("vgId:%d, already exist", createReq.vgId);
     vmReleaseVnode(pMgmt, pVnode);
     terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED;
@@ -88,12 +90,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   vnodeCfg.dbId = wrapperCfg.dbUid;
   SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
   if (pImpl == NULL) {
+    tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
     return -1;
   }
 
   int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
   if (code != 0) {
+    tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
     vnodeClose(pImpl);
     vnodeDestroy(wrapperCfg.path);
@@ -103,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
 
   code = vmWriteVnodesToFile(pMgmt);
   if (code != 0) {
+    tFreeSCreateVnodeReq(&createReq);
     vnodeClose(pImpl);
     vnodeDestroy(wrapperCfg.path);
     terrno = code;

source/dnode/mnode/impl/src/mndConsumer.c

@@ -160,10 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
-  pOldConsumer->epoch++;
-
-  // TODO handle update
   /*taosWLockLatch(&pOldConsumer->lock);*/
+  atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   /*taosWUnLockLatch(&pOldConsumer->lock);*/
 
   return 0;

source/dnode/mnode/impl/src/mndDb.c

@@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {}
 static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER,
+                              sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE);
   if (pRaw == NULL) goto DB_ENCODE_OVER;
 
   int32_t dataPos = 0;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     consumerEp.consumerId = -1;
     consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
+    
+    mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
+
     int32_t msgLen;
     if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -212,19 +212,24 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
     terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
     return -1;
   }
+  //TODO add lock
   ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
+  int32_t           serverEpoch = pConsumer->epoch;
 
   // TODO
   int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);
-  mTrace("try to get sub ep, old val: %d", hbStatus);
+  mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, hbStatus);
   atomic_store_32(&pConsumer->hbStatus, 0);
   /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/
   /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/
   /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/
 
   strcpy(rsp.cgroup, pReq->cgroup);
-  if (epoch != pConsumer->epoch) {
+  if (epoch != serverEpoch) {
-    mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch);
+    mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, serverEpoch);
+    mDebug("consumer %ld try r lock", consumerId);
+    taosRLockLatch(&pConsumer->lock);
+    mDebug("consumer %ld r locked", consumerId);
     SArray *pTopics = pConsumer->currentTopics;
     int32_t sz = taosArrayGetSize(pTopics);
     rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
@@ -238,7 +243,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
         if (consumerId == pSubConsumer->consumerId) {
           int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
-          mInfo("topic %s has %d vg", topicName, pConsumer->epoch);
+          mInfo("topic %s has %d vg", topicName, serverEpoch);
           SMqSubTopicEp topicEp;
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
@@ -264,6 +269,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
       }
       mndReleaseSubscribe(pMnode, pSub);
     }
+    taosRUnLockLatch(&pConsumer->lock);
+    mDebug("consumer %ld r unlock", consumerId);
   }
   int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
   void   *buf = rpcMallocCont(tlen);
@@ -272,7 +279,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
     return -1;
   }
   ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
-  ((SMqRspHead *)buf)->epoch = pConsumer->epoch;
+  ((SMqRspHead *)buf)->epoch = serverEpoch;
   ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
 
   void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
@@ -395,7 +402,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
     SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
     taosMemoryFreeClear(pRebSub->key);
 
-    mInfo("mq rebalance subscription: %s", pSub->key);
+    mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg));
 
     // remove lost consumer
     for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) {
@@ -442,6 +449,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
         }
 
         SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
+        mDebug("consumer %ld try w lock", pRebConsumer->consumerId);
+        taosWLockLatch(&pRebConsumer->lock);
+        mDebug("consumer %ld w locked", pRebConsumer->consumerId);
         int32_t         status = atomic_load_32(&pRebConsumer->status);
         if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
             (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
@@ -462,6 +472,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
           sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
           mndTransAppendCommitlog(pTrans, pConsumerRaw);
         }
+        taosWUnLockLatch(&pRebConsumer->lock);
+        mDebug("consumer %ld w unlock", pRebConsumer->consumerId);
         mndReleaseConsumer(pMnode, pRebConsumer);
       }
 

source/dnode/vnode/inc/tsdb.h

@@ -55,13 +55,14 @@ typedef struct STsdbCfg {
   int8_t   precision;
   int8_t   update;
   int8_t   compression;
-  uint64_t lruCacheSize;
   int32_t  daysPerFile;
   int32_t  minRowsPerFileBlock;
   int32_t  maxRowsPerFileBlock;
   int32_t  keep;
   int32_t  keep1;
   int32_t  keep2;
+  uint64_t lruCacheSize;
+  SArray  *retentions;
 } STsdbCfg;
 
 // query condition to build multi-table data block iterator
@@ -171,6 +172,8 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo
 
 tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
 
+int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo);
+
 bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle);
 
 /**

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -112,10 +112,10 @@ typedef struct {
 #else
 typedef struct {
   int16_t  colId;
-  uint8_t  bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint16_t type : 6;
-  uint8_t  reserve : 7;
+  uint16_t blen : 10;   // bitmap length(TODO: full UT for the bitmap compress of various data input)
-  uint8_t  type;
+  uint32_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
-  int32_t  len;
+  uint32_t len : 31;    // data length + bitmap length
   uint32_t offset;
 } SBlockColV0;
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest);
   int32_t  keyLen = 0;
   int32_t  nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
-  int32_t  tBitmaps = 0;
+  // int32_t  tBitmaps = 0;
 
   for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) {
     // All not NULL columns finish
@@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
 
 #ifdef TD_SUPPORT_BITMAP
     int32_t tBitmaps = 0;
+    int32_t tBitmapsLen = 0;
     if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+      tBitmaps = nBitmaps;
+#if 0
       if (IS_VAR_DATA_TYPE(pDataCol->type)) {
         tBitmaps = nBitmaps;
         tlen += tBitmaps;
@@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
         tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
         tlen += tBitmaps * TYPE_BYTES[pDataCol->type];
       }
+#endif
       // move bitmap parts ahead
       // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove
-      memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
+      // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
     }
 #endif
 
-    void *tptr;
+    void *tptr, *bptr;
 
     // Make room
-    if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) {
+    if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) {
       return -1;
     }
     pBlockData = (SBlockData *)(*ppBuf);
@@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
 
     // Compress or just copy
     if (pCfg->compression) {
+#if 0
       flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr,
                                                       tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
                                                       tlen + COMP_OVERFLOW_BYTES);
+#endif
+      flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
+                                                      tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
+                                                      tlen + COMP_OVERFLOW_BYTES);
+      if (tBitmaps > 0) {
+        bptr = POINTER_SHIFT(pBlockData, lsize + flen);
+        tBitmapsLen =
+            tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES,
+                              pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES);
+        TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES)));
+        flen += tBitmapsLen;
+      }
     } else {
       flen = tlen;
       memcpy(tptr, pDataCol->pData, flen);
+      if (tBitmaps > 0) {
+        bptr = POINTER_SHIFT(pBlockData, lsize + flen);
+        memcpy(bptr, pDataCol->pBitmap, tBitmaps);
+        tBitmapsLen = tBitmaps;
+        flen += tBitmapsLen;
+      }
     }
 
     // Add checksum
     ASSERT(flen > 0);
+    ASSERT(tBitmapsLen <= 1024);
     flen += sizeof(TSCKSUM);
     taosCalcChecksumAppend(0, (uint8_t *)tptr, flen);
     tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
 
     if (ncol != 0) {
       tsdbSetBlockColOffset(pBlockCol, toffset);
-      pBlockCol->len = flen;
+      pBlockCol->len = flen;  // data + bitmaps
+      pBlockCol->blen = tBitmapsLen;
       ++tcol;
     } else {
       keyLen = flen;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -14,7 +14,7 @@
  */
 
 #include "tsdbDef.h"
-#include <tdatablock.h>
+#include "tdatablock.h"
 #include "os.h"
 #include "talgo.h"
 #include "tcompare.h"
@@ -31,6 +31,7 @@
 #include "tlosertree.h"
 #include "tsdbDef.h"
 #include "tmsg.h"
+#include "tsdbCommit.h"
 
 #define EXTRA_BYTES 2
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)
@@ -209,34 +210,34 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load
   return pLocalIdList;
 }
 
-//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
+int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
-//  STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
+  STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
-//
+
-//  int64_t rows = 0;
+  int64_t rows = 0;
-//  STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable;
+  STsdbMemTable* pMemTable = NULL;//pTsdbReadHandle->pMemTable;
-//  if (pMemTable == NULL) { return rows; }
+  if (pMemTable == NULL) { return rows; }
-//
+
-////  STableData* pMem  = NULL;
+//  STableData* pMem  = NULL;
-////  STableData* pIMem = NULL;
+//  STableData* pIMem = NULL;
-//
+
-////  SMemTable* pMemT = pMemRef->snapshot.mem;
+//  SMemTable* pMemT = pMemRef->snapshot.mem;
-////  SMemTable* pIMemT = pMemRef->snapshot.imem;
+//  SMemTable* pIMemT = pMemRef->snapshot.imem;
-//
+
-//  size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
+  size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
-//  for (int32_t i = 0; i < size; ++i) {
+  for (int32_t i = 0; i < size; ++i) {
-//    STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
+    STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
-//
+
-////    if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
+//    if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
-////      pMem = pMemT->tData[pCheckInfo->tableId];
+//      pMem = pMemT->tData[pCheckInfo->tableId];
-////      rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0;
+//      rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0;
-////    }
+//    }
-////    if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) {
+//    if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) {
-////      pIMem = pIMemT->tData[pCheckInfo->tableId];
+//      pIMem = pIMemT->tData[pCheckInfo->tableId];
-////      rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0;
+//      rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0;
-////    }
+//    }
-//  }
+  }
-//  return rows;
+  return rows;
-//}
+}
 
 static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) {
   size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
@@ -2261,12 +2262,13 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) {
   cur->mixBlock       = false;
   cur->blockCompleted = false;
 }
-#if 0
+
-int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) {
+int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo) {
   STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle;
 
   pTableBlockInfo->totalSize = 0;
   pTableBlockInfo->totalRows = 0;
+
   STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);
 
   // find the start data block in file
@@ -2284,9 +2286,11 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
   int32_t     code = TSDB_CODE_SUCCESS;
   int32_t     numOfBlocks = 0;
   int32_t     numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
-  int         defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
+  int         defaultRows = 4096;//TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
   STimeWindow win = TSWINDOW_INITIALIZER;
 
+  bool ascTraverse = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+
   while (true) {
     numOfBlocks = 0;
     tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
@@ -2299,8 +2303,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
     tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
 
     // current file are not overlapped with query time window, ignore remain files
-    if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
+    if ((ascTraverse && win.skey > pTsdbReadHandle->window.ekey) || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)) {
-    (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
       tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
       tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle,
                 pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
@@ -2342,19 +2345,26 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
 
         int32_t numOfRows = pBlock[j].numOfRows;
         pTableBlockInfo->totalRows += numOfRows;
-        if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows;
+        if (numOfRows > pTableBlockInfo->maxRows) {
-        if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows;
+          pTableBlockInfo->maxRows = numOfRows;
-        if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1;
+        }
-        int32_t  stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS;
+
-        SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex);
+        if (numOfRows < pTableBlockInfo->minRows) {
-        blockInfo->numBlocksOfStep++;
+          pTableBlockInfo->minRows = numOfRows;
+        }
+
+        if (numOfRows < defaultRows) {
+          pTableBlockInfo->numOfSmallBlocks += 1;
+        }
+//        int32_t  stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS;
+//        SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex);
+//        blockInfo->numBlocksOfStep++;
       }
     }
   }
 
   return code;
 }
-#endif
 
 static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) {
   STsdbFS*       pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh);
 static void tsdbResetReadFile(SReadH *pReadh);
 static int  tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
 static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
-static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
-                                         int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
+                                         int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize);
-                                         int bufferSize);
 static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
                                       int numOfColIds);
 static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
@@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
     if (dcol != 0 && ccol >= pBlockData->numOfCols) {
       // Set current column as NULL and forward
       dataColReset(pDataCol);
-      dcol++;
+      ++dcol;
       continue;
     }
 
@@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       TD_SET_COL_ROWS_NORM(pDataCol);
     }
 
-    int32_t tBitmaps = 0;
+    // int32_t tBitmaps = 0;
     int32_t tLenBitmap = 0;
     if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+      tLenBitmap = nBitmaps;
+#if 0
       if (IS_VAR_DATA_TYPE(pDataCol->type)) {
         tBitmaps = nBitmaps;
         tLenBitmap = tBitmaps;
@@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
         tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
         tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
       }
+#endif
     }
 
     if (tcolId == pDataCol->colId) {
       if (pBlock->algorithm == TWO_STAGE_COMP) {
-        int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+        int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
         if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
       }
 
-      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
+      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen,
-                                       pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints,
-                                       (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
+                                       TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
         tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
         return -1;
@@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
   return 0;
 }
 
-static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
-                                        int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
+                                        int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) {
-                                        int bufferSize) {
   if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
     return -1;
@@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
   if (comp) {
     // Need to decompress
     int tlen =
-        (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps,
+        (*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows,
                                                    pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
     if (tlen <= 0) {
-      tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
+      tsdbError(
-                len, comp, numOfRows, maxPoints, bufferSize);
+          "Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
+          (int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize);
       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
       return -1;
     }
     pDataCol->len = tlen;
+
+    if (numOfBitmaps > 0) {
+      tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps,
+                                 pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize);
+      if (tlen <= 0) {
+        tsdbError(
+            "Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d "
+            "bufferSize:%d",
+            bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize);
+        terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+        return -1;
+      }
+      // pDataCol->blen = tlen;
+    }
   } else {
     // No need to decompress, just memcpy it
-    pDataCol->len = len - sizeof(TSCKSUM);
+    pDataCol->len = len - bitmapLen - sizeof(TSCKSUM);
     memcpy(pDataCol->pData, content, pDataCol->len);
+    if (numOfBitmaps > 0) {
+      // pDataCol->blen = bitmapLen;
+      memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen);
+    }
   }
 
+#if 0
   if (lenOfBitmaps > 0) {
     pDataCol->len -= lenOfBitmaps;
 
@@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
   } else if (IS_VAR_DATA_TYPE(pDataCol->type)) {
     dataColSetOffset(pDataCol, numOfRows);
   }
-
+#endif
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    dataColSetOffset(pDataCol, numOfRows);
+  }
   return 0;
 }
 
@@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
   ASSERT(pDataCol->colId == pBlockCol->colId);
 
-  STsdb *   pRepo = TSDB_READ_REPO(pReadh);
+  STsdb    *pRepo = TSDB_READ_REPO(pReadh);
   STsdbCfg *pCfg = REPO_CFG(pRepo);
 
-  int     nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
+  int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
-  int32_t tBitmaps = 0;
+  // int32_t tBitmaps = 0;
   int32_t tLenBitmap = 0;
 
   if (!TD_COL_ROWS_NORM(pBlockCol)) {
+    tLenBitmap = nBitmaps;
+#if 0
     if (IS_VAR_DATA_TYPE(pDataCol->type)) {
       tBitmaps = nBitmaps;
       tLenBitmap = tBitmaps;
@@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
       tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
       tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
     }
+#endif
   }
 
-  int       tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES;
+  int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
 
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
@@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
     return -1;
   }
 
-  if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
+  if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm,
-                                   tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
+                                   pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
                                    (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
     tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
               pBlockCol->colId, offset);

source/libs/executor/inc/executorimpl.h

@@ -128,6 +128,11 @@ typedef struct {
   int64_t sumRunTimes;
 } SOperatorProfResult;
 
+typedef struct SLimit {
+  int64_t limit;
+  int64_t offset;
+} SLimit;
+
 typedef struct STaskCostInfo {
   int64_t created;
   int64_t start;
@@ -163,6 +168,11 @@ typedef struct SOperatorCostInfo {
   uint64_t execCost;
 } SOperatorCostInfo;
 
+typedef struct SOrder {
+  uint32_t order;
+  SColumn  col;
+} SOrder;
+
 // The basic query information extracted from the SQueryInfo tree to support the
 // execution of query in a data node.
 typedef struct STaskAttr {
@@ -196,7 +206,6 @@ typedef struct STaskAttr {
 
   STimeWindow    window;
   SInterval      interval;
-  SSessionWindow sw;
   int16_t        precision;
   int16_t        numOfOutput;
   int16_t        fillType;
@@ -206,13 +215,8 @@ typedef struct STaskAttr {
   int32_t       intermediateResultRowSize;  // intermediate result row size, in case of top-k query.
   int32_t       maxTableColumnWidth;
   int32_t       tagLen;  // tag value length of current query
-  SGroupbyExpr* pGroupbyExpr;
 
   SExprInfo* pExpr1;
-  SExprInfo* pExpr2;
-  int32_t    numOfExpr2;
-  SExprInfo* pExpr3;
-  int32_t    numOfExpr3;
 
   SColumnInfo*         tableCols;
   SColumnInfo*         tagColList;
@@ -220,8 +224,6 @@ typedef struct STaskAttr {
   int64_t*             fillVal;
 
   SSingleColumnFilterInfo* pFilterInfo;
-  //  SFilterInfo     *pFilters;
-
   void*           tsdb;
   STableGroupInfo tableGroupInfo;  // table <tid, last_key> list  SArray<STableKeyInfo>
   int32_t         vgId;
@@ -384,7 +386,7 @@ typedef struct SExchangeInfo {
 } SExchangeInfo;
 
 typedef struct STableScanInfo {
-  void*           pTsdbReadHandle;
+  void*           dataReader;
   int32_t         numOfBlocks;  // extract basic running information.
   int32_t         numOfSkipped;
   int32_t         numOfBlockStatis;
@@ -644,12 +646,13 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
 SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
                                        SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
+SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
+
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock,
                                       int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
                                           int32_t numOfOutput);
-SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
 
 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
 SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,

source/libs/executor/src/executorimpl.c

@@ -249,7 +249,6 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {}
 static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType,
                                   SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset);
 
-static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock);
 static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type,
                                           int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo,
                                           SAggSupporter* pAggSup);
@@ -287,44 +286,6 @@ static int compareRowData(const void* a, const void* b, const void* userData) {
   return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
 }
 
-static void sortGroupResByOrderList(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv,
-                                    SSDataBlock* pDataBlock) {
-  SArray* columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr);
-  size_t  size = taosArrayGetSize(columnOrderList);
-  taosArrayDestroy(columnOrderList);
-
-  if (size <= 0) {
-    return;
-  }
-
-  int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.colId;
-  if (orderId <= 0) {
-    return;
-  }
-
-  bool    found = false;
-  int16_t dataOffset = 0;
-
-  for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) {
-    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, j);
-    if (orderId == j) {
-      found = true;
-      break;
-    }
-
-    dataOffset += pColInfoData->info.bytes;
-  }
-
-  if (found == false) {
-    return;
-  }
-
-  int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type;
-
-  SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
-  taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support);
-}
-
 // setup the output buffer for each operator
 SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
   int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
@@ -375,17 +336,6 @@ static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput
   //  return (numOfSelectivity > 0 && hasTags);
 }
 
-static bool isProjQuery(STaskAttr* pQueryAttr) {
-  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
-    int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
-    if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) {
   if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) ||
       pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@@ -1411,15 +1361,10 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo,
 }
 
 static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos,
-                                              int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols,
+                                              int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) {
-                                              STimeWindow* win) {
+  bool  ascQuery = true;
-  STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
-  STaskAttr*       pQueryAttr = pRuntimeEnv->pQueryAttr;
-
-  bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
-
   TSKEY curTs = tsCols[pos];
-  TSKEY lastTs = *(TSKEY*)pRuntimeEnv->prevRow[0];
+  TSKEY lastTs = 0;//*(TSKEY*)pRuntimeEnv->prevRow[0];
 
   // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
   // start exactly from this point, no need to do interpolation
@@ -1434,27 +1379,24 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlF
     return true;
   }
 
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
+  int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
   TSKEY   prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step];
 
-  doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key,
+  doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP);
-                            RESULT_ROW_START_INTERP);
   return true;
 }
 
 static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex,
                                             SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
                                             STimeWindow* win) {
-  STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
+  int32_t order = TSDB_ORDER_ASC;
-  STaskAttr*       pQueryAttr = pRuntimeEnv->pQueryAttr;
+  int32_t numOfOutput = pOperatorInfo->numOfOutput;
-  int32_t          numOfOutput = pOperatorInfo->numOfOutput;
 
   TSKEY actualEndKey = tsCols[endRowIndex];
-
+  TSKEY key = order ? win->ekey : win->skey;
-  TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr) ? win->ekey : win->skey;
 
   // not ended in current data block, do not invoke interpolation
-  if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) {
+  if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) {
     setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP);
     return false;
   }
@@ -1465,7 +1407,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun
     return true;
   }
 
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
   int32_t nextRowIndex = endRowIndex + step;
   assert(nextRowIndex >= 0);
 
@@ -1667,10 +1609,9 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
 
   STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
   int32_t          numOfOutput = pOperatorInfo->numOfOutput;
-  STaskAttr*       pQueryAttr = pRuntimeEnv->pQueryAttr;
 
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
+  int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
-  bool    ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
+  bool    ascQuery = true;
 
   TSKEY* tsCols = NULL;
   if (pSDataBlock->pDataBlock != NULL) {
@@ -1683,7 +1624,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
   int32_t startPos = ascQuery ? 0 : (pSDataBlock->info.rows - 1);
   TSKEY   ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery);
 
-  STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr);
+  STimeWindow win = {0};//getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr);
   bool        masterScan = IS_MAIN_SCAN(pRuntimeEnv);
 
   SResultRow* pResult = NULL;
@@ -1699,7 +1640,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
       longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-    TSKEY ekey = reviseWindowEkey(pQueryAttr, &win);
+    TSKEY ekey = 0;//reviseWindowEkey(pQueryAttr, &win);
     //    forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey,
     //    binarySearchForKey, true);
 
@@ -1713,31 +1654,31 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
     //    startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey,
     //    prevEndPos);
     if (startPos < 0) {
-      if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) {
+//      if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) {
-        int32_t code =
+//        int32_t code =
-            setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult,
+//            setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult,
-                                    tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
+//                                    tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
-        if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
+//        if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
-          longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+//          longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
-        }
+//        }
-
+//
-        startPos = pSDataBlock->info.rows - 1;
+//        startPos = pSDataBlock->info.rows - 1;
 
         // window start(end) key interpolation
         //        doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos,
         //        forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos,
         //        forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput);
-      }
+//      }
 
       break;
     }
     setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
   }
 
-  if (pQueryAttr->timeWindowInterpo) {
+//  if (pQueryAttr->timeWindowInterpo) {
-    int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0;
+//    int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0;
     //    saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex);
-  }
+//  }
 
   //  updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey);
 }
@@ -2023,28 +1964,6 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock) {
-  size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo);
-  for (int32_t k = 0; k < num; ++k) {
-    SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
-    if (TSDB_COL_IS_TAG(pColIndex->flag)) {
-      continue;
-    }
-
-    int32_t colId = pColIndex->colId;
-
-    for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
-      SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
-      if (pColInfo->info.colId == colId) {
-        return i;
-      }
-    }
-  }
-
-  assert(0);
-  return -1;
-}
-
 static bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
   struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
 
@@ -2264,64 +2183,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   return NULL;
 }
 
-static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv* pRuntimeEnv, int32_t numOfTables, SArray* pOperator,
-                                    void* merger) {
-  // qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv));
-  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
-
-  pRuntimeEnv->prevGroupId = INT32_MIN;
-  pRuntimeEnv->pQueryAttr = pQueryAttr;
-
-  pRuntimeEnv->pResultRowHashTable =
-      taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-  pRuntimeEnv->pResultRowListSet =
-      taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES);
-  //  pRuntimeEnv->pool    = initResultRowPool(getResultRowSize(pRuntimeEnv));
-  pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell));
-
-  pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize);
-  pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen);
-
-  // NOTE: pTableCheckInfo need to update the query time range and the lastKey info
-  pRuntimeEnv->pTableRetrieveTsMap =
-      taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-
-  // pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
-
-  if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL ||
-      pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) {
-    goto _clean;
-  }
-
-  if (pQueryAttr->numOfCols) {
-    char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*)pRuntimeEnv->prevRow;
-    pRuntimeEnv->prevRow[0] = start;
-    for (int32_t i = 1; i < pQueryAttr->numOfCols; ++i) {
-      pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i - 1].bytes;
-    }
-
-    if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) {
-      *(int64_t*)pRuntimeEnv->prevRow[0] = INT64_MIN;
-    }
-  }
-
-  // qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv));
-
-  // group by normal column, sliding window query, interval query are handled by interval query processor
-  // interval (down sampling operation)
-  return TSDB_CODE_SUCCESS;
-
-_clean:
-  // destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
-  taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable);
-  taosMemoryFreeClear(pRuntimeEnv->keyBuf);
-  taosMemoryFreeClear(pRuntimeEnv->prevRow);
-  taosMemoryFreeClear(pRuntimeEnv->tagVal);
-
-  return TSDB_CODE_QRY_OUT_OF_MEMORY;
-}
-
 static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) {
   STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
 
@@ -2400,17 +2261,6 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q
 //  return false;
 //}
 
-static bool isFirstLastRowQuery(STaskAttr* pQueryAttr) {
-  for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
-    int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]);
-    if (functionID == FUNCTION_LAST_ROW) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 static bool isCachedLastQuery(STaskAttr* pQueryAttr) {
   for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
     int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
@@ -2493,18 +2343,6 @@ static bool onlyOneQueryType(STaskAttr* pQueryAttr, int32_t functId, int32_t fun
   return true;
 }
 
-static bool onlyFirstQuery(STaskAttr* pQueryAttr) {
-  return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST);
-}
-
-static bool onlyLastQuery(STaskAttr* pQueryAttr) {
-  return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST);
-}
-
-static bool notContainSessionOrStateWindow(STaskAttr* pQueryAttr) {
-  return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow);
-}
-
 static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) {
   bool hasFirstLastFunc = false;
   bool hasOtherFunc = false;
@@ -3006,7 +2844,7 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo,
 
   *status = BLK_DATA_ALL_NEEDED;
 
-  SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
+  SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
   if (pCols == NULL) {
     return terrno;
   }
@@ -3899,76 +3737,6 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S
   return 0;
 }
 
-// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function.
-void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput,
-                             SExprInfo* pExprInfo) {
-#if 0
-  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
-
-  int32_t numOfExprs = pQueryAttr->numOfOutput;
-  for(int32_t i = 0; i < numOfExprs; ++i) {
-    SExprInfo* pExprInfo1 = &(pExprInfo[i]);
-    if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) {
-      continue;
-    }
-
-    SExprBasicInfo* pExpr = &pExprInfo1->base;
-
-    pCtx[i].param[0].arr = NULL;
-    pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT;  // avoid freeing the memory by setting the type to be int
-
-    // TODO use hash to speedup this loop
-    int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
-    for (int32_t j = 0; j < numOfGroup; ++j) {
-      SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
-      if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) {
-        int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
-        for (int32_t k = 0; k < numOfCols; ++k) {
-          SStddevInterResult* pres = taosArrayGet(p->pResult, k);
-          if (pres->info.colId == pExpr->colInfo.colId) {
-            pCtx[i].param[0].arr = pres->pResult;
-            break;
-          }
-        }
-      }
-    }
-  }
-#endif
-}
-
-void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput,
-                                      SExprInfo* pExpr, char* val, int16_t bytes) {
-  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
-#if 0
-  int32_t numOfExprs = pQueryAttr->numOfOutput;
-  for(int32_t i = 0; i < numOfExprs; ++i) {
-    SExprBasicInfo* pExpr1 = &pExpr[i].base;
-    if (pExpr1->functionId != FUNCTION_STDDEV_DST) {
-      continue;
-    }
-
-    pCtx[i].param[0].arr = NULL;
-    pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT;  // avoid freeing the memory by setting the type to be int
-
-    // TODO use hash to speedup this loop
-    int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
-    for (int32_t j = 0; j < numOfGroup; ++j) {
-      SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
-      if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) {
-        int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
-        for (int32_t k = 0; k < numOfCols; ++k) {
-          SStddevInterResult* pres = taosArrayGet(p->pResult, k);
-          if (pres->info.colId == pExpr1->colInfo.colId) {
-            pCtx[i].param[0].arr = pres->pResult;
-            break;
-          }
-        }
-      }
-    }
-  }
-#endif
-}
-
 /*
  * There are two cases to handle:
  *
@@ -4679,13 +4447,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
 
   *newgroup = false;
 
-  while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) {
+  while (tsdbNextDataBlock(pTableScanInfo->dataReader)) {
     if (isTaskKilled(pOperator->pTaskInfo)) {
       longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
     }
 
     pTableScanInfo->numOfBlocks += 1;
-    tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info);
+    tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info);
 
     // todo opt
     //    if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
@@ -4722,7 +4490,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
   // The read handle is not initialized yet, since no qualified tables exists
-  if (pTableScanInfo->pTsdbReadHandle == NULL) {
+  if (pTableScanInfo->dataReader == NULL) {
     return NULL;
   }
 
@@ -4750,11 +4518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
     setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
     pTableScanInfo->scanFlag = REPEAT_SCAN;
 
-    //    if (pTaskInfo->pTsBuf) {
-    //      bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf);
-    //      assert(ret);
-    //    }
-    //
     if (pResultRowInfo->size > 0) {
       pResultRowInfo->curPos = 0;
     }
@@ -4790,44 +4553,44 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
 
   STableScanInfo* pTableScanInfo = pOperator->info;
   *newgroup = false;
-#if 0
+
-  STableBlockDist tableBlockDist = {0};
+  STableBlockDistInfo tableBlockDist = {0};
-  tableBlockDist.numOfTables     = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
+  tableBlockDist.numOfTables = 1;  // TODO set the correct number of tables
 
   int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS;
   if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
     ++numRowSteps;
   }
+
   tableBlockDist.dataBlockInfos  = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
   taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps);
+
   tableBlockDist.maxRows = INT_MIN;
   tableBlockDist.minRows = INT_MAX;
 
-  tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist);
+  tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist);
-  tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle);
+  tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader);
 
   SSDataBlock* pBlock = &pTableScanInfo->block;
   pBlock->info.rows   = 1;
   pBlock->info.numOfCols = 1;
 
-  SBufferWriter bw = tbufInitWriter(NULL, false);
+//  SBufferWriter bw = tbufInitWriter(NULL, false);
-  blockDistInfoToBinary(&tableBlockDist, &bw);
+//  blockDistInfoToBinary(&tableBlockDist, &bw);
   SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
 
-  int32_t len = (int32_t) tbufTell(&bw);
+//  int32_t len = (int32_t) tbufTell(&bw);
-  pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
+//  pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
+//  *(int32_t*) pColInfo->pData = len;
+//  memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
+//
+//  tbufCloseWriter(&bw);
 
-  *(int32_t*) pColInfo->pData = len;
+//  SArray* g = GET_TABLEGROUP(pOperator->, 0);
-  memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
+//  pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
-
-  tbufCloseWriter(&bw);
-
-  SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0);
-  pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
 
   pOperator->status = OP_EXEC_DONE;
   return pBlock;
-#endif
 }
 
 static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
@@ -5472,7 +5235,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
   }
 
   pInfo->pFilterNode      = pCondition;
-  pInfo->pTsdbReadHandle  = pTsdbReadHandle;
+  pInfo->dataReader       = pTsdbReadHandle;
   pInfo->times            = repeatTime;
   pInfo->reverseTimes     = reverseTime;
   pInfo->order            = order;
@@ -5494,7 +5257,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
 
-  pInfo->pTsdbReadHandle = pTsdbReadHandle;
+  pInfo->dataReader = pTsdbReadHandle;
   pInfo->times = 1;
   pInfo->reverseTimes = 0;
   pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
@@ -5515,32 +5278,42 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
   return pOperator;
 }
 
-SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
+SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) {
-  STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
+  STableScanInfo* pInfo    = taosMemoryCalloc(1, sizeof(STableScanInfo));
+  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _error;
+  }
 
-  pInfo->pTsdbReadHandle = pTsdbReadHandle;
+  pInfo->dataReader       = dataReader;
   pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
 
-  SColumnInfoData infoData = {{0}};
+  SColumnInfoData infoData = {0};
-  infoData.info.type = TSDB_DATA_TYPE_BINARY;
+  infoData.info.type  = TSDB_DATA_TYPE_BINARY;
   infoData.info.bytes = 1024;
   infoData.info.colId = 0;
   taosArrayPush(pInfo->block.pDataBlock, &infoData);
 
-  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  pOperator->name          = "DataBlockInfoScanOperator";
-  pOperator->name = "TableBlockInfoScanOperator";
   //  pOperator->operatorType = OP_TableBlockInfoScan;
-  pOperator->blockingOptr = false;
+  pOperator->blockingOptr  = false;
-  pOperator->status = OP_NOT_OPENED;
+  pOperator->status        = OP_NOT_OPENED;
-  pOperator->info = pInfo;
+  pOperator->_openFn       = operatorDummyOpenFn;
-  //  pOperator->numOfOutput  = pRuntimeEnv->pQueryAttr->numOfCols;
+  pOperator->getNextFn     = doBlockInfoScan;
-  pOperator->getNextFn = doBlockInfoScan;
+
+  pOperator->info          = pInfo;
+  pOperator->pTaskInfo     = pTaskInfo;
 
   return pOperator;
+
+  _error:
+  taosMemoryFreeClear(pInfo);
+  taosMemoryFreeClear(pOperator);
+  return NULL;
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList,
+SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
-                                            SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
   SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -5579,16 +5352,17 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   pInfo->readerHandle = streamReadHandle;
   pInfo->pRes = pResBlock;
 
-  pOperator->name = "StreamBlockScanOperator";
+  pOperator->name         = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
   pOperator->blockingOptr = false;
-  pOperator->status = OP_NOT_OPENED;
+  pOperator->status       = OP_NOT_OPENED;
-  pOperator->info = pInfo;
+  pOperator->info         = pInfo;
-  pOperator->numOfOutput = pResBlock->info.numOfCols;
+  pOperator->numOfOutput  = pResBlock->info.numOfCols;
-  pOperator->_openFn = operatorDummyOpenFn;
+  pOperator->_openFn      = operatorDummyOpenFn;
-  pOperator->getNextFn = doStreamBlockScan;
+  pOperator->getNextFn    = doStreamBlockScan;
-  pOperator->closeFn = operatorDummyCloseFn;
+  pOperator->closeFn      = operatorDummyCloseFn;
-  pOperator->pTaskInfo = pTaskInfo;
+  pOperator->pTaskInfo    = pTaskInfo;
+
   return pOperator;
 }
 
@@ -5704,7 +5478,7 @@ void getDBNameFromCondition(SNode* pCondition, char* dbName) {
     return;
   }
 
-  nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName);
+  nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName);
 }
 
 static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
@@ -5922,80 +5696,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
   return pOperator;
 }
 
-SArray* getOrderCheckColumns(STaskAttr* pQuery) {
-  int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);
-
-  SArray* pOrderColumns = NULL;
-  if (numOfCols > 0) {
-    pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
-  } else {
-    pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
-  }
-
-  if (pQuery->interval.interval > 0) {
-    if (pOrderColumns == NULL) {
-      pOrderColumns = taosArrayInit(1, sizeof(SColIndex));
-    }
-
-    SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL};
-    taosArrayPush(pOrderColumns, &colIndex);
-  }
-
-  {
-    numOfCols = (int32_t)taosArrayGetSize(pOrderColumns);
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      SColIndex* index = taosArrayGet(pOrderColumns, i);
-      for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
-        SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base;
-        int32_t         functionId = getExprFunctionId(&pQuery->pExpr1[j]);
-
-        if (index->colId == pExpr->pParam[0].pCol->colId &&
-            (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) {
-          index->colIndex = j;
-          index->colId = pExpr->resSchema.colId;
-        }
-      }
-    }
-  }
-
-  return pOrderColumns;
-}
-
-SArray* getResultGroupCheckColumns(STaskAttr* pQuery) {
-  int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);
-
-  SArray* pOrderColumns = NULL;
-  if (numOfCols > 0) {
-    pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
-  } else {
-    pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
-  }
-
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColIndex* index = taosArrayGet(pOrderColumns, i);
-
-    bool found = false;
-    for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
-      SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base;
-      int32_t         functionId = getExprFunctionId(&pQuery->pExpr1[j]);
-
-      // FUNCTION_TAG_DUMMY function needs to be ignored
-      //      if (index->colId == pExpr->pColumns->info.colId &&
-      //          ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) ||
-      //           (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) {
-      //        index->colIndex = j;
-      //        index->colId = pExpr->resSchema.colId;
-      //        found = true;
-      //        break;
-      //      }
-    }
-
-    assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput);
-  }
-
-  return pOrderColumns;
-}
-
 static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey);
 static void    cleanupAggSup(SAggSupporter* pAggSup);
 
@@ -7044,9 +6744,6 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr
     return pIntervalInfo->binfo.pRes;
   }
 
-  STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
-  int32_t    order = pQueryAttr->order.order;
-
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   while (1) {
@@ -7062,14 +6759,14 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr
     STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
 
     //    setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput);
-    setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
+//    setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
     setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey);
 
     hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex);
   }
 
   pOperator->status = OP_RES_TO_RETURN;
-  pQueryAttr->order.order = order;  // TODO : restore the order
+//  pQueryAttr->order.order = order;  // TODO : restore the order
   doCloseAllTimeWindow(pRuntimeEnv);
   setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
 
@@ -8153,7 +7850,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
 
   int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
   if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
-    STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
     assert(pQueryAttr->numOfOutput == 1);
 
     SExprInfo* pExprInfo = &pOperator->pExpr[0];
@@ -8936,8 +8632,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
   return pList;
 }
 
-int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo,
+int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) {
-                           uint64_t queryId, uint64_t taskId) {
   int32_t code = 0;
   if (tableType == TSDB_SUPER_TABLE) {
     code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId);
@@ -9041,43 +8736,6 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
   return TSDB_CODE_SUCCESS;
 }
 
-static void doUpdateExprColumnIndex(STaskAttr* pQueryAttr) {
-  assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL);
-
-  for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) {
-    SExprBasicInfo* pSqlExprMsg = &pQueryAttr->pExpr1[k].base;
-    //    if (pSqlExprMsg->functionId == FUNCTION_ARITHM) {
-    //      continue;
-    //    }
-
-    // todo opt performance
-    SColIndex* pColIndex = NULL; /*&pSqlExprMsg->colInfo;*/
-    if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) {
-      int32_t f = 0;
-      for (f = 0; f < pQueryAttr->numOfCols; ++f) {
-        if (pColIndex->colId == pQueryAttr->tableCols[f].colId) {
-          pColIndex->colIndex = f;
-          break;
-        }
-      }
-
-      assert(f < pQueryAttr->numOfCols);
-    } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) {
-      // do nothing for user-defined constant value result columns
-    } else {
-      int32_t f = 0;
-      for (f = 0; f < pQueryAttr->numOfTags; ++f) {
-        if (pColIndex->colId == pQueryAttr->tagColList[f].colId) {
-          pColIndex->colIndex = f;
-          break;
-        }
-      }
-
-      assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX);
-    }
-  }
-}
-
 void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) {
   const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512);
 
@@ -9087,16 +8745,16 @@ void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) {
 
   const float THRESHOLD_RATIO = 0.85f;
 
-  if (isProjQuery(pQueryAttr)) {
+//  if (isProjQuery(pQueryAttr)) {
-    int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize;
+//    int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize;
-    if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) {
+//    if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) {
-      numOfRes = MIN_ROWS_FOR_PRJ_QUERY;
+//      numOfRes = MIN_ROWS_FOR_PRJ_QUERY;
-    }
+//    }
-
+//
-    pResultInfo->capacity = numOfRes;
+//    pResultInfo->capacity = numOfRes;
-  } else {  // in case of non-prj query, a smaller output buffer will be used.
+//  } else {  // in case of non-prj query, a smaller output buffer will be used.
-    pResultInfo->capacity = DEFAULT_MIN_ROWS;
+//    pResultInfo->capacity = DEFAULT_MIN_ROWS;
-  }
+//  }
 
   pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO);
   pResultInfo->totalRows = 0;

source/libs/function/inc/builtinsimpl.h

@@ -38,9 +38,14 @@ void minFunction(SqlFunctionCtx* pCtx);
 void maxFunction(SqlFunctionCtx *pCtx);
 
 bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 void stddevFunction(SqlFunctionCtx* pCtx);
 void stddevFinalize(SqlFunctionCtx* pCtx);
 
+bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+void percentileFunction(SqlFunctionCtx *pCtx);
+
 bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 void firstFunction(SqlFunctionCtx *pCtx);
 void lastFunction(SqlFunctionCtx *pCtx);

source/libs/function/src/builtins.c

@@ -68,9 +68,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
       .classification = FUNC_MGT_AGG_FUNC,
       .checkFunc    = stubCheckAndGetResultType,
       .getEnvFunc   = getStddevFuncEnv,
-      .initFunc     = maxFunctionSetup,
+      .initFunc     = stddevFunctionSetup,
-      .processFunc  = maxFunction,
+      .processFunc  = stddevFunction,
-      .finalizeFunc = functionFinalize
+      .finalizeFunc = stddevFinalize
   },
   {
       .name = "percentile",
@@ -282,6 +282,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = atanFunction,
     .finalizeFunc = NULL
   },
+  {
+    .name = "length",
+    .type = FUNCTION_TYPE_LENGTH,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = lengthFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "char_length",
+    .type = FUNCTION_TYPE_CHAR_LENGTH,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = charLengthFunction,
+    .finalizeFunc = NULL
+  },
   {
     .name = "concat",
     .type = FUNCTION_TYPE_CONCAT,
@@ -289,7 +309,67 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .checkFunc    = stubCheckAndGetResultType,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
-    .sprocessFunc = NULL,
+    .sprocessFunc = concatFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "concat_ws",
+    .type = FUNCTION_TYPE_CONCAT_WS,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = concatWsFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "lower",
+    .type = FUNCTION_TYPE_LOWER,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = lowerFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "upper",
+    .type = FUNCTION_TYPE_UPPER,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = upperFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "ltrim",
+    .type = FUNCTION_TYPE_LTRIM,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = ltrimFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "rtrim",
+    .type = FUNCTION_TYPE_RTRIM,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = rtrimFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "substr",
+    .type = FUNCTION_TYPE_SUBSTR,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = substrFunction,
     .finalizeFunc = NULL
   },
   {
@@ -409,12 +489,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType };
       break;
     }
-    case FUNCTION_TYPE_CONCAT:
-    case FUNCTION_TYPE_ROWTS:
-    case FUNCTION_TYPE_TBNAME: {
-      // todo
-      break;
-    }
 
     case FUNCTION_TYPE_QENDTS:
     case FUNCTION_TYPE_QSTARTTS:
@@ -434,6 +508,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       break;
     }
 
+    case FUNCTION_TYPE_STDDEV:
     case FUNCTION_TYPE_SIN:
     case FUNCTION_TYPE_COS:
     case FUNCTION_TYPE_TAN:
@@ -446,6 +521,65 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
       break;
     }
+
+    case FUNCTION_TYPE_LENGTH:
+    case FUNCTION_TYPE_CHAR_LENGTH: {
+      pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT };
+      break;
+    }
+
+    case FUNCTION_TYPE_CONCAT:
+    case FUNCTION_TYPE_CONCAT_WS: {
+      int32_t paraType, paraBytes = 0;
+      for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) {
+        SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i);
+        paraBytes += pParam->node.resType.bytes;
+        paraType = pParam->node.resType.type;
+      }
+      pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType };
+      break;
+      //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0;
+      //int32_t firstParamIndex = 0;
+      //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) {
+      //  firstParamIndex = 1;
+      //  SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0);
+      //  sepBytes = pSep->node.resType.type;
+      //}
+      //for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) {
+      //  SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i);
+      //  int32_t paraType = pParam->node.resType.type;
+      //  if (i == firstParamIndex) {
+      //    paraTypeFirst = paraType;
+      //  }
+      //  if (paraType != paraTypeFirst) {
+      //    return TSDB_CODE_FAILED;
+      //  }
+      //  //TODO: for constants also needs numOfRows
+      //  totalBytes += pParam->node.resType.bytes;
+      //}
+      ////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100.
+      //totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100;
+      //pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst };
+      //break;
+    }
+    case FUNCTION_TYPE_LOWER:
+    case FUNCTION_TYPE_UPPER:
+    case FUNCTION_TYPE_LTRIM:
+    case FUNCTION_TYPE_RTRIM:
+    case FUNCTION_TYPE_SUBSTR: {
+      SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
+      int32_t paraType  = pParam->node.resType.type;
+      int32_t paraBytes = pParam->node.resType.bytes;
+      pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType };
+      break;
+    }
+
+    case FUNCTION_TYPE_ROWTS:
+    case FUNCTION_TYPE_TBNAME: {
+      // todo
+      break;
+    }
+
     case FUNCTION_TYPE_NOW:
       // todo
       break;

source/libs/function/src/builtinsimpl.c

@@ -14,6 +14,7 @@
  */
 
 #include "builtinsimpl.h"
+#include "tpercentile.h"
 #include "querynodes.h"
 #include "taggfunction.h"
 #include "tdatablock.h"
@@ -453,6 +454,7 @@ bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
 }
 
 typedef struct SStddevRes {
+  double  result;
   int64_t count;
   union  {double  quadraticDSum; int64_t quadraticISum;};
   union  {double  dsum; int64_t isum;};
@@ -463,38 +465,129 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   return true;
 }
 
+bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  SStddevRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
+  memset(pRes, 0, sizeof(SStddevRes));
+  return true;
+}
+
 void stddevFunction(SqlFunctionCtx* pCtx) {
   int32_t numOfElem = 0;
 
   // Only the pre-computing information loaded and actual data does not loaded
   SInputColumnInfoData* pInput = &pCtx->input;
-  SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0];
+  SColumnDataAgg*       pAgg = pInput->pColumnDataAgg[0];
-  int32_t type = pInput->pData[0]->info.type;
+  int32_t               type = pInput->pData[0]->info.type;
 
   SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
 
-//  } else {  // computing based on the true data block
+  // computing based on the true data block
-    SColumnInfoData* pCol = pInput->pData[0];
+  SColumnInfoData* pCol = pInput->pData[0];
 
-    int32_t start     = pInput->startRowIndex;
+  int32_t start = pInput->startRowIndex;
-    int32_t numOfRows = pInput->numOfRows;
+  int32_t numOfRows = pInput->numOfRows;
 
-    switch(type) {
+  switch (type) {
-      case TSDB_DATA_TYPE_INT: {
+    case TSDB_DATA_TYPE_TINYINT: {
-        int32_t* plist = (int32_t*)pCol->pData;
+        int8_t* plist = (int8_t*)pCol->pData;
         for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
           if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
             continue;
           }
 
+          numOfElem += 1;
           pStddevRes->count += 1;
-          pStddevRes->isum  += plist[i];
+          pStddevRes->isum += plist[i];
           pStddevRes->quadraticISum += plist[i] * plist[i];
         }
+
+        break;
+      }
+
+      case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t* plist = (int16_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pStddevRes->count += 1;
+        pStddevRes->isum += plist[i];
+        pStddevRes->quadraticISum += plist[i] * plist[i];
       }
       break;
     }
 
+    case TSDB_DATA_TYPE_INT: {
+      int32_t* plist = (int32_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pStddevRes->count += 1;
+        pStddevRes->isum += plist[i];
+        pStddevRes->quadraticISum += plist[i] * plist[i];
+      }
+
+      break;
+    }
+
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t* plist = (int64_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pStddevRes->count += 1;
+        pStddevRes->isum += plist[i];
+        pStddevRes->quadraticISum += plist[i] * plist[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_FLOAT: {
+      float* plist = (float*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pStddevRes->count += 1;
+        pStddevRes->isum += plist[i];
+        pStddevRes->quadraticISum += plist[i] * plist[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double* plist = (double*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pStddevRes->count += 1;
+        pStddevRes->isum += plist[i];
+        pStddevRes->quadraticISum += plist[i] * plist[i];
+      }
+      break;
+    }
+
+    default:
+      break;
+  }
+
   // data in the check operation are all null, not output
   SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
 }
@@ -503,11 +596,122 @@ void stddevFinalize(SqlFunctionCtx* pCtx) {
   functionFinalize(pCtx);
 
   SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-  double res = pStddevRes->quadraticISum/pStddevRes->count - (pStddevRes->isum / pStddevRes->count) * (pStddevRes->isum / pStddevRes->count);
+  double avg = pStddevRes->isum / ((double) pStddevRes->count);
+  pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
 }
 
+typedef struct SPercentileInfo {
+  tMemBucket *pMemBucket;
+  int32_t     stage;
+  double      minval;
+  double      maxval;
+  int64_t     numOfElems;
+} SPercentileInfo;
 
+bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(SPercentileInfo);
+  return true;
+}
 
+bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  // in the first round, get the min-max value of all involved data
+  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
+  SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
+  pInfo->numOfElems = 0;
+
+  return true;
+}
+
+void percentileFunction(SqlFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+#if 0
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) {
+    pInfo->stage += 1;
+
+    // all data are null, set it completed
+    if (pInfo->numOfElems == 0) {
+      pResInfo->complete = true;
+      return;
+    } else {
+      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    }
+  }
+
+  // the first stage, only acquire the min/max value
+  if (pInfo->stage == 0) {
+    if (pCtx->preAggVals.isSet) {
+      double tmin = 0.0, tmax = 0.0;
+      if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+        tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min);
+        tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max);
+      } else if (IS_FLOAT_TYPE(pCtx->inputType)) {
+        tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min);
+        tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max);
+      } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+        tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min);
+        tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max);
+      } else {
+        assert(true);
+      }
+
+      if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) {
+        SET_DOUBLE_VAL(&pInfo->minval, tmin);
+      }
+
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) {
+        SET_DOUBLE_VAL(&pInfo->maxval, tmax);
+      }
+
+      pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
+    } else {
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        char *data = GET_INPUT_DATA(pCtx, i);
+        if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+          continue;
+        }
+
+        double v = 0;
+        GET_TYPED_DATA(v, double, pCtx->inputType, data);
+
+        if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+          SET_DOUBLE_VAL(&pInfo->minval, v);
+        }
+
+        if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+          SET_DOUBLE_VAL(&pInfo->maxval, v);
+        }
+
+        pInfo->numOfElems += 1;
+      }
+    }
+
+    return;
+  }
+
+  // the second stage, calculate the true percentile value
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+
+    notNullElems += 1;
+    tMemBucketPut(pInfo->pMemBucket, data, 1);
+  }
+
+  SET_VAL(pCtx, notNullElems, 1);
+  pResInfo->hasResult = DATA_SET_FLAG;
+#endif
+
+}
 
 bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);

source/libs/function/src/functionMgt.c

@@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) {
   return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC);
 }
 
+bool fmIsPseudoColumnFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC);
+}
+
 bool fmIsWindowPseudoColumnFunc(int32_t funcId) {
   return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC);
 }

source/libs/function/src/taggfunction.c

@@ -176,26 +176,6 @@ typedef struct SResPair {
   double avg;
 } SResPair;
 
-#define TSDB_BLOCK_DIST_STEP_ROWS 16
-
-typedef struct STableBlockDist {
-  uint16_t  rowSize;
-  uint16_t  numOfFiles;
-  uint32_t  numOfTables;
-  uint64_t  totalSize;
-  uint64_t  totalRows;
-  int32_t   maxRows;
-  int32_t   minRows;
-  int32_t   firstSeekTimeUs;
-  uint32_t  numOfRowsInMemTable;
-  uint32_t  numOfSmallBlocks;
-  SArray   *dataBlockInfos;
-} STableBlockDist;
-
-typedef struct SFileBlockInfo {
-  int32_t numBlocksOfStep;
-} SFileBlockInfo;
-
 void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) {
   pCell->initialized = false;
 }
@@ -3984,7 +3964,7 @@ static void irate_function(SqlFunctionCtx *pCtx) {
   }
 }
 
-static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) {
+static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) {
   SBufferReader br = tbufInitReader(data, len, false);
 
   pDist->numOfTables = tbufReadUint32(&br);
@@ -4024,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi
 
 static void blockInfo_func(SqlFunctionCtx* pCtx) {
   SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
-  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+  STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
 
   int32_t len = *(int32_t*) pCtx->pInput;
   blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist);
@@ -4036,8 +4016,8 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) {
   //pResInfo->hasResult = DATA_SET_FLAG;
 }
 
-static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) {
+static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) {
-  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+  STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
   assert(pDist != NULL && pSrc != NULL);
 
   pDist->numOfTables += pSrc->numOfTables;
@@ -4071,7 +4051,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock
 }
 
 void block_func_merge(SqlFunctionCtx* pCtx) {
-  STableBlockDist info = {0};
+  STableBlockDistInfo info = {0};
   int32_t len = *(int32_t*) pCtx->pInput;
   blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info);
   SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
@@ -4082,7 +4062,7 @@ void block_func_merge(SqlFunctionCtx* pCtx) {
   //pResInfo->hasResult = DATA_SET_FLAG;
 }
 
-void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
+void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
                     double* percents, int32_t* percentiles) {
   if (totalBlocks == 0) {
     for (int32_t i = 0; i < numOfPercents; ++i) {
@@ -4117,7 +4097,7 @@ void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32
   }
 }
 
-void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
+void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) {
   if (pTableBlockDist == NULL) {
     return;
   }
@@ -4178,7 +4158,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
 
 void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) {
   SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
-  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+  STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
 
   pDist->rowSize = (uint16_t)pCtx->param[0].i;
   generateBlockDistResult(pDist, pCtx->pOutput);

source/libs/index/src/indexFst.c

@@ -642,6 +642,9 @@ static const char* fstNodeState(FstNode* node) {
 }
 
 void fstNodeDestroy(FstNode* node) {
+  if (node == NULL) {
+    return;
+  }
   fstSliceDestroy(&node->data);
   taosMemoryFree(node);
 }
@@ -1247,7 +1250,10 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
       // autState = sws->aut->accept(preState, b);
       autState = automFuncs[aut->type].accept(aut, preState, b);
       taosArrayPush(sws->inp, &b);
+
       StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState};
+      node = NULL;
+
       taosArrayPush(sws->stack, &s);
       out += trn.out;
       node = fstGetNode(sws->fst, trn.addr);
@@ -1271,6 +1277,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
       return true;
     }
   }
+
+  fstNodeDestroy(node);
+
   uint32_t sz = taosArrayGetSize(sws->stack);
   if (sz != 0) {
     StreamState* s = taosArrayGet(sws->stack, sz - 1);

source/libs/index/test/fstTest.cc

@@ -480,6 +480,15 @@ void checkFstCheckIteratorRange2() {
     assert(result.size() == 4);
     automCtxDestroy(ctx);
   }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "bb", GE, "ed", LT, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
   {
     // range  search
     std::vector<uint64_t> result;
@@ -635,11 +644,11 @@ int main(int argc, char* argv[]) {
   // path suid colName ver
   // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
   //}
-  // checkFstCheckIterator1();
+  checkFstCheckIterator1();
-  // checkFstCheckIterator2();
+  checkFstCheckIterator2();
-  // checkFstCheckIteratorPrefix();
+  checkFstCheckIteratorPrefix();
-  // checkFstCheckIteratorRange1();
+  checkFstCheckIteratorRange1();
-  // checkFstCheckIteratorRange2();
+  checkFstCheckIteratorRange2();
   checkFstCheckIteratorRange3();
   // checkFstLongTerm();
   // checkFstPrefixSearch();

source/libs/nodes/src/nodesCloneFuncs.c

@@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
 }
 
 static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
   exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_CHAR_POINT_FIELD(literal);
-  COPY_SCALAR_FIELD(isDuration);
-  COPY_SCALAR_FIELD(translate);
   if (!pSrc->translate) {
     return (SNode*)pDst;
   }
   switch (pSrc->node.resType.type) {
-    case TSDB_DATA_TYPE_NULL:
-      break;
-    case TSDB_DATA_TYPE_BOOL:
-      COPY_SCALAR_FIELD(datum.b);
-      break;
-    case TSDB_DATA_TYPE_TINYINT:
-    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_INT:
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TIMESTAMP:
-      COPY_SCALAR_FIELD(datum.i);
-      break;
-    case TSDB_DATA_TYPE_UTINYINT:
-    case TSDB_DATA_TYPE_USMALLINT:
-    case TSDB_DATA_TYPE_UINT:
-    case TSDB_DATA_TYPE_UBIGINT:
-      COPY_SCALAR_FIELD(datum.u);
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-    case TSDB_DATA_TYPE_DOUBLE:
-      COPY_SCALAR_FIELD(datum.d);
-      break;
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -1491,6 +1491,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkValueGenByCalc = "GenByCalc";
 static const char* jkValueLiteral = "Literal";
 static const char* jkValueDuration = "Duration";
 static const char* jkValueTranslate = "Translate";
@@ -1544,6 +1545,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
 
   int32_t code = exprNodeToJson(pObj, pJson);
   if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc);
+  }
+  if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
     code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
   }
   if (TSDB_CODE_SUCCESS == code) {
@@ -1614,6 +1618,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
 
   int32_t code = jsonToExprNode(pJson, pObj);
   if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc);
+  }
+  if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
     code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal);
   }
   if (TSDB_CODE_SUCCESS == code) {

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke
   return DEAL_RES_CONTINUE;
 }
 
-void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) {
+void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) {
   (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext);
 }
 
-void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) {
+void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) {
   (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext);
 }
 
-void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) {
+void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) {
   (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext);
 }
 
-void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) {
+void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) {
   (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext);
 }
 
@@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe
   return DEAL_RES_CONTINUE;
 }
 
-void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
+void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
   (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext);
 }
 
-void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
+void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
   (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext);
 }
 
-void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
+void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
   (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext);
 }
 
-void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
+void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
   (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext);
 }
 
@@ -290,20 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
 
   switch (clause) {
     case SQL_CLAUSE_FROM:
-      nodesWalkNode(pSelect->pFromTable, walker, pContext);
+      nodesWalkExpr(pSelect->pFromTable, walker, pContext);
-      nodesWalkNode(pSelect->pWhere, walker, pContext);
+      nodesWalkExpr(pSelect->pWhere, walker, pContext);
     case SQL_CLAUSE_WHERE:
-      nodesWalkList(pSelect->pPartitionByList, walker, pContext);
+      nodesWalkExprs(pSelect->pPartitionByList, walker, pContext);
     case SQL_CLAUSE_PARTITION_BY:
-      nodesWalkNode(pSelect->pWindow, walker, pContext);
+      nodesWalkExpr(pSelect->pWindow, walker, pContext);
     case SQL_CLAUSE_WINDOW:
-      nodesWalkList(pSelect->pGroupByList, walker, pContext);
+      nodesWalkExprs(pSelect->pGroupByList, walker, pContext);
     case SQL_CLAUSE_GROUP_BY:
-      nodesWalkNode(pSelect->pHaving, walker, pContext);
+      nodesWalkExpr(pSelect->pHaving, walker, pContext);
     case SQL_CLAUSE_HAVING:
-      nodesWalkList(pSelect->pOrderByList, walker, pContext);
+      nodesWalkExprs(pSelect->pOrderByList, walker, pContext);
     case SQL_CLAUSE_ORDER_BY:
-      nodesWalkList(pSelect->pProjectionList, walker, pContext);
+      nodesWalkExprs(pSelect->pProjectionList, walker, pContext);
     case SQL_CLAUSE_SELECT:
     default:
       break;
@@ -319,20 +319,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
 
   switch (clause) {
     case SQL_CLAUSE_FROM:
-      nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext);
+      nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext);
-      nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext);
+      nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext);
     case SQL_CLAUSE_WHERE:
-      nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext);
+      nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext);
     case SQL_CLAUSE_PARTITION_BY:
-      nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext);
+      nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext);
     case SQL_CLAUSE_WINDOW:
-      nodesRewriteList(pSelect->pGroupByList, rewriter, pContext);
+      nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext);
     case SQL_CLAUSE_GROUP_BY:
-      nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext);
+      nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext);
     case SQL_CLAUSE_HAVING:
-      nodesRewriteList(pSelect->pProjectionList, rewriter, pContext);
+      nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext);
     case SQL_CLAUSE_SELECT:
-      nodesRewriteList(pSelect->pOrderByList, rewriter, pContext);
+      nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext);
     default:
       break;
   }

source/libs/nodes/test/nodesTest.cpp

@@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) {
 
 	EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR);
 	EDealRes res = DEAL_RES_CONTINUE;
-	nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res);
+	nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res);
 	EXPECT_EQ(res, DEAL_RES_CONTINUE);
 	EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE);
 	EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18");

source/libs/parser/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
         parser
-        PRIVATE os util nodes catalog function transport qcom
+        PRIVATE os util nodes catalog function scalar transport qcom
 )
 
 if(${BUILD_TEST})

source/libs/parser/inc/parInt.h

@@ -23,9 +23,10 @@ extern "C" {
 #include "parser.h"
 
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery);
-int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery);
+int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
-int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery);
+int32_t translate(SParseContext* pParseCxt, SQuery* pQuery);
 int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
+int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
 
 #ifdef __cplusplus
 }

source/libs/parser/src/parAstCreater.c

@@ -243,27 +243,27 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio
     return pOptions;
   }
 
-  char val[20] = {0};
+  if (pVal->n > 2) {
-  int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val));
+    char* pStart = pVal->z + 1;
-  char* pStart = val;
+    char* pEnd = pVal->z + pVal->n - 1;
-  char* pEnd = val + len;
+    int32_t sepOrder = 1;
-  int32_t sepOrder = 1;
+    while (1) {
-  while (1) {
+      char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':');
-    char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':');
+      SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)};
-    SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)};
+      if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) {
-    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) {
+        pCxt->valid = false;
-      pCxt->valid = false;
+        snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
-      snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
+        return pOptions;
-      return pOptions;
+      }
+      if (NULL == pPos) {
+        break;
+      }
+      pStart = pPos + 1;
     }
-    if (NULL == pPos) {
-      break;
-    }
-    pStart = pPos + 1;
   }
 
-  if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
+  if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
-    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val);
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z);
     pCxt->valid = false;
   }
 

source/libs/parser/src/parAstParser.c

@@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*);
 extern void ParseFree(void*, FFree);
 extern void ParseTrace(FILE*, char*);
 
-int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) {
+int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
   SAstCreateContext cxt;
   initAstCreateContext(pParseCxt, &cxt);
   void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc);

source/libs/parser/src/parCalcConst.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "functionMgt.h"
+#include "parInt.h"
+#include "scalar.h"
+
+typedef struct SCalcConstContext {
+  int32_t code;
+} SCalcConstContext;
+
+static int32_t calcConstQuery(SNode* pStmt);
+
+static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) {
+  SNode* pNew = NULL;
+  pCxt->code = scalarCalculateConstants(*pNode, &pNew);
+  if (TSDB_CODE_SUCCESS != pCxt->code) {
+    return DEAL_RES_ERROR;
+  }
+  ((SValueNode*)pNew)->genByCalc = true;
+  ((SValueNode*)pNew)->translate = true;
+  *pNode = pNew;
+  return DEAL_RES_CONTINUE;
+}
+
+static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) {
+  SOperatorNode* pOp = *pNode;
+  if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) {
+    return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) {
+  SFunctionNode* pFunc = *pNode;
+  if (fmIsPseudoColumnFunc(pFunc->funcId)) {
+    return DEAL_RES_CONTINUE;
+  }
+  SNode* pParam = NULL;
+  FOREACH(pParam, pFunc->pParameterList) {
+    if (QUERY_NODE_VALUE != nodeType(pParam)) {
+      return DEAL_RES_CONTINUE;
+    }
+  }
+  return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
+}
+
+static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) {
+  SLogicConditionNode* pCond = *pNode;
+  SNode* pParam = NULL;
+  FOREACH(pParam, pCond->pParameterList) {
+    if (QUERY_NODE_VALUE != nodeType(pParam)) {
+      return DEAL_RES_CONTINUE;
+    }
+  }
+  return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
+}
+
+static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) {
+  SCalcConstContext* pCxt = pContext;
+  pCxt->code = calcConstQuery((*pNode)->pSubquery);
+  return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR);
+}
+
+static EDealRes calcConst(SNode** pNode, void* pContext) {
+  switch (nodeType(*pNode)) {
+    case QUERY_NODE_OPERATOR:
+      return calcConstOperator((SOperatorNode**)pNode, pContext);
+    case QUERY_NODE_FUNCTION:
+      return calcConstFunction((SFunctionNode**)pNode, pContext);
+    case QUERY_NODE_LOGIC_CONDITION:
+      return calcConstLogicCond((SLogicConditionNode**)pNode, pContext);
+    case QUERY_NODE_TEMP_TABLE:
+      return calcConstSubquery((STempTableNode**)pNode, pContext);
+    default:
+      break;
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t calcConstSelect(SSelectStmt* pSelect) {
+  SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS };
+  nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt);
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt);
+  }
+  if (TSDB_CODE_SUCCESS == cxt.code) {
+    nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt);
+  }
+  return cxt.code;
+}
+
+static int32_t calcConstQuery(SNode* pStmt) {
+  switch (nodeType(pStmt)) {
+    case QUERY_NODE_SELECT_STMT:
+      return calcConstSelect((SSelectStmt*)pStmt);    
+    default:
+      break;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) {
+  return calcConstQuery(pQuery->pRoot);
+}

source/libs/parser/src/parTranslater.c

@@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) {
   return clause < SQL_CLAUSE_HAVING;
 }
 
-static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) {
+#define generateDealNodeErrMsg(pCxt, code, ...) \
-  va_list vArgList;
+  ({ \
-  va_start(vArgList, errCode);
+    generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \
-  generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList);
+    pCxt->errCode = code; \
-  va_end(vArgList);
+    DEAL_RES_ERROR; \
-  pCxt->errCode = errCode;
+  })
-  return DEAL_RES_ERROR;
-}
 
 static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
   size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel);
@@ -440,6 +438,9 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
 
 static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
   if (nodesIsUnaryOp(pOp)) {
+    if (OP_TYPE_MINUS == pOp->opType && !IS_NUMERIC_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
+    }
     return DEAL_RES_CONTINUE;
   }
   SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
@@ -510,12 +511,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) {
 }
 
 static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) {
-  nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt);
+  nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt);
   return pCxt->errCode;
 }
 
 static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) {
-  nodesWalkListPostOrder(pList, doTranslateExpr, pCxt);
+  nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt);
   return pCxt->errCode;
 }
 
@@ -570,7 +571,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) {
 }
 
 static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) {
-  nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt);
+  nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt);
   return pCxt->errCode;
 }
 
@@ -578,7 +579,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList
   if (NULL == getGroupByList(pCxt)) {
     return TSDB_CODE_SUCCESS;
   }
-  nodesWalkList(pList, doCheckExprForGroupBy, pCxt);
+  nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt);
   return pCxt->errCode;
 }
 
@@ -605,9 +606,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
     return TSDB_CODE_SUCCESS;
   }
   CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false };
-  nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
+  nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
   if (!pSelect->isDistinct) {
-    nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
+    nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
   }
   if (cxt.existAggFunc && cxt.existCol) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP);
@@ -938,7 +939,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
     SNode* pNode = NULL;
     int32_t index = 0;
     FOREACH(pNode, pRetentions) {
-      if (0 == index % 2) {
+      if (0 == ((index++) & 1)) {
         pFreq = (SValueNode*)pNode;
       } else {
         pKeep = (SValueNode*)pNode;
@@ -951,6 +952,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
         taosArrayPush(pReq->pRetensions, &retention);
       }
     }
+    pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -2635,7 +2637,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) {
+int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) {
   STranslateContext cxt = {
     .pParseCxt = pParseCxt,
     .errCode = TSDB_CODE_SUCCESS,

source/libs/parser/src/parser.c

@@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) {
 }
 
 static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) {
-  int32_t code = doParse(pCxt, pQuery);
+  int32_t code = parse(pCxt, pQuery);
   if (TSDB_CODE_SUCCESS == code) {
-    code = doTranslate(pCxt, *pQuery);
+    code = translate(pCxt, *pQuery);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = calculateConstant(pCxt, *pQuery);
   }
   return code;
 }

source/libs/parser/test/parserAstTest.cpp

@@ -54,7 +54,7 @@ private:
   static const int max_err_len = 1024;
 
   bool runImpl(int32_t parseCode, int32_t translateCode) {
-    int32_t code = doParse(&cxt_, &query_);
+    int32_t code = parse(&cxt_, &query_);
     if (code != TSDB_CODE_SUCCESS) {
       parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_;
       return (terrno == parseCode);
@@ -63,7 +63,7 @@ private:
       return false;
     }
     parsedAstStr_ = toString(query_->pRoot);
-    code = doTranslate(&cxt_, query_);
+    code = translate(&cxt_, query_);
     if (code != TSDB_CODE_SUCCESS) {
       translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_;
       return (terrno == translateCode);

source/libs/planner/src/planLogicCreater.c

@@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
 }
 
 static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
-  nodesWalkList(pExprs, doNameExpr, NULL);
+  nodesWalkExprs(pExprs, doNameExpr, NULL);
   SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
   nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
   return cxt.errCode;
@@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  nodesWalkList(pExprs, doCreateColumn, &cxt);
+  nodesWalkExprs(pExprs, doCreateColumn, &cxt);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
     nodesDestroyList(cxt.pList);
     return cxt.errCode;

source/libs/planner/src/planPhysiCreater.c

@@ -264,7 +264,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
     .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
     .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))
   };
-  nodesWalkNode(pRes, doSetSlotId, &cxt);
+  nodesWalkExpr(pRes, doSetSlotId, &cxt);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
     nodesDestroyNode(pRes);
     return cxt.errCode;
@@ -285,7 +285,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
     .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
     .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))
   };
-  nodesWalkList(pRes, doSetSlotId, &cxt);
+  nodesWalkExprs(pRes, doSetSlotId, &cxt);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
     nodesDestroyList(pRes);
     return cxt.errCode;
@@ -606,7 +606,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN
     }
   }
   SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs };
-  nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt);
+  nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt);
   if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
     nodesDestroyList(cxt.pPrecalcExprs);
     *pPrecalcExprs = NULL;

source/libs/planner/test/plannerTest.cpp

@@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) {
   ASSERT_TRUE(run());
 }
 
+TEST_F(PlannerTest, selectConstant) {
+  setDatabase("root", "test");
+
+  bind("SELECT 2-1 FROM t1");
+  ASSERT_TRUE(run());
+}
+
 TEST_F(PlannerTest, stSimple) {
   setDatabase("root", "test");
 

source/libs/scalar/inc/sclInt.h

@@ -47,7 +47,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
 SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
 
 #define GET_PARAM_TYPE(_c)   ((_c)->columnData->info.type)
-#define GET_PARAM_BYTES(_c)  ((_c)->pColumnInfoData->info.bytes)
+#define GET_PARAM_BYTES(_c)  ((_c)->columnData->info.bytes)
 
 void sclFreeParam(SScalarParam *param);
 

source/libs/scalar/src/filter.c

@@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
         resGroup = taosArrayInit(4, sizeof(SFilterGroup));
         
         SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
-        nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
+        nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx);
         FLT_ERR_JRET(tctx.code);
         
         FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
@@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
     if (LOGIC_COND_TYPE_OR == node->condType) {
       SListCell *cell = node->pParameterList->pHead;
       for (int32_t i = 0; i < node->pParameterList->length; ++i) {
-        nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
+        nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext);
         FLT_ERR_JRET(ctx->code);
         
         cell = cell->pNext;
@@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
   filterInitUnitsFields(info);
 
   SFltBuildGroupCtx tctx = {.info = info, .group = group};
-  nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
+  nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx);
   FLT_ERR_JRET(tctx.code);
 
   filterConvertGroupFromArray(info, group);
@@ -3566,7 +3566,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
 }
 
 int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
-  nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
+  nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat);
 
   FLT_RET(pStat->code);
 }

source/libs/scalar/src/scalar.c

@@ -670,7 +670,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
     SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
   
-  nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
+  nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
   SCL_ERR_JRET(ctx.code);
   *pRes = pNode;
 
@@ -694,7 +694,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
     SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
   
-  nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx);
+  nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx);
   SCL_ERR_JRET(ctx.code);
 
   if (pDst) {

source/libs/scalar/src/sclfunc.c

@@ -4,7 +4,18 @@
 #include "sclInt.h"
 #include "sclvector.h"
 
+typedef float (*_float_fn)(float);
+typedef double (*_double_fn)(double);
+typedef double (*_double_fn_2)(double, double);
+typedef int (*_conv_fn)(int);
+typedef void (*_trim_fn)(char *, char*, int32_t, int32_t);
+typedef int16_t (*_len_fn)(char *, int32_t);
+
 /** Math functions **/
+double tlog(double v, double base) {
+  return log(v) / log(base);
+}
+
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   SColumnInfoData *pInputData  = pInput->columnData;
   SColumnInfoData *pOutputData = pOutput->columnData;
@@ -102,14 +113,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
   return TSDB_CODE_SUCCESS;
 }
 
-typedef float (*_float_fn)(float);
-typedef double (*_double_fn)(double);
-typedef double (*_double_fn_2)(double, double);
-
-double tlog(double v, double base) {
-  return log(v) / log(base);
-}
-
 int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) {
   int32_t type = GET_PARAM_TYPE(pInput);
   if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) {
@@ -211,6 +214,407 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p
   return TSDB_CODE_SUCCESS;
 }
 
+/** String functions **/
+int16_t tlength(char *input, int32_t type) {
+  return varDataLen(input);
+}
+
+int16_t tcharlength(char *input, int32_t type) {
+  if (type == TSDB_DATA_TYPE_VARCHAR) {
+    return varDataLen(input);
+  } else { //NCHAR
+    return varDataLen(input) / TSDB_NCHAR_SIZE;
+  }
+}
+
+void tltrim(char *input, char *output, int32_t type, int32_t charLen) {
+  int32_t numOfSpaces = 0;
+  if (type == TSDB_DATA_TYPE_VARCHAR) {
+    for (int32_t i = 0; i < charLen; ++i) {
+      if (!isspace(*(varDataVal(input) + i))) {
+        break;
+      }
+      numOfSpaces++;
+    }
+  } else { //NCHAR
+    for (int32_t i = 0; i < charLen; ++i) {
+      if (!iswspace(*((uint32_t *)varDataVal(input) + i))) {
+        break;
+      }
+      numOfSpaces++;
+    }
+  }
+
+  int32_t resLen;
+  if (type == TSDB_DATA_TYPE_VARCHAR) {
+    resLen = charLen - numOfSpaces;
+    memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen);
+  } else {
+    resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE;
+    memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen);
+  }
+
+  varDataSetLen(output, resLen);
+}
+
+void trtrim(char *input, char *output, int32_t type, int32_t charLen) {
+  int32_t numOfSpaces = 0;
+  if (type == TSDB_DATA_TYPE_VARCHAR) {
+    for (int32_t i = charLen - 1; i >= 0; --i) {
+      if (!isspace(*(varDataVal(input) + i))) {
+        break;
+      }
+      numOfSpaces++;
+    }
+  } else { //NCHAR
+    for (int32_t i = charLen - 1; i < charLen; ++i) {
+      if (!iswspace(*((uint32_t *)varDataVal(input) + i))) {
+        break;
+      }
+      numOfSpaces++;
+    }
+  }
+
+  int32_t resLen;
+  if (type == TSDB_DATA_TYPE_VARCHAR) {
+    resLen = charLen - numOfSpaces;
+  } else {
+    resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE;
+  }
+  memcpy(varDataVal(output), varDataVal(input), resLen);
+
+  varDataSetLen(output, resLen);
+}
+
+int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
+    return TSDB_CODE_FAILED;
+  }
+
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  char *in = pInputData->pData;
+  int16_t *out = (int16_t *)pOutputData->pData;
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_f(pInputData->nullbitmap, i)) {
+      colDataSetNull_f(pOutputData->nullbitmap, i);
+      continue;
+    }
+
+    out[i] = lenFn(in, type);
+    in += varDataTLen(in);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+  return TSDB_CODE_SUCCESS;
+}
+
+static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) {
+  pOutputData->pData = taosMemoryCalloc(len, sizeof(char));
+  pOutputData->info.type = type;
+  pOutputData->info.bytes = len;
+  pOutputData->varmeta.length = len;
+  pOutputData->varmeta.allocLen = len;
+}
+
+int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings
+    return TSDB_CODE_FAILED;
+  }
+
+  SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
+  SColumnInfoData *pOutputData = pOutput->columnData;
+  char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
+  char *outputBuf = NULL;
+
+  int32_t inputLen = 0;
+  int32_t numOfRows = 0;
+  for (int32_t i = 0; i < inputNum; ++i) {
+    if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) ||
+        GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) {
+      return TSDB_CODE_FAILED;
+    }
+    if (pInput[i].numOfRows > numOfRows) {
+      numOfRows = pInput[i].numOfRows;
+    }
+  }
+  for (int32_t i = 0; i < inputNum; ++i) {
+    pInputData[i] = pInput[i].columnData;
+    input[i] = pInputData[i]->pData;
+    if (pInput[i].numOfRows == 1) {
+      inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows;
+    } else {
+      inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE;
+    }
+  }
+
+  int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
+  outputBuf = taosMemoryCalloc(outputLen, 1);
+  char *output = outputBuf;
+
+  bool hasNull = false;
+  for (int32_t k = 0; k < numOfRows; ++k) {
+    for (int32_t i = 0; i < inputNum; ++i) {
+      if (colDataIsNull_s(pInputData[i], k)) {
+        colDataAppendNULL(pOutputData, k);
+        hasNull = true;
+        break;
+      }
+    }
+
+    if (hasNull) {
+      continue;
+    }
+
+    int16_t dataLen = 0;
+    for (int32_t i = 0; i < inputNum; ++i) {
+      memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i]));
+      dataLen += varDataLen(input[i]);
+      if (pInput[i].numOfRows != 1) {
+        input[i] += varDataTLen(input[i]);
+      }
+    }
+    varDataSetLen(output, dataLen);
+    colDataAppend(pOutputData, k, output, false);
+    output += varDataTLen(output);
+  }
+
+  pOutput->numOfRows = numOfRows;
+  taosMemoryFree(input);
+  taosMemoryFree(outputBuf);
+  taosMemoryFree(pInputData);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator
+    return TSDB_CODE_FAILED;
+  }
+
+  SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
+  SColumnInfoData *pOutputData = pOutput->columnData;
+  char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
+  char *outputBuf = NULL;
+
+  int32_t inputLen = 0;
+  int32_t numOfRows = 0;
+  for (int32_t i = 1; i < inputNum; ++i) {
+    if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) ||
+        GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) {
+      return TSDB_CODE_FAILED;
+    }
+    if (pInput[i].numOfRows > numOfRows) {
+      numOfRows = pInput[i].numOfRows;
+    }
+  }
+  for (int32_t i = 0; i < inputNum; ++i) {
+    pInputData[i] = pInput[i].columnData;
+    if (i == 0) {
+      // calculate required separator space
+      int32_t factor =  (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1;
+      inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor;
+    } else if (pInput[i].numOfRows == 1) {
+      inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows;
+    } else {
+      inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE;
+    }
+    input[i] = pInputData[i]->pData;
+  }
+
+  int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
+  outputBuf = taosMemoryCalloc(outputLen, 1);
+  char *output = outputBuf;
+
+  for (int32_t k = 0; k < numOfRows; ++k) {
+    if (colDataIsNull_s(pInputData[0], k)) {
+      colDataAppendNULL(pOutputData, k);
+      continue;
+    }
+
+    int16_t dataLen = 0;
+    for (int32_t i = 1; i < inputNum; ++i) {
+      if (colDataIsNull_s(pInputData[i], k)) {
+        continue;
+      }
+
+      memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i]));
+      dataLen += varDataLen(input[i]);
+      if (pInput[i].numOfRows != 1) {
+        input[i] += varDataTLen(input[i]);
+      }
+
+      if (i < inputNum - 1) {
+        //insert the separator
+        char *sep = pInputData[0]->pData;
+        memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep));
+        dataLen += varDataLen(sep);
+      }
+    }
+    varDataSetLen(output, dataLen);
+    colDataAppend(pOutputData, k, output, false);
+    output += varDataTLen(output);
+  }
+
+  pOutput->numOfRows = numOfRows;
+  taosMemoryFree(input);
+  taosMemoryFree(outputBuf);
+  taosMemoryFree(pInputData);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
+    return TSDB_CODE_FAILED;
+  }
+
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  char *input  = pInputData->pData;
+  char *output = NULL;
+
+  int32_t outputLen = pInputData->varmeta.length;
+  char *outputBuf = taosMemoryCalloc(outputLen, 1);
+  output = outputBuf;
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, i);
+      continue;
+    }
+
+    int32_t len = varDataLen(input);
+    if (type == TSDB_DATA_TYPE_VARCHAR) {
+      for (int32_t j = 0; j < len; ++j) {
+        *(varDataVal(output) + j) = convFn(*(varDataVal(input) + j));
+      }
+    } else { //NCHAR
+      for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) {
+        *((uint32_t *)varDataVal(output) + j) = convFn(*((uint32_t *)varDataVal(input) + j));
+      }
+    }
+    varDataSetLen(output, len);
+    colDataAppend(pOutputData, i, output, false);
+    input += varDataTLen(input);
+    output += varDataTLen(output);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+  taosMemoryFree(outputBuf);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) {
+  int32_t type = GET_PARAM_TYPE(pInput);
+  if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
+    return TSDB_CODE_FAILED;
+  }
+
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  char *input  = pInputData->pData;
+  char *output = NULL;
+
+  int32_t outputLen = pInputData->varmeta.length;
+  char *outputBuf = taosMemoryCalloc(outputLen, 1);
+  output = outputBuf;
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, i);
+      continue;
+    }
+
+    int32_t len = varDataLen(input);
+    int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE;
+    trimFn(input, output, type, charLen);
+
+    varDataSetLen(output, len);
+    colDataAppend(pOutputData, i, output, false);
+    input += varDataTLen(input);
+    output += varDataTLen(output);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+  taosMemoryFree(outputBuf);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  if (inputNum != 2 && inputNum!= 3) {
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t subPos = 0;
+  GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+  if (subPos == 0) { //subPos needs to be positive or negative values;
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t subLen = INT16_MAX;
+  if (inputNum == 3) {
+    GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
+    if (subLen < 0) { //subLen cannot be negative
+      return TSDB_CODE_FAILED;
+    }
+    subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE;
+  }
+
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  char *input  = pInputData->pData;
+  char *output = NULL;
+
+  int32_t outputLen = pInputData->varmeta.length;
+  char *outputBuf = taosMemoryCalloc(outputLen, 1);
+  output = outputBuf;
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, i);
+      continue;
+    }
+
+    int32_t len = varDataLen(input);
+    int32_t startPosBytes;
+
+    if (subPos > 0) {
+      startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE;
+      startPosBytes = MIN(startPosBytes, len);
+    } else {
+      startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE;
+      startPosBytes = MAX(startPosBytes, 0);
+    }
+
+    subLen = MIN(subLen, len - startPosBytes);
+    if (subLen > 0) {
+      memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen);
+    }
+
+    varDataSetLen(output, subLen);
+    colDataAppend(pOutputData, i , output, false);
+    input += varDataTLen(input);
+    output += varDataTLen(output);
+  }
+
+  pOutput->numOfRows = pInput->numOfRows;
+  taosMemoryFree(outputBuf);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return doScalarFunctionUnique(pInput, inputNum, pOutput, atan);
 }
@@ -259,57 +663,28 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
   return doScalarFunction(pInput, inputNum, pOutput, roundf, round);
 }
 
-static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
+int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  assert(numOfInput == 1);
+  return doCaseConvFunction(pInput, inputNum, pOutput, tolower);
-#if 0
-  int64_t* out = (int64_t*) pOutput->data;
-  char* s = pLeft->data;
-
-  for(int32_t i = 0; i < pLeft->num; ++i) {
-    out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes));
-  }
-#endif
 }
 
-static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
+int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  assert(numOfInput > 0);
+  return doCaseConvFunction(pInput, inputNum, pOutput, toupper);
-#if 0
-  int32_t rowLen = 0;
-  int32_t num = 1;
-  for(int32_t i = 0; i < numOfInput; ++i) {
-    rowLen += pLeft[i].bytes;
-
-    if (pLeft[i].num > 1) {
-      num = pLeft[i].num;
-    }
-  }
-
-  pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num);
-  assert(pOutput->data);
-
-  char* rstart = pOutput->data;
-  for(int32_t i = 0; i < num; ++i) {
-
-    char* s = rstart;
-    varDataSetLen(s, 0);
-    for (int32_t j = 0; j < numOfInput; ++j) {
-      char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes);
-
-      memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1));
-      varDataLen(s) += varDataLen(p1);
-    }
-
-    rstart += rowLen;
-  }
-#endif
 }
 
-static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
+int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-
+  return doTrimFunction(pInput, inputNum, pOutput, tltrim);
 }
 
-static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
+int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return doTrimFunction(pInput, inputNum, pOutput, trtrim);
+}
 
+int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return doLengthFunction(pInput, inputNum, pOutput, tlength);
+}
+
+int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return doLengthFunction(pInput, inputNum, pOutput, tcharlength);
 }
 
 static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {

source/libs/transport/src/transSrv.c

@@ -193,6 +193,7 @@ static void uvHandleReq(SSrvConn* pConn) {
   transMsg.ahandle = (void*)pHead->ahandle;
   transMsg.handle = NULL;
 
+  // transDestroyBuffer(&pConn->readBuf);
   transClearBuffer(&pConn->readBuf);
   pConn->inType = pHead->msgType;
   if (pConn->status == ConnNormal) {
@@ -249,6 +250,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
     conn->broken = true;
     if (conn->status == ConnAcquire) {
       if (conn->regArg.init) {
+        tTrace("server conn %p broken, notify server app", conn);
         STrans* pTransInst = conn->pTransInst;
         (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
         memset(&conn->regArg, 0, sizeof(conn->regArg));
@@ -270,7 +272,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
 
 void uvOnSendCb(uv_write_t* req, int status) {
   SSrvConn* conn = req->data;
-  transClearBuffer(&conn->readBuf);
+  // transClearBuffer(&conn->readBuf);
   if (status == 0) {
     tTrace("server conn %p data already was written on stream", conn);
     if (!transQueueEmpty(&conn->srvMsgs)) {

source/libs/transport/test/rclient.c

@@ -23,20 +23,20 @@
 #include "tutil.h"
 
 typedef struct {
-  int       index;
+  int      index;
-  SEpSet    epSet;
+  SEpSet   epSet;
-  int       num;
+  int      num;
-  int       numOfReqs;
+  int      numOfReqs;
-  int       msgSize;
+  int      msgSize;
-  tsem_t    rspSem;
+  tsem_t   rspSem;
-  tsem_t *  pOverSem;
+  tsem_t * pOverSem;
   TdThread thread;
-  void *    pRpc;
+  void *   pRpc;
 } SInfo;
 static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
   SInfo *pInfo = (SInfo *)pMsg->ahandle;
-  tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
+  // tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
-         pMsg->code);
+  //       pMsg->code);
 
   if (pEpSet) pInfo->epSet = *pEpSet;
 
@@ -51,9 +51,9 @@ static void *sendRequest(void *param) {
   SInfo * pInfo = (SInfo *)param;
   SRpcMsg rpcMsg = {0};
 
-  tDebug("thread:%d, start to send request", pInfo->index);
+  tError("thread:%d, start to send request", pInfo->index);
 
-  tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs);
+  tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs);
   int u100 = 0;
   int u500 = 0;
   int u1000 = 0;
@@ -68,7 +68,7 @@ static void *sendRequest(void *param) {
     // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
     int64_t start = taosGetTimestampUs();
     rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
-    if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
+    if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
     // tsem_wait(&pInfo->rspSem);
     tsem_wait(&pInfo->rspSem);
     int64_t end = taosGetTimestampUs() - start;
@@ -88,7 +88,7 @@ static void *sendRequest(void *param) {
   }
 
   tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000);
-  tDebug("thread:%d, it is over", pInfo->index);
+  tError("thread:%d, it is over", pInfo->index);
   tcount++;
 
   return NULL;
@@ -104,7 +104,7 @@ int main(int argc, char *argv[]) {
   char           secret[20] = "mypassword";
   struct timeval systemTime;
   int64_t        startTime, endTime;
-  TdThreadAttr thattr;
+  TdThreadAttr   thattr;
 
   // server info
   epSet.inUse = 0;
@@ -124,7 +124,7 @@ int main(int argc, char *argv[]) {
   rpcInit.ckey = "key";
   rpcInit.spi = 1;
   rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcDebugFlag = 143;
+  rpcDebugFlag = 131;
 
   for (int i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
@@ -170,6 +170,10 @@ int main(int argc, char *argv[]) {
     }
   }
 
+  const char *path = "/tmp/transport/client";
+  taosRemoveDir(path);
+  taosMkDir(path);
+  tstrncpy(tsLogDir, path, PATH_MAX);
   taosInitLog("client.log", 10);
 
   void *pRpc = rpcOpen(&rpcInit);
@@ -178,8 +182,8 @@ int main(int argc, char *argv[]) {
     return -1;
   }
 
-  tInfo("client is initialized");
+  tError("client is initialized");
-  tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs);
+  tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs);
 
   taosGetTimeOfDay(&systemTime);
   startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
@@ -208,8 +212,9 @@ int main(int argc, char *argv[]) {
   endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
   float usedTime = (endTime - startTime) / 1000.0f;  // mseconds
 
-  tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
+  tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
-  tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize);
+  tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime,
+         msgSize);
 
   int ch = getchar();
   UNUSED(ch);

source/libs/transport/test/rserver.c

@@ -125,7 +125,7 @@ int main(int argc, char *argv[]) {
   rpcInit.idleTime = 2 * 1500;
   rpcInit.afp = retrieveAuthInfo;
 
-  rpcDebugFlag = 143;
+  rpcDebugFlag = 131;
 
   for (int i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
@@ -160,6 +160,11 @@ int main(int argc, char *argv[]) {
 
   tsAsyncLog = 0;
   rpcInit.connType = TAOS_CONN_SERVER;
+
+  const char *path = "/tmp/transport/server";
+  taosRemoveDir(path);
+  taosMkDir(path);
+  tstrncpy(tsLogDir, path, PATH_MAX);
   taosInitLog("server.log", 10);
 
   void *pRpc = rpcOpen(&rpcInit);

source/os/src/osFile.c

@@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
   int64_t leftbytes = count;
   int64_t readbytes;
   char   *tbuf = (char *)buf;
@@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
   int64_t ret = pread(pFile->fd, buf, count, offset);
 #if FILE_WITH_LOCK
   taosThreadRwlockUnlock(&(pFile->rwlock));
@@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   int64_t nleft = count;
   int64_t nwritten = 0;
@@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
   int64_t ret = lseek(pFile->fd, (long)offset, whence);
 #if FILE_WITH_LOCK
   taosThreadRwlockUnlock(&(pFile->rwlock));
@@ -429,7 +429,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   struct stat fileStat;
   int32_t code = fstat(pFile->fd, &fileStat);
@@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
 #endif
@@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
 #endif
@@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
   if (pFile == NULL) {
     return 0;
   }
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   return ftruncate(pFile->fd, l_size);
 #endif
@@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
   if (pFile == NULL) {
     return NULL;
   }
-  assert(pFile->fd >= 0);
+  assert(pFile->fd >= 0); // Please check if you have closed the file.
 
   void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
   return ptr;

source/util/src/tcompression.c

@@ -28,11 +28,11 @@
  *
  * BOOLEAN Compression Algorithm:
  *   We provide two methods for compress boolean types. Because boolean types in C
- *   code are char bytes with 0 and 1 values only, only one bit can used to discrimenate
+ *   code are char bytes with 0 and 1 values only, only one bit can used to discriminate
  *   the values.
  *   1. The first method is using only 1 bit to represent the boolean value with 1 for
  *   true and 0 for false. Then the compression rate is 1/8.
- *   2. The second method is using run length encoding (RLE) methods. This methos works
+ *   2. The second method is using run length encoding (RLE) methods. This method works
  *   better when there are a lot of consecutive true values or false values.
  *
  * STRING Compression Algorithm:

source/util/src/tlog.c

@@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) {
   tsLogObj.logHandle->pFile = pFile;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
+  taosSsleep(3);
   taosCloseLogByFd(pOldFile);
 
   uInfo("   new log file:%d is opened", tsLogObj.flag);
@@ -347,16 +348,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   taosThreadMutexInit(&tsLogObj.logMutex, NULL);
 
   taosUmaskFile(0);
-  TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
+  tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
 
-  if (pFile == NULL) {
+  if (tsLogObj.logHandle->pFile == NULL) {
     printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
-  TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
-  tsLogObj.logHandle->pFile = pFile;
-  taosUnLockLogFile(pOldFile);
-  taosCloseFile(&pOldFile);
   taosLockLogFile(tsLogObj.logHandle->pFile);
 
   // only an estimate for number of lines
@@ -750,4 +747,4 @@ void taosSetAllDebugFlag(int32_t flag) {
   fsDebugFlag = flag;
 
   uInfo("all debug flag are set to %d", flag);
-}
+}

tests/script/jenkins/basic.txt

@@ -5,6 +5,7 @@
 ./test.sh -f tsim/user/basic1.sim
 
 # ---- db
+./test.sh -f tsim/db/create_all_options.sim
 ./test.sh -f tsim/db/alter_option.sim
 ./test.sh -f tsim/db/basic1.sim
 ./test.sh -f tsim/db/basic2.sim
@@ -40,7 +41,7 @@
 # ---- tmq
 ./test.sh -f tsim/tmq/basic.sim
 ./test.sh -f tsim/tmq/basic1.sim
-#./test.sh -f tsim/tmq/oneTopic.sim
+./test.sh -f tsim/tmq/oneTopic.sim
 #./test.sh -f tsim/tmq/multiTopic.sim
 
 # --- stable

tests/script/tsim/db/create_all_options.sim

@@ -0,0 +1,486 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+
+$loop_cnt = 0
+check_dnode_ready_1:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> rows: $rows 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+if $data00 != 1 then
+  return -1
+endi
+if $data01 != localhost:7100 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready_1
+endi
+if $data14 != ready then
+  goto check_dnode_ready_1
+endi
+if $data24 != ready then
+  goto check_dnode_ready_1
+endi
+
+print ============= create database with all options
+#database_option: {
+#  | BLOCKS value       [3~1000, default: 6]
+#  | CACHE value        [default: 16]
+#  | CACHELAST value    [0, 1, 2, 3, default: 0]
+#  | COMP               [0 | 1 | 2, default: 2]
+#  | DAYS value         [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
+#  | FSYNC value        [0 ~ 180000 ms, default: 3000]
+#  | MAXROWS value      [200~10000, default: 4096]
+#  | MINROWS value      [10~1000, default: 100]
+#  | KEEP value         [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
+#  | PRECISION          ['ms' | 'us' | 'ns', default: ms]
+#  | QUORUM value       [1 | 2, default: 1]
+#  | REPLICA value      [1 | 3, default: 1]
+#  | TTL value          [1d ~ , default: 1]
+#  | WAL value          [1 | 2, default: 1]
+#  | VGROUPS value      [default: 2]
+#  | SINGLE_STABLE      [0 | 1, default: ]
+#  | STREAM_MODE        [0 | 1, default: ]
+#
+#$data0_db  : name
+#$data1_db  : create_time
+#$data2_db  : vgroups
+#$data3_db  : ntables
+#$data4_db  : replica
+#$data5_db  : quorum
+#$data6_db  : days
+#$data7_db  : keep
+#$data8_db  : cache
+#$data9_db  : blocks
+#$data10_db : minrows
+#$data11_db : maxrows
+#$data12_db : wal
+#$data13_db : fsync
+#$data14_db : comp
+#$data15_db : cachelast
+#$data16_db : precision
+
+print ====> create database db, with default
+sql create database db
+sql show databases
+print rows: $rows
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $rows != 2 then
+  return -1
+endi
+if $data0_db != db then # name
+  return -1
+endi
+if $data2_db != 2 then  # vgroups
+  return -1
+endi
+if $data3_db != 0 then  # ntables
+  return -1
+endi
+if $data4_db != 1 then  # replica
+  return -1
+endi
+if $data5_db != 1 then  # quorum
+  return -1
+endi
+if $data6_db != 14400 then  # days
+  return -1
+endi
+if $data7_db != 5256000,5256000,5256000 then # keep
+  return -1
+endi
+if $data8_db != 16 then  # cache
+  return -1
+endi
+if $data9_db != 6 then  # blocks
+  return -1
+endi
+if $data10_db != 100 then  # minrows
+  return -1
+endi
+if $data11_db != 4096 then  # maxrows
+  return -1
+endi
+if $data12_db != 1 then  # wal
+  return -1
+endi
+if $data13_db != 3000 then  # fsync
+  return -1
+endi
+if $data14_db != 2 then  # comp
+  return -1
+endi
+if $data15_db != 0 then  # cachelast
+  return -1
+endi
+if $data16_db != ms then  # precision
+  return -1
+endi
+sql drop database db
+
+print ====> BLOCKS value       [3~1000, default: 6]
+sql create database db BLOCKS 3
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data9_db != 3 then
+  return -1
+endi
+sql drop database db
+
+sql create database db BLOCKS 1000
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data9_db != 1000 then
+  return -1
+endi
+sql drop database db
+sql_error create database db BLOCKS 2
+sql_error create database db BLOCKS 0
+sql_error create database db BLOCKS -1
+
+print ====> CACHE value [default: 16]
+sql create database db CACHE 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data8_db != 1 then
+  return -1
+endi
+sql drop database db
+
+sql create database db CACHE 128
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data8_db != 128 then
+  return -1
+endi
+sql drop database db
+
+print ====> CACHELAST value [0, 1, 2, 3, default: 0]
+sql create database db CACHELAST 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data15_db != 1 then
+  return -1
+endi
+sql drop database db
+
+sql create database db CACHELAST 2
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data15_db != 2 then
+  return -1
+endi
+sql drop database db
+
+sql create database db CACHELAST 3
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data15_db != 3 then
+  return -1
+endi
+sql drop database db
+sql_error create database db CACHELAST 4
+sql_error create database db CACHELAST -1
+
+print ====> COMP [0 | 1 | 2, default: 2]
+sql create database db COMP 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data14_db != 1 then
+  return -1
+endi
+sql drop database db
+
+sql create database db COMP 0
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data14_db != 0 then
+  return -1
+endi
+sql drop database db
+sql_error create database db COMP 3
+sql_error create database db COMP -1
+
+#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
+#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
+#sql create database db DAYS 60m KEEP 60m
+#sql show databases
+#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $data6_db != 60 then
+#  return -1
+#endi
+#if $data7_db != 60,60,60 then
+#  return -1
+#endi
+#sql drop database db
+#sql create database db DAYS 60m KEEP 1d
+#sql show databases
+#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $data6_db != 60 then
+#  return -1
+#endi
+#if $data7_db != 1440,1440,1440 then
+#  return -1
+#endi
+#sql create database db DAYS 3650d KEEP 365000d
+#sql show databases
+#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $data6_db != 5256000 then
+#  return -1
+#endi
+#if $data7_db != 525600000,525600000,525600000 then
+#  return -1
+#endi
+#sql drop database db
+#sql_error create database db DAYS -59m
+#sql_error create database db DAYS 59m
+#sql_error create database db DAYS 5256001m
+#sql_error create database db DAYS 3651d
+#sql_error create database db KEEP -59m
+#sql_error create database db KEEP 14399m
+#sql_error create database db KEEP 525600001m
+#sql_error create database db KEEP 365001d
+
+print ====> FSYNC value [0 ~ 180000 ms, default: 3000]
+sql create database db FSYNC 0
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data13_db != 0 then
+  return -1
+endi
+sql drop database db
+
+sql create database db FSYNC 180000
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data13_db != 180000 then
+  return -1
+endi
+sql drop database db
+sql_error create database db FSYNC 180001
+sql_error create database db FSYNC -1
+
+print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100]
+sql create database db MAXROWS 10000 MINROWS 1000
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data10_db != 1000 then
+  return -1
+endi
+if $data11_db != 10000 then
+  return -1
+endi
+sql drop database db
+
+sql create database db MAXROWS 200 MINROWS 10
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data10_db != 10 then
+  return -1
+endi
+if $data11_db != 200 then
+  return -1
+endi
+sql drop database db
+sql_error create database db MAXROWS -1
+sql_error create database db MAXROWS 0
+sql_error create database db MAXROWS 199
+sql_error create database db MAXROWS 10001
+sql_error create database db MINROWS -1
+sql_error create database db MINROWS 0
+sql_error create database db MINROWS 9
+sql_error create database db MINROWS 1001
+sql_error create database db MAXROWS 500 MINROWS 1000
+
+print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]
+sql create database db PRECISION 'us'
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data16_db != us then
+  return -1
+endi
+sql drop database db
+
+sql create database db PRECISION 'ns'
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data16_db != ns then
+  return -1
+endi
+sql drop database db
+sql_error create database db PRECISION 'as'
+sql_error create database db PRECISION -1
+
+print ====> QUORUM value [1 | 2, default: 1]
+#sql create database db QUORUM 2
+#sql show databases
+#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $data5_db != 2 then
+#  return -1
+#endi
+#sql drop database db
+
+sql create database db QUORUM 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data5_db != 1 then
+  return -1
+endi
+sql drop database db
+sql_error create database db QUORUM 3
+sql_error create database db QUORUM 0
+sql_error create database db QUORUM -1
+
+print ====> REPLICA value [1 | 3, default: 1]
+sql create database db REPLICA 3
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data4_db != 3 then
+  return -1
+endi
+sql drop database db
+
+sql create database db REPLICA 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data4_db != 1 then
+  return -1
+endi
+sql drop database db
+sql_error create database db REPLICA 2
+sql_error create database db REPLICA 0
+sql_error create database db REPLICA -1
+sql_error create database db REPLICA 4
+
+print ====> TTL value [1d ~ , default: 1]
+sql create database db TTL 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXX_db != 1 then
+#  return -1
+#endi
+sql drop database db
+
+sql create database db TTL 10
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXX_db != 10 then
+#  return -1
+#endi
+sql drop database db
+sql_error create database db TTL 0
+sql_error create database db TTL -1
+
+print ====> WAL value [1 | 2, default: 1]
+sql create database db WAL 2
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data12_db != 2 then
+  return -1
+endi
+sql drop database db
+
+sql create database db WAL 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data12_db != 1 then
+  return -1
+endi
+sql drop database db
+sql_error create database db WAL 3
+sql_error create database db WAL -1
+sql_error create database db WAL 0
+
+print ====> VGROUPS value [1~4096, default: 2]
+sql create database db VGROUPS 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data2_db != 1 then
+  return -1
+endi
+sql drop database db
+
+sql create database db VGROUPS 16
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+if $data2_db != 16 then
+  return -1
+endi
+sql drop database db
+sql_error create database db VGROUPS 4097
+sql_error create database db VGROUPS -1
+sql_error create database db VGROUPS 0
+
+print ====> SINGLE_STABLE [0 | 1, default: ]
+sql create database db SINGLE_STABLE 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXXXX_db != 1 then
+#  return -1
+#endi
+sql drop database db
+
+sql create database db SINGLE_STABLE 0
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXXXX_db != 0 then
+#  return -1
+#endi
+sql drop database db
+sql_error create database db SINGLE_STABLE 2
+sql_error create database db SINGLE_STABLE -1
+
+print ====> STREAM_MODE [0 | 1, default: ]
+sql create database db STREAM_MODE 1
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXXX_db != 1 then
+#  return -1
+#endi
+sql drop database db
+
+sql create database db STREAM_MODE 0
+sql show databases
+print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
+#if $dataXXX_db != 0 then
+#  return -1
+#endi
+sql drop database db
+sql_error create database db STREAM_MODE 2
+sql_error create database db STREAM_MODE -1
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/session.sim

@@ -79,7 +79,7 @@ loop_test_pos:
 
 # session(ts,5a)
 print ====> select count(*) from dev_001 session(ts,5a)
-sql select count(*) from dev_001 session(ts,5a)
+sql select _wstartts, count(*) from dev_001 session(ts,5a)
 print ====> rows: $rows
 print ====> $data00 $data01 $data02 $data03 $data04 $data05
 print ====> $data10 $data11 $data12 $data13 $data14 $data15
@@ -100,7 +100,7 @@ endi
 
 
 print ====> select count(*) from (select * from dev_001) session(ts,5a)
-sql select count(*) from (select * from dev_001) session(ts,5a)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,5a)
 if $rows != 15 then
   return -1
 endi
@@ -109,7 +109,7 @@ if $data01 != 2 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1s)
-sql select count(*) from dev_001 session(ts,1s)
+sql select _wstartts, count(*) from dev_001 session(ts,1s)
 if $rows != 12 then
   return -1
 endi
@@ -118,7 +118,7 @@ if $data01 != 5 then
 endi
 
 print ====> select count(*) from (select * from dev_001) session(ts,1s)
-sql select count(*) from (select * from dev_001) session(ts,1s)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,1s)
 if $rows != 12 then
   return -1
 endi
@@ -127,7 +127,7 @@ if $data01 != 5 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1000a)
-sql select count(*) from dev_001 session(ts,1000a)
+sql select _wstartts, count(*) from dev_001 session(ts,1000a)
 if $rows != 12 then
   return -1
 endi
@@ -136,7 +136,7 @@ if $data01 != 5 then
 endi
 
 print ====> select count(*) from  (select * from dev_001) session(ts,1000a)
-sql select count(*) from  (select * from dev_001) session(ts,1000a)
+sql select _wstartts, count(*) from  (select * from dev_001) session(ts,1000a)
 if $rows != 12 then
   return -1
 endi
@@ -145,7 +145,7 @@ if $data01 != 5 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1m)
-sql select count(*) from dev_001 session(ts,1m)
+sql select _wstartts, count(*) from dev_001 session(ts,1m)
 if $rows != 9 then
   return -1
 endi
@@ -154,7 +154,7 @@ if $data01 != 8 then
 endi
 
 print ====> select count(*) from (select * from dev_001) session(ts,1m)
-sql select count(*) from (select * from dev_001) session(ts,1m)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,1m)
 if $rows != 9 then
   return -1
 endi
@@ -163,7 +163,7 @@ if $data01 != 8 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1h)
-sql select count(*) from dev_001 session(ts,1h)
+sql select _wstartts, count(*) from dev_001 session(ts,1h)
 if $rows != 6 then
   return -1
 endi
@@ -172,7 +172,7 @@ if $data01 != 11 then
 endi
 
 print ====> select count(*) from (select * from dev_001) session(ts,1h)
-sql select count(*) from (select * from dev_001) session(ts,1h)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,1h)
 if $rows != 6 then
   return -1
 endi
@@ -181,7 +181,7 @@ if $data01 != 11 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1d)
-sql select count(*) from dev_001 session(ts,1d)
+sql select _wstartts, count(*) from dev_001 session(ts,1d)
 if $rows != 4 then
   return -1
 endi
@@ -190,7 +190,7 @@ if $data01 != 13 then
 endi
 
 print ====> select count(*) from (select * from dev_001) session(ts,1d)
-sql select count(*) from (select * from dev_001) session(ts,1d)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,1d)
 if $rows != 4 then
   return -1
 endi
@@ -199,7 +199,7 @@ if $data01 != 13 then
 endi
 
 print ====> select count(*) from dev_001 session(ts,1w)
-sql select count(*) from dev_001 session(ts,1w)
+sql select _wstartts, count(*) from dev_001 session(ts,1w)
 if $rows != 2 then
   return -1
 endi
@@ -208,7 +208,7 @@ if $data01 != 15 then
 endi
 
 print ====> select count(*) from (select * from dev_001) session(ts,1w)
-sql select count(*) from (select * from dev_001) session(ts,1w)
+sql select _wstartts, count(*) from (select * from dev_001) session(ts,1w)
 if $rows != 2 then
   return -1
 endi
@@ -216,71 +216,73 @@ if $data01 != 15 then
   return -1
 endi
 
+
+print ====> leastsquares not supported yet.
 print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0)  from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
-sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0)  from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
+#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0)  from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
-if $rows != 2 then
+#if $rows != 2 then
-  return -1
+#  return -1
-endi
+#endi
-if $data01 != 13 then
+#if $data01 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data02 != 1 then
+#if $data02 != 1 then
-  return -1
+#  return -1
-endi
+#endi
-if $data03 != 13 then
+#if $data03 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data04 != 7 then
+#if $data04 != 7 then
-  return -1
+#  return -1
-endi
+#endi
-if $data05 != 91 then
+#if $data05 != 91 then
-  return -1
+#  return -1
-endi
+#endi
-if $data06 != 1 then
+#if $data06 != 1 then
-  return -1
+#  return -1
-endi
+#endi
-if $data07 != 13 then
+#if $data07 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data08 != @{slop:1.000000, intercept:0.000000}@ then
+#if $data08 != @{slop:1.000000, intercept:0.000000}@ then
-  return -1
+#  return -1
-endi
+#endi
-if $data09 != 12 then
+#if $data09 != 12 then
-  return -1
+#  return -1
-endi
+#endi
 # $data0-10 != 3.741657387
 # $data0-11 !=  1
 # $data1-11 !=  14
 
 print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
-sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
+#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
-if $rows != 2 then
+#if $rows != 2 then
-  return -1
+#  return -1
-endi
+#endi
-if $data01 != 13 then
+#if $data01 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data02 != 1 then
+#if $data02 != 1 then
-  return -1
+#  return -1
-endi
+#endi
-if $data03 != 13 then
+#if $data03 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data04 != 7 then
+#if $data04 != 7 then
-  return -1
+#  return -1
-endi
+#endi
-if $data05 != 91 then
+#if $data05 != 91 then
-  return -1
+#  return -1
-endi
+#endi
-if $data06 != 1 then
+#if $data06 != 1 then
-  return -1
+#  return -1
-endi
+#endi
-if $data07 != 13 then
+#if $data07 != 13 then
-  return -1
+#  return -1
-endi
+#endi
-if $data08 != @{slop:1.000000, intercept:0.000000}@ then
+#if $data08 != @{slop:1.000000, intercept:0.000000}@ then
-  return -1
+#  return -1
-endi
+#endi
 
 sql_error select * from dev_001 session(ts,1w)
 sql_error select count(*) from st session(ts,1w)