Merge pull request #1964 from taosdata/feature/query

Feature/query

src/client/inc/tscSecondaryMerge.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 #include "qextbuffer.h"
-#include "qinterpolation.h"
+#include "qfill.h"
 #include "taosmsg.h"
 #include "tlosertree.h"
 #include "tsclient.h"
@@ -60,7 +60,7 @@ typedef struct SLocalReducer {
   char *                 prevRowOfInput;
   tFilePage *            pResultBuf;
   int32_t                nResultBufSize;
-  char *                 pBufForInterpo;  // intermediate buffer for interpolation
+//  char *                 pBufForInterpo;  // intermediate buffer for interpolation
   tFilePage *            pTempBuffer;
   struct SQLFunctionCtx *pCtx;
   int32_t                rowSize;     // size of each intermediate result.
@@ -68,9 +68,9 @@ typedef struct SLocalReducer {
   bool                   hasPrevRow;  // cannot be released
   bool                   hasUnprocessedRow;
   tOrderDescriptor *     pDesc;
-  SColumnModel *            resColModel;
+  SColumnModel *         resColModel;
   tExtMemBuffer **       pExtMemBuffer;      // disk-based buffer
-  SInterpolationInfo     interpolationInfo;  // interpolation support structure
+  SFillInfo*             pFillInfo;         // interpolation support structure
   char *                 pFinalRes;          // result data after interpo
   tFilePage *            discardData;
   SResultInfo *          pResInfo;

src/client/inc/tsclient.h

@@ -30,10 +30,10 @@ extern "C" {
 #include "tsqlfunction.h"
 #include "tutil.h"
 
+#include "qExecutor.h"
 #include "qsqlparser.h"
 #include "qsqltype.h"
 #include "qtsbuf.h"
-#include "queryExecutor.h"
 
 // forward declaration
 struct SSqlInfo;
@@ -210,7 +210,7 @@ typedef struct SQueryInfo {
   SLimitVal        slimit;
   STagCond         tagCond;
   SOrderVal        order;
-  int16_t          interpoType;  // interpolate type
+  int16_t          fillType;  // interpolate type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf *  tsBuf;
@@ -263,7 +263,7 @@ typedef struct SResRec {
 typedef struct {
   int64_t               numOfRows;                  // num of results in current retrieved
   int64_t               numOfTotal;                 // num of total results
-  int64_t               numOfTotalInCurrentClause;  // num of total result in current subclause
+  int64_t               numOfClauseTotal;           // num of total result in current subclause
   char *                pRsp;
   int32_t               rspType;
   int32_t               rspLen;

src/client/src/tscAsync.c

@@ -147,7 +147,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
   
   // local merge has handle this situation during super table non-projection query.
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
-    pRes->numOfTotalInCurrentClause += pRes->numOfRows;
+    pRes->numOfClauseTotal += pRes->numOfRows;
   }
 
   (*pSql->fetchFp)(param, tres, numOfRows);

src/client/src/tscFunctionImpl.c

@@ -16,8 +16,8 @@
 #include "os.h"
 #include "qast.h"
 #include "qextbuffer.h"
+#include "qfill.h"
 #include "qhistogram.h"
-#include "qinterpolation.h"
 #include "qpercentile.h"
 #include "qsyntaxtreefunction.h"
 #include "qtsbuf.h"
@@ -3418,6 +3418,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
   
   int32_t numOfElems = pCtx->size;
   
+  // todo : opt with pre-calculated result
   // column missing cause the hasNull to be true
   if (usePreVal(pCtx)) {
     numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
@@ -3446,13 +3447,13 @@ static void spread_function(SQLFunctionCtx *pCtx) {
       }
     }
   } else {
-    if (pInfo->min > pCtx->param[1].dKey) {
+//    if (pInfo->min > pCtx->param[1].dKey) {
-      pInfo->min = pCtx->param[1].dKey;
+//      pInfo->min = pCtx->param[1].dKey;
-    }
+//    }
-    
+//
-    if (pInfo->max < pCtx->param[2].dKey) {
+//    if (pInfo->max < pCtx->param[2].dKey) {
-      pInfo->max = pCtx->param[2].dKey;
+//      pInfo->max = pCtx->param[2].dKey;
-    }
+//    }
   }
   
   void *pData = GET_INPUT_CHAR(pCtx);
@@ -3866,16 +3867,16 @@ static void interp_function(SQLFunctionCtx *pCtx) {
     SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail;
     
     /* set no output result */
-    if (pInfoDetail->type == TSDB_INTERPO_NONE) {
+    if (pInfoDetail->type == TSDB_FILL_NONE) {
       pCtx->param[3].i64Key = 0;
     } else if (pInfoDetail->primaryCol == 1) {
       *(TSKEY *)pCtx->aOutputBuf = pInfoDetail->ts;
     } else {
-      if (pInfoDetail->type == TSDB_INTERPO_NULL) {
+      if (pInfoDetail->type == TSDB_FILL_NULL) {
         setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
-      } else if (pInfoDetail->type == TSDB_INTERPO_SET_VALUE) {
+      } else if (pInfoDetail->type == TSDB_FILL_SET_VALUE) {
         tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType);
-      } else if (pInfoDetail->type == TSDB_INTERPO_PREV) {
+      } else if (pInfoDetail->type == TSDB_FILL_PREV) {
         char *data = pCtx->param[1].pz;
         char *pVal = data + TSDB_KEYSIZE;
         
@@ -3886,7 +3887,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
           assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType);
         }
         
-      } else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) {
+      } else if (pInfoDetail->type == TSDB_FILL_LINEAR) {
         char *data1 = pCtx->param[1].pz;
         char *data2 = pCtx->param[2].pz;
         

src/client/src/tscPrepare.c

@@ -447,7 +447,7 @@ static int insertStmtExecute(STscStmt* stmt) {
   SSqlRes *pRes = &pSql->res;
   pRes->numOfRows = 0;
   pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
   
   pRes->qhandle = 0;
 

src/client/src/tscSQLParser.c

@@ -4020,19 +4020,19 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
   }
 
   if (strncasecmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) {
-    pQueryInfo->interpoType = TSDB_INTERPO_NONE;
+    pQueryInfo->fillType = TSDB_FILL_NONE;
   } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) {
-    pQueryInfo->interpoType = TSDB_INTERPO_NULL;
+    pQueryInfo->fillType = TSDB_FILL_NULL;
     for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) {
       TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
       setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes);
     }
   } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) {
-    pQueryInfo->interpoType = TSDB_INTERPO_PREV;
+    pQueryInfo->fillType = TSDB_FILL_PREV;
   } else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) {
-    pQueryInfo->interpoType = TSDB_INTERPO_LINEAR;
+    pQueryInfo->fillType = TSDB_FILL_LINEAR;
   } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) {
-    pQueryInfo->interpoType = TSDB_INTERPO_SET_VALUE;
+    pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
 
     if (pFillToken->nExpr == 1) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
@@ -5562,7 +5562,15 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
       }
       
       ret = tVariantDump(&(pList->a[i].pVar), varDataVal(tagVal), pTagSchema[i].type);
-      varDataSetLen(tagVal, pList->a[i].pVar.nLen);
+      if (pList->a[i].pVar.nType == TSDB_DATA_TYPE_NULL) {
+        if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY) {
+          varDataSetLen(tagVal, sizeof(uint8_t));
+        } else {
+          varDataSetLen(tagVal, sizeof(uint32_t));
+        }
+      } else { // todo refactor
+        varDataSetLen(tagVal, pList->a[i].pVar.nLen);
+      }
     } else {
       ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type);
     }

src/client/src/tscSecondaryMerge.c

@@ -25,7 +25,7 @@
 typedef struct SCompareParam {
   SLocalDataSource **pLocalData;
   tOrderDescriptor * pDesc;
-  int32_t            numOfElems;
+  int32_t            num;
   int32_t            groupOrderType;
 } SCompareParam;
 
@@ -47,11 +47,11 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
   }
 
   if (pParam->groupOrderType == TSDB_ORDER_DESC) {  // desc
-    return compare_d(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
+    return compare_d(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
-                     pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
+                     pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
   } else {
-    return compare_a(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
+    return compare_a(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data,
-                     pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
+                     pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data);
   }
 }
 
@@ -132,6 +132,26 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
   }
 }
 
+static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) {
+  int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
+  int32_t offset = 0;
+  
+  SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+    
+    pFillCol[i].col.bytes  = pExpr->resBytes;
+    pFillCol[i].col.type   = pExpr->resType;
+    pFillCol[i].flag       = pExpr->colInfo.flag;
+    pFillCol[i].col.offset = offset;
+    pFillCol[i].functionId = pExpr->functionId;
+    pFillCol[i].defaultVal.i = pQueryInfo->defaultVal[i];
+    offset += pExpr->resBytes;
+  }
+  
+  return pFillCol;
+}
+
 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
                            SColumnModel *finalmodel, SSqlObj* pSql) {
   SSqlCmd* pCmd = &pSql->cmd;
@@ -217,24 +237,24 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
       ds->pMemBuffer = pMemBuffer[i];
       ds->flushoutIdx = j;
-      ds->filePage.numOfElems = 0;
+      ds->filePage.num = 0;
       ds->pageId = 0;
       ds->rowIdx = 0;
 
       tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSql, i + 1, idx + 1);
       tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0);
 #ifdef _DEBUG_VIEW
-      printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.numOfElems);
+      printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.num);
       SSrcColumnInfo colInfo[256] = {0};
       SQueryInfo *   pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
       tscGetSrcColumnInfo(colInfo, pQueryInfo);
 
-      tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.numOfElems,
+      tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.num,
                          pMemBuffer[0]->numOfElemsPerPage, colInfo);
 #endif
       
-      if (ds->filePage.numOfElems == 0) {  // no data in this flush, the index does not increase
+      if (ds->filePage.num == 0) {  // no data in this flush, the index does not increase
         tscTrace("%p flush data is empty, ignore %d flush record", pSql, idx);
         tfree(ds);
         continue;
@@ -254,7 +274,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   SCompareParam *param = malloc(sizeof(SCompareParam));
   param->pLocalData = pReducer->pLocalDataSrc;
   param->pDesc = pReducer->pDesc;
-  param->numOfElems = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
+  param->num = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
   param->groupOrderType = pQueryInfo->groupbyExpr.orderType;
@@ -295,25 +315,25 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   assert(finalRowLength <= pReducer->rowSize);
 
   pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
-  pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize);
+//  pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize);
 
   if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
-      pReducer->pBufForInterpo == NULL || pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
+      /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
     tfree(pReducer->pTempBuffer);
     tfree(pReducer->discardData);
     tfree(pReducer->pResultBuf);
     tfree(pReducer->pFinalRes);
-    tfree(pReducer->pBufForInterpo);
+//    tfree(pReducer->pBufForInterpo);
     tfree(pReducer->prevRowOfInput);
 
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
     return;
   }
-
-  size = tscSqlExprNumOfExprs(pQueryInfo);
   
-  pReducer->pTempBuffer->numOfElems = 0;
+  size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
-  pReducer->pResInfo = calloc(size, sizeof(SResultInfo));
+  
+  pReducer->pTempBuffer->num = 0;
+  pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));
 
   tscCreateResPointerInfo(pRes, pQueryInfo);
   tscInitSqlContext(pCmd, pReducer, pDesc);
@@ -333,55 +353,58 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pRes->numOfGroups = 0;
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
+  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);;
   
-  int16_t prec = tinfo.precision;
+  TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
-  int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
   int64_t revisedSTime =
-      taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec);
+      taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
-
+  
-  SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo;
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-  taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
+    SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
-                      pReducer->rowSize);
+    pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
+                                           4096, numOfCols, pQueryInfo->slidingTime, pQueryInfo->fillType, pFillCol);
+  }
 
   int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
 
-  if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
+  if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pReducer->pFillInfo != NULL) {
-    pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols;
+    pReducer->pFillInfo->pTags[0] = (char *)pReducer->pFillInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols;
     for (int32_t i = 1; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
       SSchema *pSchema = getColumnModelSchema(pReducer->resColModel, startIndex + i - 1);
-      pInterpoInfo->pTags[i] = pSchema->bytes + pInterpoInfo->pTags[i - 1];
+      pReducer->pFillInfo->pTags[i] = pSchema->bytes + pReducer->pFillInfo->pTags[i - 1];
     }
   } else {
-    assert(pInterpoInfo->pTags == NULL);
+    if (pReducer->pFillInfo != NULL) {
+      assert(pReducer->pFillInfo->pTags == NULL);
+    }
   }
 }
 
 static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage,
                                      int32_t orderType) {
-  if (pPage->numOfElems == 0) {
+  if (pPage->num == 0) {
     return 0;
   }
 
-  assert(pPage->numOfElems <= pDesc->pColumnModel->capacity);
+  assert(pPage->num <= pDesc->pColumnModel->capacity);
 
   // sort before flush to disk, the data must be consecutively put on tFilePage.
   if (pDesc->orderIdx.numOfCols > 0) {
-    tColDataQSort(pDesc, pPage->numOfElems, 0, pPage->numOfElems - 1, pPage->data, orderType);
+    tColDataQSort(pDesc, pPage->num, 0, pPage->num - 1, pPage->data, orderType);
   }
 
 #ifdef _DEBUG_VIEW
-  printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->numOfElems);
+  printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->num);
-  tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->numOfElems, pPage->numOfElems);
+  tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->num, pPage->num);
 #endif
 
   // write to cache after being sorted
-  if (tExtMemBufferPut(pMemoryBuf, pPage->data, pPage->numOfElems) < 0) {
+  if (tExtMemBufferPut(pMemoryBuf, pPage->data, pPage->num) < 0) {
     tscError("failed to save data in temporary buffer");
     return -1;
   }
 
-  pPage->numOfElems = 0;
+  pPage->num = 0;
   return 0;
 }
 
@@ -402,17 +425,17 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
                      int32_t numOfRows, int32_t orderType) {
   SColumnModel *pModel = pDesc->pColumnModel;
 
-  if (pPage->numOfElems + numOfRows <= pModel->capacity) {
+  if (pPage->num + numOfRows <= pModel->capacity) {
     tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows);
     return 0;
   }
 
   // current buffer is overflow, flush data to extensive buffer
-  int32_t numOfRemainEntries = pModel->capacity - pPage->numOfElems;
+  int32_t numOfRemainEntries = pModel->capacity - pPage->num;
   tColModelAppend(pModel, pPage, data, 0, numOfRemainEntries, numOfRows);
 
   // current buffer is full, need to flushed to disk
-  assert(pPage->numOfElems == pModel->capacity);
+  assert(pPage->num == pModel->capacity);
   int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
   if (ret != 0) {
     return -1;
@@ -430,12 +453,12 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
 
     tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows);
 
-    if (pPage->numOfElems == pModel->capacity) {
+    if (pPage->num == pModel->capacity) {
       if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) {
         return -1;
       }
     } else {
-      pPage->numOfElems = numOfWriteElems;
+      pPage->num = numOfWriteElems;
     }
 
     remain -= numOfWriteElems;
@@ -470,7 +493,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
       tscTrace("%p waiting for delete procedure, status: %d", pSql, status);
     }
 
-    taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo);
+    taosDestoryFillInfo(pLocalReducer->pFillInfo);
 
     if (pLocalReducer->pCtx != NULL) {
       for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
@@ -503,8 +526,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
       tfree(pLocalReducer->pLoserTree);
     }
 
-    tfree(pLocalReducer->pBufForInterpo);
-
     tfree(pLocalReducer->pFinalRes);
     tfree(pLocalReducer->discardData);
 
@@ -740,7 +761,7 @@ int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *p
 #if defined(_DEBUG_VIEW)
     printf("new page load to buffer\n");
     tColModelDisplay(pOneInterDataSrc->pMemBuffer->pColumnModel, pOneInterDataSrc->filePage.data,
-                     pOneInterDataSrc->filePage.numOfElems, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity);
+                     pOneInterDataSrc->filePage.num, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity);
 #endif
     *needAdjustLoserTree = true;
   } else {
@@ -761,7 +782,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
    * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree
    */
   bool needToAdjust = true;
-  if (pOneInterDataSrc->filePage.numOfElems <= pOneInterDataSrc->rowIdx) {
+  if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) {
     loadNewDataFromDiskFor(pLocalReducer, pOneInterDataSrc, &needToAdjust);
   }
 
@@ -788,22 +809,20 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
 }
 
 void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo,
-                                       SInterpolationInfo *pInterpoInfo) {
+                                       SFillInfo *pFillInfo) {
   // discard following dataset in the same group and reset the interpolation information
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  
+
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-  
+
   int16_t prec = tinfo.precision;
   int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
   int64_t revisedSTime =
       taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec);
-
+  taosResetFillInfo(pFillInfo, revisedSTime);
-  taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
-                      pLocalReducer->rowSize);
 
   pLocalReducer->discard = true;
-  pLocalReducer->discardData->numOfElems = 0;
+  pLocalReducer->discardData->num = 0;
 
   SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel;
   tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1);
@@ -856,6 +875,7 @@ static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRe
 static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) {
   SSqlCmd *   pCmd = &pSql->cmd;
   SSqlRes *   pRes = &pSql->res;
+  
   tFilePage * pFinalDataPage = pLocalReducer->pResultBuf;
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
@@ -868,91 +888,66 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
     assert(pRes->pLocalReducer == NULL);
   }
 
-  if (pQueryInfo->intervalTime == 0 || pQueryInfo->interpoType == TSDB_INTERPO_NONE) {
+  if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) {
-    // no interval query, no interpolation
+    // no interval query, no fill operation
     pRes->data = pLocalReducer->pFinalRes;
-    pRes->numOfRows = pFinalDataPage->numOfElems;
+    pRes->numOfRows = pFinalDataPage->num;
-    pRes->numOfTotalInCurrentClause += pRes->numOfRows;
+    pRes->numOfClauseTotal += pRes->numOfRows;
 
     if (pQueryInfo->limit.offset > 0) {
       if (pQueryInfo->limit.offset < pRes->numOfRows) {
-        int32_t prevSize = pFinalDataPage->numOfElems;
+        int32_t prevSize = pFinalDataPage->num;
         tColModelErase(pLocalReducer->resColModel, pFinalDataPage, prevSize, 0, pQueryInfo->limit.offset - 1);
 
         /* remove the hole in column model */
         tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize);
 
         pRes->numOfRows -= pQueryInfo->limit.offset;
-        pRes->numOfTotalInCurrentClause -= pQueryInfo->limit.offset;
+        pRes->numOfClauseTotal -= pQueryInfo->limit.offset;
         pQueryInfo->limit.offset = 0;
       } else {
         pQueryInfo->limit.offset -= pRes->numOfRows;
         pRes->numOfRows = 0;
-        pRes->numOfTotalInCurrentClause = 0;
+        pRes->numOfClauseTotal = 0;
       }
     }
 
-    if (pQueryInfo->limit.limit >= 0 && pRes->numOfTotalInCurrentClause > pQueryInfo->limit.limit) {
+    if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
       /* impose the limitation of output rows on the final result */
-      int32_t prevSize = pFinalDataPage->numOfElems;
+      int32_t prevSize = pFinalDataPage->num;
-      int32_t overFlow = pRes->numOfTotalInCurrentClause - pQueryInfo->limit.limit;
+      int32_t overFlow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
 
       assert(overFlow < pRes->numOfRows);
 
-      pRes->numOfTotalInCurrentClause = pQueryInfo->limit.limit;
+      pRes->numOfClauseTotal = pQueryInfo->limit.limit;
       pRes->numOfRows -= overFlow;
-      pFinalDataPage->numOfElems -= overFlow;
+      pFinalDataPage->num -= overFlow;
 
       tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize);
 
       /* set remain data to be discarded, and reset the interpolation information */
-      savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo);
+      savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo);
     }
 
     int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList);
     memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize);
 
-    pFinalDataPage->numOfElems = 0;
+    pFinalDataPage->num = 0;
     return;
   }
 
-  int64_t *pPrimaryKeys = (int64_t *)pLocalReducer->pBufForInterpo;
+  SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
-
+  int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey);
-  SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
-
-  int64_t actualETime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
 
   tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput);
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
     pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity);
   }
-
-  char **  srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutput);
-  int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutput * sizeof(void *));
-
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    srcData[i] =
-        pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows;
-    functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId;
-  }
-
-  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
   
-  int8_t precision = tinfo.precision;
-
   while (1) {
-    int32_t remains = taosNumOfRemainPoints(pInterpoInfo);
+    int64_t newRows = -1;
-    TSKEY   etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->intervalTime,
+    taosGenerateDataBlock(pFillInfo, pResPages, &newRows, pLocalReducer->resColModel->capacity);
-                                       pQueryInfo->slidingTimeUnit, precision);
-    int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->intervalTime, etime,
-                                                  pLocalReducer->resColModel->capacity);
-
-    int32_t newRows = taosDoInterpoResult(pInterpoInfo, pQueryInfo->interpoType, pResPages, remains, nrows,
-                                          pQueryInfo->intervalTime, pPrimaryKeys, pLocalReducer->resColModel, srcData,
-                                          pQueryInfo->defaultVal, functions, pLocalReducer->resColModel->capacity);
-    assert(newRows <= nrows);
 
     if (pQueryInfo->limit.offset < newRows) {
       newRows -= pQueryInfo->limit.offset;
@@ -967,7 +962,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
 
       pRes->data = pLocalReducer->pFinalRes;
       pRes->numOfRows = newRows;
-      pRes->numOfTotalInCurrentClause += newRows;
+      pRes->numOfClauseTotal += newRows;
 
       pQueryInfo->limit.offset = 0;
       break;
@@ -975,16 +970,15 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
       pQueryInfo->limit.offset -= newRows;
       pRes->numOfRows = 0;
 
-      int32_t rpoints = taosNumOfRemainPoints(pInterpoInfo);
+      int32_t rpoints = taosNumOfRemainRows(pFillInfo);
       if (rpoints <= 0) {
-        if (!doneOutput) {
+        if (!doneOutput) { // reduce procedure has not completed yet, but current results for fill are exhausted
-          /* reduce procedure is not completed, but current results for interpolation are exhausted */
           break;
         }
 
         /* all output for current group are completed */
         int32_t totalRemainRows =
-            taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeys, rpoints, pQueryInfo->intervalTime, actualETime);
+            taosGetNumOfResultWithFill(pFillInfo, rpoints, pFillInfo->slidingTime, actualETime);
         if (totalRemainRows <= 0) {
           break;
         }
@@ -993,17 +987,17 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
   }
 
   if (pRes->numOfRows > 0) {
-    if (pQueryInfo->limit.limit >= 0 && pRes->numOfTotalInCurrentClause > pQueryInfo->limit.limit) {
+    if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) {
-      int32_t overFlow = pRes->numOfTotalInCurrentClause - pQueryInfo->limit.limit;
+      int32_t overFlow = pRes->numOfClauseTotal - pQueryInfo->limit.limit;
       pRes->numOfRows -= overFlow;
 
       assert(pRes->numOfRows >= 0);
 
-      pRes->numOfTotalInCurrentClause = pQueryInfo->limit.limit;
+      pRes->numOfClauseTotal = pQueryInfo->limit.limit;
-      pFinalDataPage->numOfElems -= overFlow;
+      pFinalDataPage->num -= overFlow;
 
       /* set remain data to be discarded, and reset the interpolation information */
-      savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pInterpoInfo);
+      savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pFillInfo);
     }
 
     if (pQueryInfo->order.order == TSDB_ORDER_ASC) {
@@ -1017,18 +1011,17 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
     }
   }
 
-  pFinalDataPage->numOfElems = 0;
+  pFinalDataPage->num = 0;
   for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
     tfree(pResPages[i]);
   }
+  
   tfree(pResPages);
-
-  free(srcData);
 }
 
 static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
   SColumnModel *pColumnModel = pLocalReducer->pDesc->pColumnModel;
-  assert(pColumnModel->capacity == 1 && tmpBuffer->numOfElems == 1);
+  assert(pColumnModel->capacity == 1 && tmpBuffer->num == 1);
 
   // copy to previous temp buffer
   for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) {
@@ -1038,7 +1031,7 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer)
     memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes);
   }
 
-  tmpBuffer->numOfElems = 0;
+  tmpBuffer->num = 0;
   pLocalReducer->hasPrevRow = true;
 }
 
@@ -1168,7 +1161,7 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {
   pLocalReducer->hasPrevRow = false;
 
   int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalReducer->pCtx);
-  pLocalReducer->pResultBuf->numOfElems += numOfRes;
+  pLocalReducer->pResultBuf->num += numOfRes;
 
   fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalReducer);
   return numOfRes;
@@ -1221,7 +1214,7 @@ static bool saveGroupResultInfo(SSqlObj *pSql) {
 
   //    pRes->pGroupRec = realloc(pRes->pGroupRec, pRes->numOfGroups*sizeof(SResRec));
   //    pRes->pGroupRec[pRes->numOfGroups-1].numOfRows = pRes->numOfRows;
-  //    pRes->pGroupRec[pRes->numOfGroups-1].numOfTotalInCurrentClause = pRes->numOfTotalInCurrentClause;
+  //    pRes->pGroupRec[pRes->numOfGroups-1].numOfClauseTotal = pRes->numOfClauseTotal;
 
   return false;
 }
@@ -1244,37 +1237,38 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
   pRes->code = TSDB_CODE_SUCCESS;
 
   /*
-   * ignore the output of the current group since this group is skipped by user
+   * Ignore the output of the current group since this group is skipped by user
    * We set the numOfRows to be 0 and discard the possible remain results.
    */
   if (pQueryInfo->slimit.offset > 0) {
     pRes->numOfRows = 0;
     pQueryInfo->slimit.offset -= 1;
     pLocalReducer->discard = !noMoreCurrentGroupRes;
+    
     return false;
   }
 
   tColModelCompact(pModel, pResBuf, pModel->capacity);
-  memcpy(pLocalReducer->pBufForInterpo, pResBuf->data, pLocalReducer->nResultBufSize);
 
 #ifdef _DEBUG_VIEW
   printf("final result before interpo:\n");
-  tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->numOfElems, pResBuf->numOfElems);
+  assert(0);
+//  tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
 #endif
-
+  
-  SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
+  SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
-  int32_t             startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
+  if (pFillInfo != NULL) {
-
+    STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
+    STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-    int16_t  offset = getColumnModelOffset(pModel, startIndex + i);
+  
-    SSchema *pSchema = getColumnModelSchema(pModel, startIndex + i);
+    TSKEY ekey = taosGetRevisedEndKey(pQueryInfo->window.ekey, pFillInfo->order, pFillInfo->slidingTime,
-
+                                      pQueryInfo->slidingTimeUnit, tinfo.precision);
-    memcpy(pInterpoInfo->pTags[i], pLocalReducer->pBufForInterpo + offset * pResBuf->numOfElems, pSchema->bytes);
+  
+    taosFillSetStartInfo(pFillInfo, pResBuf->num, ekey);
+    taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf);
   }
-
+  
-  taosInterpoSetStartInfo(&pLocalReducer->interpolationInfo, pResBuf->numOfElems, pQueryInfo->interpoType);
   doInterpolateResult(pSql, pLocalReducer, noMoreCurrentGroupRes);
-
   return true;
 }
 
@@ -1290,7 +1284,7 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {  //
 static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) {
   // In handling data in other groups, we need to reset the interpolation information for a new group data
   pRes->numOfRows = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
@@ -1302,13 +1296,13 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer
   int8_t precision = tinfo.precision;
 
   // for group result interpolation, do not return if not data is generated
-  if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-    int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
+    TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
     int64_t newTime =
-        taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision);
+        taosGetIntervalStartTimestamp(skey, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision);
-
+//    taosResetFillInfo(pLocalReducer->pFillInfo, pQueryInfo->order.order, newTime,
-    taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime,
+//                        pQueryInfo->groupbyExpr.numOfGroupCols, 4096, 0, NULL, pLocalReducer->rowSize);
-                        pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize);
+    taosResetFillInfo(pLocalReducer->pFillInfo, newTime);
   }
 }
 
@@ -1320,26 +1314,26 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
-  SQueryInfo *        pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  SLocalReducer *     pLocalReducer = pRes->pLocalReducer;
+  SLocalReducer *pLocalReducer = pRes->pLocalReducer;
-  SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
+  SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
 
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
   
   int8_t p = tinfo.precision;
 
-  if (taosHasRemainsDataForInterpolation(pInterpoInfo)) {
+  if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) {
-    assert(pQueryInfo->interpoType != TSDB_INTERPO_NONE);
+    assert(pQueryInfo->fillType != TSDB_FILL_NONE);
 
     tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf;
-    int64_t    etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pInterpoInfo->numOfRawDataInRows - 1));
+    int64_t    etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
 
-    int32_t remain = taosNumOfRemainPoints(pInterpoInfo);
+    int32_t remain = taosNumOfRemainRows(pFillInfo);
-    TSKEY   ekey =
+    TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, p);
-        taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, p);
+    
-    int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain,
+    // the first column must be the timestamp column
-                                                 pQueryInfo->intervalTime, ekey, pLocalReducer->resColModel->capacity);
+    int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pLocalReducer->resColModel->capacity);
     if (rows > 0) {  // do interpo
       doInterpolateResult(pSql, pLocalReducer, false);
     }
@@ -1355,7 +1349,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
 
   SLocalReducer *     pLocalReducer = pRes->pLocalReducer;
-  SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
+  SFillInfo *pFillInfo = pLocalReducer->pFillInfo;
 
   bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
 
@@ -1363,18 +1357,16 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-  int8_t precision = tinfo.precision;
 
   if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
       prevGroupCompleted) {
-    // if interpoType == TSDB_INTERPO_NONE, return directly
+    // if fillType == TSDB_FILL_NONE, return directly
-    if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
+    if (pQueryInfo->fillType != TSDB_FILL_NONE) {
       int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey;
 
       etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime,
-                                   pQueryInfo->slidingTimeUnit, precision);
+                                   pQueryInfo->slidingTimeUnit, tinfo.precision);
-      int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->intervalTime, etime,
+      int32_t rows = taosGetNumOfResultWithFill(pFillInfo, 0, etime, pLocalReducer->resColModel->capacity);
-                                                   pLocalReducer->resColModel->capacity);
       if (rows > 0) {  // do interpo
         doInterpolateResult(pSql, pLocalReducer, true);
       }
@@ -1474,7 +1466,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
     printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index);
 #endif
     assert((pTree->pNode[0].index < pLocalReducer->numOfBuffer) && (pTree->pNode[0].index >= 0) &&
-           tmpBuffer->numOfElems == 0);
+           tmpBuffer->num == 0);
 
     // chosen from loser tree
     SLocalDataSource *pOneDataSrc = pLocalReducer->pLocalDataSrc[pTree->pNode[0].index];
@@ -1487,7 +1479,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
     SSrcColumnInfo colInfo[256] = {0};
     tscGetSrcColumnInfo(colInfo, pQueryInfo);
 
-    tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->numOfElems, pModel->capacity, colInfo);
+    tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo);
 #endif
 
     if (pLocalReducer->discard) {
@@ -1495,7 +1487,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
 
       /* current record belongs to the same group of previous record, need to discard it */
       if (isSameGroup(pCmd, pLocalReducer, pLocalReducer->discardData->data, tmpBuffer)) {
-        tmpBuffer->numOfElems = 0;
+        tmpBuffer->num = 0;
         pOneDataSrc->rowIdx += 1;
 
         adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree);
@@ -1509,7 +1501,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
         continue;
       } else {
         pLocalReducer->discard = false;
-        pLocalReducer->discardData->numOfElems = 0;
+        pLocalReducer->discardData->num = 0;
 
         if (saveGroupResultInfo(pSql)) {
           pLocalReducer->status = TSC_LOCALREDUCE_READY;
@@ -1538,17 +1530,17 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
         tFilePage *pResBuf = pLocalReducer->pResultBuf;
 
         /*
-         * if the previous group does NOT generate any result (pResBuf->numOfElems == 0),
+         * if the previous group does NOT generate any result (pResBuf->num == 0),
          * continue to process results instead of return results.
          */
-        if ((!sameGroup && pResBuf->numOfElems > 0) || (pResBuf->numOfElems == pLocalReducer->resColModel->capacity)) {
+        if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalReducer->resColModel->capacity)) {
           // does not belong to the same group
           bool notSkipped = doGenerateFinalResults(pSql, pLocalReducer, !sameGroup);
 
           // this row needs to discard, since it belongs to the group of previous
           if (pLocalReducer->discard && sameGroup) {
             pLocalReducer->hasUnprocessedRow = false;
-            tmpBuffer->numOfElems = 0;
+            tmpBuffer->num = 0;
           } else {
             // current row does not belongs to the previous group, so it is not be handled yet.
             pLocalReducer->hasUnprocessedRow = true;
@@ -1611,7 +1603,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
     finalizeRes(pQueryInfo, pLocalReducer);
   }
 
-  if (pLocalReducer->pResultBuf->numOfElems) {
+  if (pLocalReducer->pResultBuf->num) {
     doGenerateFinalResults(pSql, pLocalReducer, true);
   }
 
@@ -1641,6 +1633,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
   size_t allocSize = numOfRes * rowLen + sizeof(tFilePage) + 1;
   pRes->pLocalReducer->pResultBuf = (tFilePage *)calloc(1, allocSize);
 
-  pRes->pLocalReducer->pResultBuf->numOfElems = numOfRes;
+  pRes->pLocalReducer->pResultBuf->num = numOfRes;
   pRes->data = pRes->pLocalReducer->pResultBuf->data;
 }

src/client/src/tscServer.c

@@ -652,7 +652,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   pQueryMsg->order          = htons(pQueryInfo->order.order);
   pQueryMsg->orderColId     = htons(pQueryInfo->order.orderColId);
-  pQueryMsg->interpoType    = htons(pQueryInfo->interpoType);
+  pQueryMsg->fillType    = htons(pQueryInfo->fillType);
   pQueryMsg->limit          = htobe64(pQueryInfo->limit.limit);
   pQueryMsg->offset         = htobe64(pQueryInfo->limit.offset);
   pQueryMsg->numOfCols      = htons(taosArrayGetSize(pQueryInfo->colList));
@@ -780,7 +780,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     }
   }
 
-  if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
     for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
       *((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]);
       pMsg += sizeof(pQueryInfo->defaultVal[0]);

src/client/src/tscSql.c

@@ -228,7 +228,7 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) {
   
   pRes->numOfRows  = 1;
   pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   pCmd->curSql = NULL;
   if (NULL != pCmd->pTableList) {
@@ -407,7 +407,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
 
   // secondary merge has handle this situation
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
-    pRes->numOfTotalInCurrentClause += pRes->numOfRows;
+    pRes->numOfClauseTotal += pRes->numOfRows;
   }
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
@@ -490,8 +490,8 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
     pSql->cmd.command = pQueryInfo->command;
     pCmd->clauseIndex++;
 
-    pRes->numOfTotal += pRes->numOfTotalInCurrentClause;
+    pRes->numOfTotal += pRes->numOfClauseTotal;
-    pRes->numOfTotalInCurrentClause = 0;
+    pRes->numOfClauseTotal = 0;
     pRes->rspType = 0;
 
     pSql->numOfSubs = 0;
@@ -790,7 +790,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
   
   pRes->numOfRows  = 1;
   pRes->numOfTotal = 0;
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   tscTrace("%p Valid SQL: %s pObj:%p", pSql, sql, pObj);
 
@@ -921,7 +921,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   SSqlRes *pRes = &pSql->res;
 
   pRes->numOfTotal = 0;  // the number of getting table meta from server
-  pRes->numOfTotalInCurrentClause = 0;
+  pRes->numOfClauseTotal = 0;
 
   pRes->code = 0;
 

src/client/src/tscStream.c

@@ -208,7 +208,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
     
     if (pStream->numOfRes == 0) {
-      if (pQueryInfo->interpoType == TSDB_INTERPO_SET_VALUE || pQueryInfo->interpoType == TSDB_INTERPO_NULL) {
+      if (pQueryInfo->fillType == TSDB_FILL_SET_VALUE || pQueryInfo->fillType == TSDB_FILL_NULL) {
         SSqlRes *pRes = &pSql->res;
 
         /* failed to retrieve any result in this retrieve */

src/client/src/tscSubquery.c

@@ -771,7 +771,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         }
     
         SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
-        pRes1->numOfTotalInCurrentClause += pRes1->numOfRows;
+        pRes1->numOfClauseTotal += pRes1->numOfRows;
       }
   
       // data has retrieved to client, build the join results
@@ -1390,7 +1390,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);
       
       // clear local saved number of results
-      trsupport->localBuffer->numOfElems = 0;
+      trsupport->localBuffer->num = 0;
       pthread_mutex_unlock(&trsupport->queryMutex);
       
       tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql,
@@ -1457,7 +1457,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
   
   // data in from current vnode is stored in cache and disk
-  uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
+  uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num;
     tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
         pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
         numOfRowsFromSubquery, idx);
@@ -1465,11 +1465,11 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
 
 #ifdef _DEBUG_VIEW
-  printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
+  printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->num);
     SSrcColumnInfo colInfo[256] = {0};
     tscGetSrcColumnInfo(colInfo, pQueryInfo);
-    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
+    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->num,
-                       trsupport->localBuffer->numOfElems, colInfo);
+                       trsupport->localBuffer->num, colInfo);
 #endif
   
   if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
@@ -1834,7 +1834,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
         pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex];
       }
       
-      pRes->numOfTotalInCurrentClause++;
+      pRes->numOfClauseTotal++;
       break;
     } else {  // continue retrieve data from vnode
       if (!tscHashRemainDataInSubqueryResultSet(pSql)) {
@@ -1879,9 +1879,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
 static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
   SSqlRes *pRes = &pSql->res;
   
-  if (pRes->tsrow[columnIndex] != NULL && isNull(pRes->tsrow[columnIndex], pField->type)) {
+  if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
-    pRes->tsrow[columnIndex] = NULL;
-  } else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
     // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
     if (pRes->buffer[columnIndex] == NULL) {
       pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
@@ -1893,7 +1891,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
     if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) {
       pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
     } else {
-      tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);
+      tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow[columnIndex]);
       pRes->tsrow[columnIndex] = NULL;
     }
   }

src/client/src/tscUtil.c

@@ -280,7 +280,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
     return;
   }
 
-  pQueryInfo->interpoType = TSDB_INTERPO_NONE;
+  pQueryInfo->fillType = TSDB_FILL_NONE;
   tfree(pQueryInfo->defaultVal);
 }
 
@@ -1779,7 +1779,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   
   tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond);
 
-  if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
     pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
     memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
   }
@@ -1989,7 +1989,7 @@ int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* s
 
 bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
   assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0);
-  return (pQueryInfo->clauseLimit > 0 && pRes->numOfTotalInCurrentClause >= pQueryInfo->clauseLimit);
+  return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
 }
 
 char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
@@ -2037,7 +2037,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
   int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
   while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
     tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
-             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfTotalInCurrentClause);
+             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
 
     /*
      * update the limit and offset value for the query on the next vnode,
@@ -2045,11 +2045,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
      *
      * NOTE:
      * if the pRes->offset is larger than 0, the start returned position has not reached yet.
-     * Therefore, the pRes->numOfRows, as well as pRes->numOfTotalInCurrentClause, must be 0.
+     * Therefore, the pRes->numOfRows, as well as pRes->numOfClauseTotal, must be 0.
      * The pRes->offset value will be updated by virtual node, during query execution.
      */
     if (pQueryInfo->clauseLimit >= 0) {
-      pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfTotalInCurrentClause;
+      pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfClauseTotal;
     }
 
     pQueryInfo->limit.offset = pRes->offset;
@@ -2092,7 +2092,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
   pSql->cmd.command = pQueryInfo->command;
 
   //backup the total number of result first
-  int64_t num = pRes->numOfTotal + pRes->numOfTotalInCurrentClause;
+  int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal;
   tscFreeSqlResult(pSql);
   
   pRes->numOfTotal = num;
@@ -2126,16 +2126,26 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column
     int32_t realLen = varDataLen(pData);
     assert(realLen <= bytes - VARSTR_HEADER_SIZE);
     
+    if (isNull(pData, type)) {
+      pRes->tsrow[columnIndex] = NULL;
+    } else {
+      pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
+    }
+  
     if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
       *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0;
     }
     
-    pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
     pRes->length[columnIndex] = realLen;
   } else {
     assert(bytes == tDataTypeDesc[type].nSize);
     
-    pRes->tsrow[columnIndex] = pData;
+    if (isNull(pData, type)) {
+      pRes->tsrow[columnIndex] = NULL;
+    } else {
+      pRes->tsrow[columnIndex] = pData;
+    }
+    
     pRes->length[columnIndex] = bytes;
   }
 }

src/common/src/ttypes.c

@@ -402,7 +402,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
         *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
       }
       break;
-    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_NCHAR: // todo : without length?
       for (int32_t i = 0; i < numOfElems; ++i) {
         *(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
       }

src/inc/taosdef.h

@@ -32,7 +32,7 @@ extern "C" {
 #define TSKEY int64_t
 #endif
 
-#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX};
+#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX})
 #define TSKEY_INITIAL_VAL    INT64_MIN
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR

src/inc/taosmsg.h

@@ -140,19 +140,19 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_MAX,
 };
 
-#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
+#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN    1
-#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
+#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN   2
 #define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3
-#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
+#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL    4
 
-#define TSDB_ALTER_TABLE_ADD_COLUMN 5
+#define TSDB_ALTER_TABLE_ADD_COLUMN        5
-#define TSDB_ALTER_TABLE_DROP_COLUMN 6
+#define TSDB_ALTER_TABLE_DROP_COLUMN       6
 
-#define TSDB_INTERPO_NONE 0
+#define TSDB_FILL_NONE      0
-#define TSDB_INTERPO_NULL 1
+#define TSDB_FILL_NULL      1
-#define TSDB_INTERPO_SET_VALUE 2
+#define TSDB_FILL_SET_VALUE 2
-#define TSDB_INTERPO_LINEAR 3
+#define TSDB_FILL_LINEAR    3
-#define TSDB_INTERPO_PREV 4
+#define TSDB_FILL_PREV      4
 
 #define TSDB_ALTER_USER_PASSWD 0x1
 #define TSDB_ALTER_USER_PRIVILEGES 0x2
@@ -164,8 +164,8 @@ enum _mgmt_table {
 #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS)
 
 #define TSDB_COL_NORMAL 0x0u
-#define TSDB_COL_TAG 0x1u
+#define TSDB_COL_TAG    0x1u
-#define TSDB_COL_JOIN 0x2u
+#define TSDB_COL_JOIN   0x2u
 
 extern char *taosMsg[];
 
@@ -440,7 +440,7 @@ typedef struct {
   uint16_t    queryType;        // denote another query process
   int16_t     numOfOutput;  // final output columns numbers
   int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
-  int16_t     interpoType;      // interpolate type
+  int16_t     fillType;      // interpolate type
   uint64_t    defaultVal;       // default value array list
   int32_t     tsOffset;       // offset value in current msg body, NOTE: ts list is compressed
   int32_t     tsLen;          // total length of ts comp block

src/inc/tsdb.h

@@ -167,12 +167,6 @@ typedef struct {
   SArray *pGroupList;
 } STableGroupInfo;
 
-typedef struct {
-} SFields;
-
-#define TSDB_TS_GREATER_EQUAL 1
-#define TSDB_TS_LESS_EQUAL 2
-
 typedef struct SQueryRowCond {
   int32_t rel;
   TSKEY   ts;

src/query/inc/qExecutor.h

@@ -18,20 +18,20 @@
 #include "os.h"
 
 #include "hash.h"
-#include "tsdb.h"
+#include "qfill.h"
-#include "qinterpolation.h"
 #include "qresultBuf.h"
 #include "qsqlparser.h"
 #include "qtsbuf.h"
 #include "taosdef.h"
-#include "tref.h"
-#include "tsqlfunction.h"
 #include "tarray.h"
+#include "tref.h"
+#include "tsdb.h"
+#include "tsqlfunction.h"
 
-typedef struct SData {
+//typedef struct tFilePage {
-  int32_t num;
+//  int64_t num;
-  char    data[];
+//  char    data[];
-} SData;
+//} tFilePage;
 
 struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
@@ -129,7 +129,7 @@ typedef struct SQuery {
   char              slidingTimeUnit;  // interval data type, used for daytime revise
   int8_t            precision;
   int16_t           numOfOutput;
-  int16_t           interpoType;
+  int16_t           fillType;
   int16_t           checkBuffer;  // check if the buffer is full during scan each block
   SLimitVal         limit;
   int32_t           rowSize;
@@ -139,11 +139,10 @@ typedef struct SQuery {
   SColumnInfo*      tagColList;
   int32_t           numOfFilterCols;
   int64_t*          defaultVal;
-//  TSKEY             lastKey;
   uint32_t          status;  // query status
   SResultRec        rec;
   int32_t           pos;
-  SData**           sdata;
+  tFilePage**       sdata;
   STableQueryInfo*  current;
   SSingleColumnFilterInfo* pFilterInfo;
 } SQuery;
@@ -151,12 +150,11 @@ typedef struct SQuery {
 typedef struct SQueryRuntimeEnv {
   SResultInfo*       resultInfo;  // todo refactor to merge with SWindowResInfo
   SQuery*            pQuery;
-  SData**            pInterpoBuf;
   SQLFunctionCtx*    pCtx;
   int16_t            numOfRowsPerPage;
   int16_t            offset[TSDB_MAX_COLUMNS];
   uint16_t           scanFlag;  // denotes reversed scan of data or not
-  SInterpolationInfo interpoInfo;
+  SFillInfo*         pFillInfo;
   SWindowResInfo     windowResInfo;
   STSBuf*            pTSBuf;
   STSCursor          cur;

src/query/inc/qextbuffer.h

@@ -68,7 +68,7 @@ typedef struct SExtFileInfo {
 } SExtFileInfo;
 
 typedef struct tFilePage {
-  uint64_t numOfElems;
+  uint64_t num;
   char     data[];
 } tFilePage;
 

src/query/inc/qfill.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_QFILL_H
+#define TDENGINE_QFILL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+#include "taosdef.h"
+#include "qextbuffer.h"
+
+typedef struct {
+  STColumn col;             // column info
+  int16_t  functionId;      // sql function id
+  int16_t  flag;            // column flag: TAG COLUMN|NORMAL COLUMN
+  union {int64_t i; double d;} defaultVal;
+} SFillColInfo;
+  
+typedef struct SFillInfo {
+  TSKEY   start;                // start timestamp
+  TSKEY   endKey;               // endKey for fill
+  int32_t order;                // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
+  int32_t fillType;             // fill type
+  int32_t numOfRows;            // number of rows in the input data block
+  int32_t rowIdx;               // rowIdx
+  int32_t numOfTotal;           // number of filled rows in one round
+  int32_t numOfCurrent;         // number of filled rows in current results
+
+  int32_t numOfTags;            // number of tags
+  int32_t numOfCols;            // number of columns, including the tags columns
+  int32_t rowSize;              // size of each row
+  char ** pTags;                // tags value for current interpolation
+  
+  int64_t slidingTime;         // sliding value to determine the number of result for a given time window
+  char *  prevValues;           // previous row of data, to generate the interpolation results
+  char *  nextValues;           // next row of data
+  SFillColInfo* pFillCol;       // column info for fill operations
+  char** pData;                 // original result data block involved in filling data
+} SFillInfo;
+
+typedef struct SPoint {
+  int64_t key;
+  void *  val;
+} SPoint;
+
+int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision);
+
+SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity,
+                            int32_t numOfCols, int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol);
+
+void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
+
+void taosDestoryFillInfo(SFillInfo *pFillInfo);
+
+void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
+
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput);
+
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
+
+TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision);
+
+int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows);
+
+int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
+
+int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData);
+  
+int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
+
+void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_QFILL_H

src/query/inc/qinterpolation.h

@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TINTERPOLATION_H
-#define TDENGINE_TINTERPOLATION_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "os.h"
-#include "taosdef.h"
-#include "qextbuffer.h"
-
-typedef struct SInterpolationInfo {
-  int64_t startTimestamp;
-  int32_t order;                // order [asc/desc]
-  int32_t numOfRawDataInRows;   // number of points in pQuery->sdata
-  int32_t rowIdx;               // rowIdx in pQuery->sdata
-  int32_t numOfTotalInterpo;    // number of interpolated rows in one round
-  int32_t numOfCurrentInterpo;  // number of interpolated rows in current results
-  char *  prevValues;           // previous row of data
-  char *  nextValues;           // next row of data
-  int32_t numOfTags;
-  char ** pTags;  // tags value for current interoplation
-} SInterpolationInfo;
-
-typedef struct SPoint {
-  int64_t key;
-  void *  val;
-} SPoint;
-
-int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision);
-
-void taosInitInterpoInfo(SInterpolationInfo *pInterpoInfo, int32_t order, int64_t startTimeStamp, int32_t numOfTags,
-                         int32_t rowSize);
-
-void taosDestoryInterpoInfo(SInterpolationInfo *pInterpoInfo);
-
-void taosInterpoSetStartInfo(SInterpolationInfo *pInterpoInfo, int32_t numOfRawDataInRows, int32_t type);
-
-TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision);
-
-/**
- *
- * @param pInterpoInfo
- * @param pPrimaryKeyArray
- * @param numOfRows
- * @param nInterval
- * @param ekey
- * @param maxNumOfRows
- * @return
- */
-int32_t taosGetNumOfResultWithInterpo(SInterpolationInfo *pInterpoInfo, int64_t *pPrimaryKeyArray, int32_t numOfRows,
-                                      int64_t nInterval, int64_t ekey, int32_t maxNumOfRows);
-
-int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo *pInterpoInfo, int64_t *pPrimaryKeyArray,
-                                    int32_t numOfRawDataInRows, int64_t nInterval, int64_t ekey);
-/**
- *
- * @param pInterpoInfo
- * @return
- */
-bool taosHasRemainsDataForInterpolation(SInterpolationInfo *pInterpoInfo);
-
-int32_t taosNumOfRemainPoints(SInterpolationInfo *pInterpoInfo);
-
-/**
- *
- */
-int32_t taosDoInterpoResult(SInterpolationInfo *pInterpoInfo, int16_t interpoType, tFilePage **data,
-                            int32_t numOfRawDataInRows, int32_t outputRows, int64_t nInterval,
-                            const int64_t *pPrimaryKeyArray, SColumnModel *pModel, char **srcData, int64_t *defaultVal,
-                            const int32_t *functionIDs, int32_t bufSize);
-
-int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TINTERPOLATION_H

src/query/inc/queryLog.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_RPC_LOG_H
+#ifndef TDENGINE_QUERY_LOG_H
-#define TDENGINE_RPC_LOG_H
+#define TDENGINE_QUERY_LOG_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,22 +24,23 @@ extern "C" {
 
 extern int32_t qDebugFlag;
 
-#define qTrace(...)                               \
+#define qTrace(...)                                \
-  if (qDebugFlag & DEBUG_TRACE) {                 \
+  if (qDebugFlag & DEBUG_TRACE) {                  \
-    taosPrintLog("DND QRY ", qDebugFlag, __VA_ARGS__); \
+    taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); \
   }
 
-#define qError(...)                                   \
+#define qError(...)                                      \
-  if (qDebugFlag & DEBUG_ERROR) {                   \
+  if (qDebugFlag & DEBUG_ERROR) {                        \
     taosPrintLog("ERROR QRY ", qDebugFlag, __VA_ARGS__); \
   }
-#define qWarn(...)                                    \
+
-  if (qDebugFlag & DEBUG_WARN) {                    \
+#define qWarn(...)                                      \
-    taosPrintLog("WARN  QRY ", qDebugFlag, __VA_ARGS__); \
+  if (qDebugFlag & DEBUG_WARN) {                        \
-  }  
+    taosPrintLog("WARN QRY ", qDebugFlag, __VA_ARGS__); \
+  }
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_RPC_CACHE_H
+#endif  // TDENGINE_QUERY_CACHE_H

src/query/inc/tlosertree.h

@@ -32,7 +32,7 @@ typedef struct SLoserTreeNode {
 typedef struct SLoserTreeInfo {
   int32_t              numOfEntries;
   int32_t              totalEntries;
-  __merge_compare_fn_t comparaFn;
+  __merge_compare_fn_t comparFn;
   void *               param;
 
   SLoserTreeNode *pNode;

src/query/src/qExecutor.c

@@ -12,22 +12,23 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include <qfill.h>
 #include "os.h"
 
 #include "hash.h"
 #include "hashfunc.h"
+#include "qExecutor.h"
+#include "qUtil.h"
 #include "qast.h"
 #include "qresultBuf.h"
 #include "query.h"
-#include "queryExecutor.h"
 #include "queryLog.h"
-#include "queryUtil.h"
 #include "taosmsg.h"
+#include "tdataformat.h"
 #include "tlosertree.h"
+#include "tscUtil.h"  // todo move the function to common module
 #include "tscompression.h"
 #include "ttime.h"
-#include "tscUtil.h"   // todo move the function to common module
-#include "tdataformat.h"
 
 #define DEFAULT_INTERN_BUF_SIZE 16384L
 
@@ -529,10 +530,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
     pageId = getLastPageId(&list);
     pData = getResultBufferPageById(pResultBuf, pageId);
 
-    if (pData->numOfElems >= numOfRowsPerPage) {
+    if (pData->num >= numOfRowsPerPage) {
       pData = getNewDataBuf(pResultBuf, sid, &pageId);
       if (pData != NULL) {
-        assert(pData->numOfElems == 0);  // number of elements must be 0 for new allocated buffer
+        assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
       }
     }
   }
@@ -544,7 +545,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   // set the number of rows in current disk page
   if (pWindowRes->pos.pageId == -1) {  // not allocated yet, allocate new buffer
     pWindowRes->pos.pageId = pageId;
-    pWindowRes->pos.rowId = pData->numOfElems++;
+    pWindowRes->pos.rowId = pData->num++;
   }
 
   return 0;
@@ -1202,9 +1203,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
       while (1) {
         getNextTimeWindow(pQuery, &nextWin);
-        if (pWindowResInfo->startTime > nextWin.skey ||
+        if (/*pWindowResInfo->startTime > nextWin.skey ||*/
             (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
-            (nextWin.skey > pQuery->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+            (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
           break;
         }
 
@@ -1334,7 +1335,6 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
     // last_dist or first_dist function
     // store the first&last timestamp into the intermediate buffer [1], the true
     // value may be null but timestamp will never be null
-    //    pCtx->ptsList = tsCol;
   } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA ||
              functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
     /*
@@ -1415,7 +1415,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
   pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
 
   if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) {
-    goto _error_clean;
+    goto _clean;
   }
 
   pRuntimeEnv->offset[0] = 0;
@@ -1427,7 +1427,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     
     int32_t index = pSqlFuncMsg->colInfo.colIndex;
     if (TSDB_COL_IS_TAG(pIndex->flag)) {
-      if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) {
+      if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) {  // todo refactor
         pCtx->inputBytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
         pCtx->inputType = TSDB_DATA_TYPE_BINARY;
       } else {
@@ -1489,7 +1489,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
   setCtxTagColumnInfo(pQuery, pRuntimeEnv->pCtx);
   return TSDB_CODE_SUCCESS;
 
-_error_clean:
+_clean:
   tfree(pRuntimeEnv->resultInfo);
   tfree(pRuntimeEnv->pCtx);
 
@@ -1524,15 +1524,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
     tfree(pRuntimeEnv->pCtx);
   }
 
-  taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo);
+  taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
-
-  if (pRuntimeEnv->pInterpoBuf != NULL) {
-    for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-      tfree(pRuntimeEnv->pInterpoBuf[i]);
-    }
-
-    tfree(pRuntimeEnv->pInterpoBuf);
-  }
 
   destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
   tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
@@ -1975,7 +1967,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
     // set the direct previous(next) point for process
     count = 2;
 
-    if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) {
+    if (pQuery->fillType == TSDB_FILL_SET_VALUE) {
       for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
         SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
 
@@ -2005,7 +1997,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
         }
 
         pInterpDetail->ts = pQuery->window.skey;
-        pInterpDetail->type = pQuery->interpoType;
+        pInterpDetail->type = pQuery->fillType;
       }
     } else {
       TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0];
@@ -2040,7 +2032,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
         tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT);
 
         pInterpDetail->ts = pQInfo->runtimeEnv.pQuery->window.skey;
-        pInterpDetail->type = pQuery->interpoType;
+        pInterpDetail->type = pQuery->fillType;
       }
     }
   }
@@ -2094,23 +2086,6 @@ void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) {
 #endif
 }
 
-static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) {
-#if 0
-  if (pQuery->interpoType != TSDB_INTERPO_NONE) {
-    assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery)));
-    
-    if (isIntervalQuery(pQuery)) {
-      pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutput);
-      
-      for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-        pQInfo->runtimeEnv.pInterpoBuf[i] =
-            calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].bytes * pMeterObj->pointsPerFileBlock);
-      }
-    }
-  }
-#endif
-}
-
 static int32_t getInitialPageNum(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   int32_t INITIAL_RESULT_ROWS_VALUE = 16;
@@ -2396,7 +2371,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle);
 
     // todo extract methods
-    if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) {
+    if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) {
       TSKEY           skey1, ekey1;
       STimeWindow     w = TSWINDOW_INITIALIZER;
       SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
@@ -2414,6 +2389,10 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
         pWindowResInfo->startTime = pQuery->window.skey;
         pWindowResInfo->prevSKey = w.skey;
       }
+      
+      if (pRuntimeEnv->pFillInfo != NULL) {
+        pRuntimeEnv->pFillInfo->start = w.skey;
+      }
     }
 
     // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
@@ -2427,10 +2406,10 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
         for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
           int32_t bytes = pQuery->pSelectExpr[i].bytes;
 
-          char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData));
+          char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage));
           if (tmp == NULL) {  // todo handle the oom
           } else {
-            pQuery->sdata[i] = (SData *)tmp;
+            pQuery->sdata[i] = (tFilePage *)tmp;
           }
 
           // set the pCtx output buffer position
@@ -2648,7 +2627,7 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t
   }
 }
 
-void UNUSED_FUNC displayInterResult(SData **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) {
+void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) {
   SQuery* pQuery = pRuntimeEnv->pQuery;
   int32_t numOfCols = pQuery->numOfOutput;
   printf("super table query intermediate result, total:%d\n", numOfRows);
@@ -2787,7 +2766,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   int32_t total = 0;
   for (int32_t i = 0; i < list.size; ++i) {
     tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[i]);
-    total += pData->numOfElems;
+    total += pData->num;
   }
 
   int32_t rows = total;
@@ -2800,11 +2779,11 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
       char *  pDest = pQuery->sdata[i]->data;
 
-      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems,
+      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num,
-             bytes * pData->numOfElems);
+             bytes * pData->num);
     }
 
-    offset += pData->numOfElems;
+    offset += pData->num;
   }
 
   assert(pQuery->rec.rows == 0);
@@ -2907,7 +2886,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
       if (ts == lastTimestamp) {  // merge with the last one
         doMerge(pRuntimeEnv, ts, pWindowRes, true);
       } else {  // copy data to disk buffer
-        if (buffer[0]->numOfElems == pQuery->rec.capacity) {
+        if (buffer[0]->num == pQuery->rec.capacity) {
           if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) {
             return -1;
           }
@@ -2916,7 +2895,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
         }
 
         doMerge(pRuntimeEnv, ts, pWindowRes, false);
-        buffer[0]->numOfElems += 1;
+        buffer[0]->num += 1;
       }
 
       lastTimestamp = ts;
@@ -2935,7 +2914,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     tLoserTreeAdjust(pTree, pos + pTree->numOfEntries);
   }
 
-  if (buffer[0]->numOfElems != 0) {  // there are data in buffer
+  if (buffer[0]->num != 0) {  // there are data in buffer
     if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) {
       qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo);
 
@@ -2993,10 +2972,10 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) {
     // pagewise copy to dest buffer
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
-      buf->numOfElems = r;
+      buf->num = r;
 
-      memcpy(buf->data + pRuntimeEnv->offset[i] * buf->numOfElems, ((char *)pQuery->sdata[i]->data) + offset * bytes,
+      memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes,
-             buf->numOfElems * bytes);
+             buf->num * bytes);
     }
 
     offset += r;
@@ -3276,16 +3255,19 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
   return toContinue;
 }
 
-static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv) {
+static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo* pTableQueryInfo = pQuery->current;
   
+  assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) ||
+      (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery)));
+  
   SQueryStatusInfo info = {
       .status      = pQuery->status,
       .windowIndex = pRuntimeEnv->windowResInfo.curIndex,
-      .lastKey     = pTableQueryInfo->lastKey,
+      .lastKey     = start,
       .w           = pQuery->window,
-      .curWindow   = {.skey = pTableQueryInfo->lastKey, .ekey = pTableQueryInfo->win.ekey},
+      .curWindow   = {.skey = start, .ekey = pTableQueryInfo->win.ekey},
   };
 
   return info;
@@ -3348,7 +3330,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
   pTableQueryInfo->win = pStatus->w;
 }
 
-void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
+void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
   SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv);
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo *pTableQueryInfo = pQuery->current;
@@ -3356,7 +3338,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
 
   // store the start query position
-  SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv);
+  SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start);
 
   SET_MASTER_SCAN_FLAG(pRuntimeEnv);
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@@ -3534,7 +3516,7 @@ void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIdx, T
 static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group
+  // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
     pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
@@ -3788,80 +3770,43 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo
   updateWindowResNumOfRes(pRuntimeEnv, pTableQueryInfo);
 }
 
-bool vnodeHasRemainResults(void *handle) {
+bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
-  SQInfo *pQInfo = (SQInfo *)handle;
+  SQuery *pQuery = pRuntimeEnv->pQuery;
-
+  SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo;
-  if (pQInfo == NULL || pQInfo->runtimeEnv.pQuery->interpoType == TSDB_INTERPO_NONE) {
+  
+  if (pQuery->fillType == TSDB_FILL_NONE) {
+    assert(pFillInfo == NULL);
     return false;
   }
 
-  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  SQuery *          pQuery = pRuntimeEnv->pQuery;
-
-  SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo;
   if (pQuery->limit.limit > 0 && pQuery->rec.rows >= pQuery->limit.limit) {
     return false;
   }
 
-  int32_t remain = taosNumOfRemainPoints(pInterpoInfo);
+  // There are results not returned to client, fill operation applied to the remain result set in the
+  // first place is required.
+  int32_t remain = taosNumOfRemainRows(pFillInfo);
   if (remain > 0) {
     return true;
-  } else {
-    if (pRuntimeEnv->pInterpoBuf == NULL) {
-      return false;
-    }
-
-    // query has completed
-    if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-      /*TSKEY ekey =*/taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime,
-                                           pQuery->slidingTimeUnit, pQuery->precision);
-      //      int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY
-      //      *)pRuntimeEnv->pInterpoBuf[0]->data,
-      //                                                         remain, pQuery->intervalTime, ekey,
-      //                                                         pQuery->pointsToRead);
-      //      return numOfTotal > 0;
-      assert(0);
-      return false;
-    }
-
-    return false;
-  }
-}
-
-static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows,
-                                             int32_t outputRows) {
-#if 0
-  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  SQuery *pQuery = &pRuntimeEnv->pQuery;
-  
-  assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE);
-  
-  // build support structure for performing interpolation
-  SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutput);
-  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes;
-    pSchema[i].type = pQuery->pSelectExpr[i].type;
   }
   
-//  SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutput, pQuery->pointsToRead);
+  /*
-  
+   * There are no results returned to client now.
-  char *  srcData[TSDB_MAX_COLUMNS] = {0};
+   * If query is not completed yet, the gaps between two results blocks need to be handled after next data block
-  int32_t functions[TSDB_MAX_COLUMNS] = {0};
+   * is retrieved from TSDB.
-  
+   *
-  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+   * NOTE: If the result set is not the first block, the gap in front of the result set will be filled. If the result
-    srcData[i] = pDataSrc[i]->data;
+   * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the
-    functions[i] = pQuery->pSelectExpr[i].base.functionId;
+   * first result row in the actual result set will fill nothing.
+   */
+  if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
+    TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
+                                         pQuery->slidingTimeUnit, pQuery->precision);
+    int32_t numOfTotal = taosGetNumOfResultWithFill(pFillInfo, remain, ekey, pQuery->rec.capacity);
+    return numOfTotal > 0;
   }
-  
+
-  assert(0);
+  return false;
-//  int32_t numOfRes = taosDoInterpoResult(&pRuntimeEnv->interpoInfo, pQuery->interpoType, data, numOfRows, outputRows,
-//                                         pQuery->intervalTime, (int64_t *)pDataSrc[0]->data, pModel, srcData,
-//                                         pQuery->defaultVal, functions, pRuntimeEnv->pTabObj->pointsPerFileBlock);
-  
-  destroyColumnModel(pModel);
-  free(pSchema);
-#endif
-  return 0;
 }
 
 static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) {
@@ -3887,37 +3832,30 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
 
   // all data returned, set query over
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-    if (pQInfo->runtimeEnv.stableQuery && isIntervalQuery(pQuery)) {
+    if (pQInfo->runtimeEnv.stableQuery) {
       if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) {
         setQueryStatus(pQuery, QUERY_OVER);
       }
     } else {
-      setQueryStatus(pQuery, QUERY_OVER);
+      if (!queryHasRemainResults(&pQInfo->runtimeEnv)) {
+        setQueryStatus(pQuery, QUERY_OVER);
+      }
     }
   }
 }
 
-int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows,
+int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t numOfRows, int32_t *numOfInterpo) {
-                                    int32_t *numOfInterpo) {
+  SQuery *pQuery = pRuntimeEnv->pQuery;
-//  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-//  SQuery *          pQuery = pRuntimeEnv->pQuery;
-#if 0
   while (1) {
-    numOfRows = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo);
+    taosGenerateDataBlock(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata, &pQuery->rec.rows, pQuery->rec.capacity);
-    
+    int32_t ret = pQuery->rec.rows;
-    TSKEY   ekey = taosGetRevisedEndKey(pQuery->window.skey, pQuery->order.order, pQuery->intervalTime,
-                                        pQuery->slidingTimeUnit, pQuery->precision);
-    int32_t numOfFinalRows = taosGetNumOfResultWithInterpo(&pRuntimeEnv->interpoInfo, (TSKEY *)pDataSrc[0]->data,
-                                                           numOfRows, pQuery->intervalTime, ekey, pQuery->pointsToRead);
-    
-    int32_t ret = resultInterpolate(pQInfo, pDst, pDataSrc, numOfRows, numOfFinalRows);
-    assert(ret == numOfFinalRows);
     
+    // todo apply limit output function
     /* reached the start position of according to offset value, return immediately */
     if (pQuery->limit.offset == 0) {
       return ret;
     }
-    
+
     if (pQuery->limit.offset < ret) {
       ret -= pQuery->limit.offset;
       // todo !!!!there exactly number of interpo is not valid.
@@ -3932,13 +3870,12 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage
       pQuery->limit.offset -= ret;
       ret = 0;
     }
-    
+
-    if (!vnodeHasRemainResults(pQInfo)) {
+    if (!queryHasRemainResults(pRuntimeEnv)) {
       return ret;
     }
   }
-#endif
+  
-
   return 0;
 }
 
@@ -4058,11 +3995,12 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
   }
 }
 
-static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
+static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
+  *start = pQuery->current->lastKey;
 
   // if queried with value filter, do NOT forward query start position
-  if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) {
+  if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) {
     return true;
   }
 
@@ -4082,13 +4020,14 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
   while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) {
     SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle);
 
-    if (QUERY_IS_ASC_QUERY(pQuery) && pWindowResInfo->prevSKey == 0) {
+    if (QUERY_IS_ASC_QUERY(pQuery)) {
-      getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1,
+      if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
-                              &w);
+        getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1,
-      pWindowResInfo->startTime = w.skey;
+                                &ekey1, &w);
-      pWindowResInfo->prevSKey = w.skey;
+        pWindowResInfo->startTime = w.skey;
+        pWindowResInfo->prevSKey = w.skey;
+      }
     } else {
-      // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
       getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1,
                               &w);
 
@@ -4112,7 +4051,8 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
       if (pQuery->limit.offset == 0) {
         if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
             (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
-          // load the data block
+          // load the data block and check data remaining in current data block
+          // TODO optimize performance
           SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
           SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
 
@@ -4123,24 +4063,38 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
 
           // set the abort info
           pQuery->pos = startPos;
-          pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
+          
+          // reset the query start timestamp
+          pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
+          pQuery->window.skey = pTableQueryInfo->win.skey;
+          *start = pTableQueryInfo->win.skey;
+          
           pWindowResInfo->prevSKey = tw.skey;
-
+          int32_t index = pRuntimeEnv->windowResInfo.curIndex;
+          
           int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
-
+          pRuntimeEnv->windowResInfo.curIndex = index;  // restore the window index
+          
           qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d",
                  GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes);
           return true;
-        } else {
+        } else { // do nothing
-          // do nothing,
+          *start = tw.skey;
+          pQuery->window.skey = tw.skey;
+          pWindowResInfo->prevSKey = tw.skey;
           return true;
         }
       }
 
-      // next time window starts from current data block
+      /*
+       * If the next time window still starts from current data block,
+       * load the primary timestamp column first, and then find the start position for the next queried time window.
+       * Note that only the primary timestamp column is required.
+       * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required
+       * time window resides in current data block.
+       */
       if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
           (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
-        // load the data block, note that only the primary timestamp column is required
         SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
         SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
 
@@ -4155,7 +4109,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
         pWindowResInfo->prevSKey = tw.skey;
         win = tw;
       } else {
-        break;  // offset is not 0, and next time window locates in the next block.
+        break;  // offset is not 0, and next time window begins or ends in the next block.
       }
     }
   }
@@ -4195,10 +4149,35 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
     cond.twindow = pItem->info->win;
   }
 
-  pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+  if (isFirstLastRowQuery(pQuery)) {
+    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+  } else {
+    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+  }
 }
 
 
+static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
+  int32_t numOfCols = pQuery->numOfOutput;
+  int32_t offset = 0;
+  
+  SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SExprInfo* pExprInfo = &pQuery->pSelectExpr[i];
+    
+    pFillCol[i].col.bytes  = pExprInfo->bytes;
+    pFillCol[i].col.type   = pExprInfo->type;
+    pFillCol[i].col.offset = offset;
+    pFillCol[i].flag       = TSDB_COL_NORMAL;    // always be ta normal column for table query
+    pFillCol[i].functionId = pExprInfo->base.functionId;
+    pFillCol[i].defaultVal.i = pQuery->defaultVal[i];
+    
+    offset += pExprInfo->bytes;
+  }
+  
+  return pFillCol;
+}
+
 int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
 
@@ -4290,16 +4269,12 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
 //  pointInterpSupporterSetData(pQInfo, &interpInfo);
 //  pointInterpSupporterDestroy(&interpInfo);
 
-//  int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit,
+  if (pQuery->fillType != TSDB_FILL_NONE) {
-//                                             pQuery->precision);
+    SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
-//  taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0);
+    pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput,
-  //  allocMemForInterpo(pQInfo, pQuery, pMeterObj);
+                                              pQuery->slidingTime, pQuery->fillType, pColInfo);
-
+  }
-//  if (!isPointInterpoQuery(pQuery)) {
+  
-    //    assert(pQuery->pos >= 0 && pQuery->slot >= 0);
-//  }
-
-  // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value
   return TSDB_CODE_SUCCESS;
 }
 
@@ -4451,41 +4426,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
   return true;
 }
 
-static UNUSED_FUNC int64_t doCheckTables(SQInfo *pQInfo, SArray* pTableList) {
-  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  SQuery *          pQuery = pRuntimeEnv->pQuery;
-
-  if (!multiTableMultioutputHelper(pQInfo, 0)) {
-    return 0;
-  }
-
-  SPointInterpoSupporter pointInterpSupporter = {0};
-  pointInterpSupporterInit(pQuery, &pointInterpSupporter);
-
-  /*
-   * here we set the value for before and after the specified time into the
-   * parameter for interpolation query
-   */
-  pointInterpSupporterSetData(pQInfo, &pointInterpSupporter);
-  pointInterpSupporterDestroy(&pointInterpSupporter);
-
-  scanAllDataBlocks(pRuntimeEnv);
-
-  // first/last_row query, do not invoke the finalize for super table query
-  finalizeQueryResult(pRuntimeEnv);
-
-  int64_t numOfRes = getNumOfResult(pRuntimeEnv);
-  assert(numOfRes == 1 || numOfRes == 0);
-
-  // accumulate the point interpolation result
-  if (numOfRes > 0) {
-    pQuery->rec.rows += numOfRes;
-    forwardCtxOutputBuf(pRuntimeEnv, numOfRes);
-  }
-
-  return numOfRes;
-}
-
 /**
  * super table query handler
  * 1. super table projection query, group-by on normal columns query, ts-comp query
@@ -4536,8 +4476,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(tx, 0), pQInfo->tsdb);
         
         // here we simply set the first table as current table
-        pRuntimeEnv->pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
+        pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
-        scanAllDataBlocks(pRuntimeEnv);
+        scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
         
         int64_t numOfRes = getNumOfResult(pRuntimeEnv);
         if (numOfRes > 0) {
@@ -4597,14 +4537,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       // TODO handle the limit offset problem
       if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) {
         //        skipBlocks(pRuntimeEnv);
-
         if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
           pQInfo->tableIndex++;
           continue;
         }
       }
 
-      scanAllDataBlocks(pRuntimeEnv);
+      scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
 
       pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
       skipResults(pRuntimeEnv);
@@ -4853,23 +4792,22 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   
   SQuery *pQuery = pRuntimeEnv->pQuery;
+  if (!isTopBottomQuery(pQuery) && pQuery->limit.offset > 0) {  // no need to execute, since the output will be ignore.
+    return;
+  }
+  
   pQuery->current = pTableInfo;  // set current query table info
   
-  scanAllDataBlocks(pRuntimeEnv);
+  scanAllDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
   finalizeQueryResult(pRuntimeEnv);
 
   if (isQueryKilled(pQInfo)) {
     return;
   }
 
-  // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously.
+  // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously.
   pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
 
-  // must be top/bottom query if offset > 0
-  if (pQuery->limit.offset > 0) {
-    assert(isTopBottomQuery(pQuery));
-  }
-
   skipResults(pRuntimeEnv);
   limitResults(pQInfo);
 }
@@ -4893,7 +4831,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   }
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv);
+    scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
     finalizeQueryResult(pRuntimeEnv);
 
     if (isQueryKilled(pQInfo)) {
@@ -4936,11 +4874,11 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   }
 }
 
-static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) {
+static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv);
+    scanAllDataBlocks(pRuntimeEnv, start);
 
     if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
       return;
@@ -4952,7 +4890,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) {
     // here we can ignore the records in case of no interpolation
     // todo handle offset, in case of top/bottom interval query
     if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 &&
-        pQuery->interpoType == TSDB_INTERPO_NONE) {
+        pQuery->fillType == TSDB_FILL_NONE) {
       // maxOutput <= 0, means current query does not generate any results
       int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo);
 
@@ -4971,19 +4909,21 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) {
 static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv);
 
-  int32_t numOfInterpo = 0;
   SQuery *pQuery = pRuntimeEnv->pQuery;
   pQuery->current = pTableInfo;
 
+  int32_t numOfInterpo = 0;
+  TSKEY newStartKey = TSKEY_INITIAL_VAL;
+  
   // skip blocks without load the actual data block from file if no filter condition present
-  skipTimeInterval(pRuntimeEnv);
+  skipTimeInterval(pRuntimeEnv, &newStartKey);
-  if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) {
+  if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) {
     setQueryStatus(pQuery, QUERY_COMPLETED);
     return;
   }
 
   while (1) {
-    tableIntervalProcessImpl(pRuntimeEnv);
+    tableIntervalProcessImpl(pRuntimeEnv, newStartKey);
 
     if (isIntervalQuery(pQuery)) {
       pQInfo->groupIndex = 0;  // always start from 0
@@ -4994,22 +4934,18 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
     }
 
     // the offset is handled at prepare stage if no interpolation involved
-    if (pQuery->interpoType == TSDB_INTERPO_NONE) {
+    if (pQuery->fillType == TSDB_FILL_NONE || pQuery->rec.rows == 0) {
       limitResults(pQInfo);
       break;
     } else {
-      taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType);
+      TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
-      SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf;
+                                        pQuery->slidingTimeUnit, pQuery->precision);
-
+      taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, ekey);
-      for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+      taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
-        memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].bytes);
-      }
-
       numOfInterpo = 0;
-      pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf,
+      pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, pQuery->rec.rows, &numOfInterpo);
-                                                     pQuery->rec.rows, &numOfInterpo);
 
-      qTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows);
+      qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
       if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
         limitResults(pQInfo);
         break;
@@ -5035,19 +4971,20 @@ static void tableQueryImpl(SQInfo *pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
-  if (vnodeHasRemainResults(pQInfo)) {
+  if (queryHasRemainResults(pRuntimeEnv)) {
     /*
      * There are remain results that are not returned due to result interpolation
      * So, we do keep in this procedure instead of launching retrieve procedure for next results.
      */
     int32_t numOfInterpo = 0;
-    int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo);
+    int32_t remain = taosNumOfRemainRows(pRuntimeEnv->pFillInfo);
-    pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata,
+    pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, remain, &numOfInterpo);
-                                                   (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo);
+  
-
+    qTrace("QInfo: %p fill results completed, final:%d", pQInfo, pQuery->rec.rows);
-    limitResults(pQInfo);
+    if (pQuery->rec.rows > 0) {
-
+      limitResults(pQInfo);
-    pQInfo->pointsInterpo += numOfInterpo;
+    }
+    
     qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total);
     return;
   }
@@ -5105,8 +5042,6 @@ static void tableQueryImpl(SQInfo *pQInfo) {
   if (isQueryKilled(pQInfo)) {
     qTrace("QInfo:%p query is killed", pQInfo);
   } else {// todo set the table uid and tid in log
-//    SArray* p = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
-//    SPair* pair = taosArrayGet(p, 0);
     qTrace("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows",
         pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
   }
@@ -5130,7 +5065,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
 
   // record the total elapsed time
   pQInfo->elapsedTime += (taosGetTimestampUs() - st);
-  //  taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType);
+  //  taosFillSetStartInfo(&pQInfo->runtimeEnv.pFillInfo, pQuery->size, pQInfo->query.fillType);
 
   if (pQuery->rec.rows == 0) {
     qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables,
@@ -5377,8 +5312,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
     pQueryMsg->orderType = htons(pQueryMsg->orderType);
   }
 
-  pQueryMsg->interpoType = htons(pQueryMsg->interpoType);
+  pQueryMsg->fillType = htons(pQueryMsg->fillType);
-  if (pQueryMsg->interpoType != TSDB_INTERPO_NONE) {
+  if (pQueryMsg->fillType != TSDB_FILL_NONE) {
     pQueryMsg->defaultVal = (uint64_t)(pMsg);
 
     int64_t *v = (int64_t *)pMsg;
@@ -5422,7 +5357,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
          "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64,
          pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
          pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
-         pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset);
+         pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset);
 
   return 0;
 }
@@ -5699,7 +5634,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   pQuery->intervalTime    = pQueryMsg->intervalTime;
   pQuery->slidingTime     = pQueryMsg->slidingTime;
   pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit;
-  pQuery->interpoType     = pQueryMsg->interpoType;
+  pQuery->fillType     = pQueryMsg->fillType;
   pQuery->numOfTags       = pQueryMsg->numOfTags;
 
   // todo do not allocate ??
@@ -5729,7 +5664,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   }
 
   // prepare the result buffer
-  pQuery->sdata = (SData **)calloc(pQuery->numOfOutput, POINTER_BYTES);
+  pQuery->sdata = (tFilePage **)calloc(pQuery->numOfOutput, POINTER_BYTES);
   if (pQuery->sdata == NULL) {
     goto _cleanup;
   }
@@ -5742,14 +5677,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
     assert(pExprs[col].interBytes >= pExprs[col].bytes);
 
     // allocate additional memory for interResults that are usually larger then final results
-    size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(SData);
+    size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage);
-    pQuery->sdata[col] = (SData *)calloc(1, size);
+    pQuery->sdata[col] = (tFilePage *)calloc(1, size);
     if (pQuery->sdata[col] == NULL) {
       goto _cleanup;
     }
   }
 
-  if (pQuery->interpoType != TSDB_INTERPO_NONE) {
+  if (pQuery->fillType != TSDB_FILL_NONE) {
     pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutput);
     if (pQuery->defaultVal == NULL) {
       goto _cleanup;
@@ -6319,10 +6254,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
             memcpy(dst, data, bytes);
           }
         }
-      
       }
     }
   
+    pQInfo->tableIndex = pQInfo->groupInfo.numOfTables;
     qTrace("QInfo:%p create tag values results completed, rows:%d", pQInfo, num);
   }
   

src/query/src/qFilterFunc.c

@@ -16,10 +16,10 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
+#include "qExecutor.h"
 #include "taosmsg.h"
-#include "tsqlfunction.h"
-#include "queryExecutor.h"
 #include "tcompare.h"
+#include "tsqlfunction.h"
 
 bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) {
   return (*(int8_t *)minval < pFilter->filterInfo.upperBndi);

src/query/src/qUtil.c

@@ -20,11 +20,11 @@
 #include "qextbuffer.h"
 #include "ttime.h"
 
-#include "qinterpolation.h"
+#include "qfill.h"
 #include "ttime.h"
 
-#include "queryExecutor.h"
+#include "qExecutor.h"
-#include "queryUtil.h"
+#include "qUtil.h"
 
 int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
                           int32_t threshold, int16_t type) {
@@ -37,7 +37,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
   pWindowResInfo->hashList = taosHashInit(threshold, fn, false);
   
   pWindowResInfo->curIndex = -1;
-  pWindowResInfo->size = 0;
+  pWindowResInfo->size     = 0;
+  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
   
   // use the pointer arraylist
   pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
@@ -96,8 +97,8 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
   _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type);
   pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false);
   
-  pWindowResInfo->startTime = 0;
+  pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
-  pWindowResInfo->prevSKey = 0;
+  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
 
 void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {

src/query/src/qast.c

@@ -476,44 +476,6 @@ typedef struct {
   SEndPoint* end;
 } SQueryCond;
 
-//static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t type) {
-//  q->lowerBndRelOptr = TSDB_RELATION_GREATER;
-//  q->upperBndRelOptr = TSDB_RELATION_LESS;
-//
-//  switch (type) {
-//    case TSDB_DATA_TYPE_BOOL:
-//    case TSDB_DATA_TYPE_TINYINT:
-//    case TSDB_DATA_TYPE_SMALLINT:
-//    case TSDB_DATA_TYPE_INT:
-//    case TSDB_DATA_TYPE_BIGINT: {
-//      q->upperBnd.nType = TSDB_DATA_TYPE_BIGINT;
-//      q->lowerBnd.nType = TSDB_DATA_TYPE_BIGINT;
-//
-//      q->upperBnd.i64Key = INT64_MAX;
-//      q->lowerBnd.i64Key = INT64_MIN;
-//      break;
-//    };
-//    case TSDB_DATA_TYPE_FLOAT:
-//    case TSDB_DATA_TYPE_DOUBLE: {
-//      q->upperBnd.nType = TSDB_DATA_TYPE_DOUBLE;
-//      q->lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE;
-//      q->upperBnd.dKey = DBL_MAX;
-//      q->lowerBnd.dKey = -DBL_MIN;
-//      break;
-//    };
-//    case TSDB_DATA_TYPE_NCHAR:
-//    case TSDB_DATA_TYPE_BINARY: {
-//      q->upperBnd.nType = type;
-//      q->upperBnd.pz = NULL;
-//      q->upperBnd.nLen = -1;
-//
-//      q->lowerBnd.nType = type;
-//      q->lowerBnd.pz = NULL;
-//      q->lowerBnd.nLen = -1;
-//    }
-//  }
-//}
-
 // todo check for malloc failure
 static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
   int32_t optr = queryColInfo->optr;
@@ -788,7 +750,6 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTravers
   taosArrayCopy(pResult, array);
 }
 
-
 static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
 
@@ -834,8 +795,6 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
   tSkipListDestroyIter(iter);
 }
 
-
-
 // post-root order traverse syntax tree
 void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) {
   if (pExpr == NULL) {
@@ -1100,7 +1059,6 @@ static char* exception_strdup(const char* str) {
   return p;
 }
 
-
 static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
   int32_t anchor = CLEANUP_GET_ANCHOR();
 

src/query/src/qextbuffer.c

@@ -136,7 +136,7 @@ static bool tExtMemBufferAlloc(tExtMemBuffer *pMemBuffer) {
   }
 
   item->pNext = NULL;
-  item->item.numOfElems = 0;
+  item->item.num = 0;
 
   if (pMemBuffer->pTail != NULL) {
     pMemBuffer->pTail->pNext = item;
@@ -167,13 +167,13 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
     pLast = pMemBuffer->pTail;
   }
 
-  if (pLast->item.numOfElems + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records
+  if (pLast->item.num + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records
     tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRows, numOfRows);
     
     pMemBuffer->numOfElemsInBuffer += numOfRows;
     pMemBuffer->numOfTotalElems += numOfRows;
   } else {
-    int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - pLast->item.numOfElems;
+    int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - pLast->item.num;
     tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRemainEntries, numOfRows);
 
     pMemBuffer->numOfElemsInBuffer += numOfRemainEntries;
@@ -271,7 +271,7 @@ bool tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
       ret = false;
     }
 
-    pMemBuffer->fileMeta.numOfElemsInFile += first->item.numOfElems;
+    pMemBuffer->fileMeta.numOfElemsInFile += first->item.num;
     pMemBuffer->fileMeta.nFileSize += 1;
 
     tFilePagesItem *ptmp = first;
@@ -985,16 +985,16 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in
 
 ////////////////////////////////////////////////////////////////////////////////////////////
 void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity) {
-  if (inputBuffer->numOfElems == 0 || maxElemsCapacity == inputBuffer->numOfElems) {
+  if (inputBuffer->num == 0 || maxElemsCapacity == inputBuffer->num) {
     return;
   }
 
   /* start from the second column */
   for (int32_t i = 1; i < pModel->numOfCols; ++i) {
     SSchemaEx* pSchemaEx = &pModel->pFields[i];
-    memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->numOfElems,
+    memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->num,
             inputBuffer->data + pSchemaEx->offset * maxElemsCapacity,
-            pSchemaEx->field.bytes * inputBuffer->numOfElems);
+            pSchemaEx->field.bytes * inputBuffer->num);
   }
 }
 
@@ -1009,13 +1009,13 @@ int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index) {
 }
 
 void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockCapacity, int32_t s, int32_t e) {
-  if (inputBuffer->numOfElems == 0 || (e - s + 1) <= 0) {
+  if (inputBuffer->num == 0 || (e - s + 1) <= 0) {
     return;
   }
 
   int32_t removed = e - s + 1;
-  int32_t remain = inputBuffer->numOfElems - removed;
+  int32_t remain = inputBuffer->num - removed;
-  int32_t secPart = inputBuffer->numOfElems - e - 1;
+  int32_t secPart = inputBuffer->num - e - 1;
 
   /* start from the second column */
   for (int32_t i = 0; i < pModel->numOfCols; ++i) {
@@ -1028,7 +1028,7 @@ void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockC
     memmove(startPos, endPos, pSchema->bytes * secPart);
   }
 
-  inputBuffer->numOfElems = remain;
+  inputBuffer->num = remain;
 }
 
 /*
@@ -1040,16 +1040,16 @@ void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockC
  */
 void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t start, int32_t numOfRows,
                      int32_t srcCapacity) {
-  assert(dstPage->numOfElems + numOfRows <= dstModel->capacity);
+  assert(dstPage->num + numOfRows <= dstModel->capacity);
 
   for (int32_t col = 0; col < dstModel->numOfCols; ++col) {
-    char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->numOfElems, col);
+    char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->num, col);
     char *src = COLMODEL_GET_VAL((char *)srcData, dstModel, srcCapacity, start, col);
 
     memmove(dst, src, dstModel->pFields[col].field.bytes * numOfRows);
   }
 
-  dstPage->numOfElems += numOfRows;
+  dstPage->num += numOfRows;
 }
 
 tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel,

src/query/src/qfill.c

@@ -0,0 +1,470 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qfill.h"
+#include "os.h"
+#include "qextbuffer.h"
+#include "taosdef.h"
+#include "taosmsg.h"
+#include "tsqlfunction.h"
+
+#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
+
+int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, char timeUnit, int16_t precision) {
+  if (slidingTime == 0) {
+    return startTime;
+  }
+
+  if (timeUnit == 'a' || timeUnit == 'm' || timeUnit == 's' || timeUnit == 'h') {
+    return (startTime / slidingTime) * slidingTime;
+  } else {
+    /*
+     * here we revised the start time of day according to the local time zone,
+     * but in case of DST, the start time of one day need to be dynamically decided.
+     *
+     * TODO dynamically decide the start time of a day
+     */
+
+    // todo refactor to extract function that is available for Linux/Windows/Mac platform
+#if defined(WINDOWS) && _MSC_VER >= 1900
+    // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
+    int64_t timezone = _timezone;
+    int32_t daylight = _daylight;
+    char**  tzname = _tzname;
+#endif
+
+    int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
+
+    int64_t revStartime = (startTime / slidingTime) * slidingTime + timezone * t;
+    int64_t revEndtime = revStartime + slidingTime - 1;
+    if (revEndtime < startTime) {
+      revStartime += slidingTime;
+    }
+
+    return revStartime;
+  }
+}
+
+SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+    int64_t slidingTime, int32_t fillType, SFillColInfo* pFillCol) {
+  if (fillType == TSDB_FILL_NONE) {
+    return NULL;
+  }
+  
+  SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
+  
+  taosResetFillInfo(pFillInfo, skey);
+  
+  pFillInfo->order     = order;
+  pFillInfo->fillType  = fillType;
+  pFillInfo->pFillCol  = pFillCol;
+  pFillInfo->numOfTags = numOfTags;
+  pFillInfo->numOfCols = numOfCols;
+  pFillInfo->slidingTime = slidingTime;
+  
+  pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
+  
+  int32_t rowsize = 0;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    int32_t bytes = pFillInfo->pFillCol[i].col.bytes;
+    pFillInfo->pData[i] = calloc(1, sizeof(tFilePage) + bytes * capacity);
+    
+    rowsize += bytes;
+  }
+  
+  if (numOfTags > 0) {
+    pFillInfo->pTags = calloc(1, pFillInfo->numOfTags * POINTER_BYTES + rowsize);
+  }
+  
+  pFillInfo->rowSize = rowsize;
+  return pFillInfo;
+}
+
+void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
+  pFillInfo->start        = startTimestamp;
+  pFillInfo->rowIdx       = -1;
+  pFillInfo->numOfRows    = 0;
+  pFillInfo->numOfCurrent = 0;
+  pFillInfo->numOfTotal   = 0;
+}
+
+void taosDestoryFillInfo(SFillInfo* pFillInfo) {
+  if (pFillInfo == NULL) {
+    return;
+  }
+
+  tfree(pFillInfo->prevValues);
+  tfree(pFillInfo->nextValues);
+  tfree(pFillInfo->pTags);
+  tfree(pFillInfo);
+}
+
+void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
+  if (pFillInfo->fillType == TSDB_FILL_NONE) {
+    return;
+  }
+
+  pFillInfo->rowIdx = 0;
+  pFillInfo->numOfRows = numOfRows;
+  
+  pFillInfo->endKey = endKey;
+}
+
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
+  // copy the data into source data buffer
+  for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
+  }
+}
+
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
+  assert(pFillInfo->numOfRows == pInput->num);
+  for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+    
+    char* s = pInput->data + pCol->col.offset * pInput->num;
+    memcpy(pFillInfo->pData[i], s, pInput->num * pCol->col.bytes);
+    
+    if (pCol->flag == TSDB_COL_TAG) {  // copy the tag value
+      memcpy(pFillInfo->pTags[i], pFillInfo->pData[i], pCol->col.bytes);
+    }
+  }
+}
+
+TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
+  if (order == TSDB_ORDER_ASC) {
+    return ekey;
+  } else {
+    return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision);
+  }
+}
+
+static int32_t taosGetTotalNumOfFilledRes(SFillInfo* pFillInfo, const TSKEY* tsArray, int32_t remain,
+    int64_t nInterval, int64_t ekey) {
+  
+  if (remain > 0) {  // still fill gap within current data block, not generating data after the result set.
+    TSKEY lastKey = tsArray[pFillInfo->numOfRows - 1];
+    int32_t total = (int32_t)(labs(lastKey - pFillInfo->start) / nInterval) + 1;
+
+    assert(total >= remain);
+    return total;
+  } else { // reach the end of data
+    if ((ekey < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
+        (ekey > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
+      return 0;
+    } else {
+      return (int32_t)(labs(ekey - pFillInfo->start) / nInterval) + 1;
+    }
+  }
+}
+
+int32_t taosGetNumOfResultWithFill(SFillInfo* pFillInfo, int32_t numOfRows, int64_t ekey, int32_t maxNumOfRows) {
+  int32_t numOfRes = taosGetTotalNumOfFilledRes(pFillInfo, (int64_t*) pFillInfo->pData[0], numOfRows,
+                                                pFillInfo->slidingTime, ekey);
+  return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
+}
+
+int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
+  if (pFillInfo->rowIdx == -1 || pFillInfo->numOfRows == 0) {
+    return 0;
+  }
+
+  return FILL_IS_ASC_FILL(pFillInfo) ? (pFillInfo->numOfRows - pFillInfo->rowIdx)
+                                                : pFillInfo->rowIdx + 1;
+}
+
+// todo: refactor
+static double linearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
+  return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
+}
+
+int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
+  switch (type) {
+    case TSDB_DATA_TYPE_INT: {
+      *(int32_t*)point->val = linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
+                                                        point2->key, point->key);
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      *(float*)point->val =
+          linearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, point1->key, point2->key, point->key);
+      break;
+    };
+    case TSDB_DATA_TYPE_DOUBLE: {
+      *(double*)point->val =
+          linearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, point1->key, point2->key, point->key);
+      break;
+    };
+    case TSDB_DATA_TYPE_TIMESTAMP:
+    case TSDB_DATA_TYPE_BIGINT: {
+      *(int64_t*)point->val = linearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
+                                                        point2->key, point->key);
+      break;
+    };
+    case TSDB_DATA_TYPE_SMALLINT: {
+      *(int16_t*)point->val = linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
+                                                        point2->key, point->key);
+      break;
+    };
+    case TSDB_DATA_TYPE_TINYINT: {
+      *(int8_t*)point->val =
+          linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key);
+      break;
+    };
+    default: {
+      // TODO: Deal with interpolation with bool and strings and timestamp
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static void setTagsValue(SFillInfo* pColInfo, tFilePage** data, char** pTags, int32_t start, int32_t num) {
+  for (int32_t j = 0, i = start; i < pColInfo->numOfCols + pColInfo->numOfTags; ++i, ++j) {
+    SFillColInfo* pCol = &pColInfo->pFillCol[i];
+    
+    char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
+    assignVal(val1, pTags[j], pCol->col.bytes, pCol->col.type);
+  }
+}
+
+static void doInterpoResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData,
+                                int64_t ts, char** pTags, bool outOfBound) {
+  char** prevValues = &pFillInfo->prevValues;
+  char** nextValues = &pFillInfo->nextValues;
+
+  SPoint point1, point2, point;
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
+
+  char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, *num);
+  *(TSKEY*) val = pFillInfo->start;
+
+  int32_t numOfValCols = pFillInfo->numOfCols - pFillInfo->numOfTags;
+
+  // set the other values
+  if (pFillInfo->fillType == TSDB_FILL_PREV) {
+    char* pInterpolationData = FILL_IS_ASC_FILL(pFillInfo) ? *prevValues : *nextValues;
+    if (pInterpolationData != NULL) {
+      for (int32_t i = 1; i < numOfValCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        
+        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+        if (isNull(pInterpolationData + pCol->col.offset, pCol->col.type)) {
+          setNull(val1, pCol->col.type, pCol->col.bytes);
+        } else {
+          assignVal(val1, pInterpolationData + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+        }
+      }
+    } else { // no prev value yet, set the value for NULL
+      for (int32_t i = 1; i < numOfValCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+
+        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+        setNull(val1, pCol->col.type, pCol->col.bytes);
+      }
+    }
+
+    setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
+  } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
+    // TODO : linear interpolation supports NULL value
+    if (*prevValues != NULL && !outOfBound) {
+      for (int32_t i = 1; i < numOfValCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        
+        int16_t type  = pCol->col.type;
+        int16_t bytes = pCol->col.bytes;
+        
+        char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+        if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
+          setNull(val1, pCol->col.type, bytes);
+          continue;
+        }
+
+        point1 = (SPoint){.key = *(TSKEY*)(*prevValues), .val = *prevValues + pCol->col.offset};
+        point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
+        point = (SPoint){.key = pFillInfo->start, .val = val1};
+        taosDoLinearInterpolation(type, &point1, &point2, &point);
+      }
+
+      setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
+
+    } else {
+      for (int32_t i = 1; i < numOfValCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+  
+        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+        setNull(val1, pCol->col.type, pCol->col.bytes);
+      }
+
+      setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
+  
+    }
+  } else { /* default value interpolation */
+    for (int32_t i = 1; i < numOfValCols; ++i) {
+      SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+      
+      char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+      assignVal(val1, (char*)&pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
+    }
+
+    setTagsValue(pFillInfo, data, pTags, numOfValCols, *num);
+  }
+
+  pFillInfo->start += (pFillInfo->slidingTime * step);
+  pFillInfo->numOfCurrent++;
+
+  (*num) += 1;
+}
+
+static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
+  if (*nextValues != NULL) {
+    return;
+  }
+  
+  *nextValues = calloc(1, pFillInfo->rowSize);
+  for (int i = 1; i < pFillInfo->numOfCols; i++) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+    setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+  }
+}
+
+int32_t taosDoInterpoResult(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
+  int32_t num = 0;
+  pFillInfo->numOfCurrent = 0;
+
+  char** prevValues = &pFillInfo->prevValues;
+  char** nextValues = &pFillInfo->nextValues;
+
+  int32_t numOfTags = pFillInfo->numOfTags;
+  char**  pTags = pFillInfo->pTags;
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
+
+  if (numOfRows == 0) {
+    /*
+     * we need to rebuild whole result set
+     * NOTE:we need to keep the last saved data, to generated the filled data
+     */
+    while (num < outputRows) {
+      doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, true);
+    }
+    
+    pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+    return outputRows;
+
+  } else {
+    while (1) {
+      int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->rowIdx];
+
+      if ((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+          (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
+        /* set the next value for interpolation */
+        initBeforeAfterDataBuf(pFillInfo, nextValues);
+        
+        int32_t offset = pFillInfo->rowIdx;
+        for (int32_t i = 0; i < pFillInfo->numOfCols - numOfTags; ++i) {
+          SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+          memcpy(*nextValues + pCol->col.offset, srcData[i] + offset * pCol->col.bytes, pCol->col.bytes);
+        }
+      }
+
+      if (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+           (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
+        
+        while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+                (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
+          doInterpoResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, pTags, false);
+        }
+
+        /* output buffer is full, abort */
+        if ((num == outputRows && FILL_IS_ASC_FILL(pFillInfo)) ||
+            (num < 0 && !FILL_IS_ASC_FILL(pFillInfo))) {
+          pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+          return outputRows;
+        }
+      } else {
+        assert(pFillInfo->start == ts);
+        initBeforeAfterDataBuf(pFillInfo, prevValues);
+        
+        // assign rows to dst buffer
+        int32_t i = 0;
+        for (; i < pFillInfo->numOfCols - numOfTags; ++i) {
+          SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+          
+          char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
+          char* src  = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->rowIdx);
+          
+          if (i == 0 ||
+              (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
+              (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
+            assignVal(val1, src, pCol->col.bytes, pCol->col.type);
+            memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
+          } else {  // i > 0 and data is null , do interpolation
+            if (pFillInfo->fillType == TSDB_FILL_PREV) {
+              assignVal(val1, *prevValues + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+            } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
+              assignVal(val1, src, pCol->col.bytes, pCol->col.type);
+              memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
+            } else {
+              assignVal(val1, (char*) &pCol->defaultVal.i, pCol->col.bytes, pCol->col.type);
+            }
+          }
+        }
+
+        // set the tag value for final result
+        setTagsValue(pFillInfo, data, pTags, pFillInfo->numOfCols - numOfTags, num);
+
+        pFillInfo->start += (pFillInfo->slidingTime * step);
+        pFillInfo->rowIdx += 1;
+        num += 1;
+      }
+
+      if ((pFillInfo->rowIdx >= pFillInfo->numOfRows && FILL_IS_ASC_FILL(pFillInfo)) ||
+          (pFillInfo->rowIdx < 0 && !FILL_IS_ASC_FILL(pFillInfo)) || num >= outputRows) {
+        if (pFillInfo->rowIdx >= pFillInfo->numOfRows || pFillInfo->rowIdx < 0) {
+          pFillInfo->rowIdx = -1;
+          pFillInfo->numOfRows = 0;
+
+          /* the raw data block is exhausted, next value does not exists */
+          tfree(*nextValues);
+        }
+
+        pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+        return num;
+      }
+    }
+  }
+}
+
+void taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int64_t* outputRows, int32_t capacity) {
+    int32_t remain = taosNumOfRemainRows(pFillInfo);  // todo use iterator?
+    
+//    TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->slidingTime,
+//                                      pQuery->slidingTimeUnit, pQuery->precision);
+//    if (QUERY_IS_ASC_QUERY(pQuery)) {
+//      assert(ekey >= pQuery->window.ekey);
+//    } else {
+//      assert(ekey <= pQuery->window.ekey);
+//    }
+    
+    int32_t rows = taosGetNumOfResultWithFill(pFillInfo, remain, pFillInfo->endKey, capacity);
+  
+    int32_t numOfRes = taosDoInterpoResult(pFillInfo, output, remain, rows, pFillInfo->pData);
+    *outputRows = rows;
+    
+    assert(numOfRes == rows);
+}

src/query/src/qinterpolation.c

@@ -1,429 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "qinterpolation.h"
-#include "os.h"
-#include "qextbuffer.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tsqlfunction.h"
-
-#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSDB_ORDER_ASC)
-
-int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision) {
-  if (timeRange == 0) {
-    return startTime;
-  }
-
-  if (intervalTimeUnit == 'a' || intervalTimeUnit == 'm' || intervalTimeUnit == 's' || intervalTimeUnit == 'h') {
-    return (startTime / timeRange) * timeRange;
-  } else {
-    /*
-     * here we revised the start time of day according to the local time zone,
-     * but in case of DST, the start time of one day need to be dynamically decided.
-     *
-     * TODO dynamically decide the start time of a day
-     */
-
-#if defined(WINDOWS) && _MSC_VER >= 1900
-    // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
-    int64_t timezone = _timezone;
-    int32_t daylight = _daylight;
-    char**  tzname = _tzname;
-#endif
-
-    int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
-
-    int64_t revStartime = (startTime / timeRange) * timeRange + timezone * t;
-    int64_t revEndtime = revStartime + timeRange - 1;
-    if (revEndtime < startTime) {
-      revStartime += timeRange;
-    }
-
-    return revStartime;
-  }
-}
-
-void taosInitInterpoInfo(SInterpolationInfo* pInterpoInfo, int32_t order, int64_t startTimestamp,
-                         int32_t numOfGroupbyTags, int32_t rowSize) {
-  pInterpoInfo->startTimestamp = startTimestamp;
-  pInterpoInfo->rowIdx = -1;
-  pInterpoInfo->numOfRawDataInRows = 0;
-  pInterpoInfo->numOfCurrentInterpo = 0;
-  pInterpoInfo->numOfTotalInterpo = 0;
-  pInterpoInfo->order = order;
-
-  pInterpoInfo->numOfTags = numOfGroupbyTags;
-  if (pInterpoInfo->pTags == NULL && numOfGroupbyTags > 0) {
-    pInterpoInfo->pTags = calloc(1, numOfGroupbyTags * POINTER_BYTES + rowSize);
-  }
-
-  // set the previous value to be null
-  tfree(pInterpoInfo->prevValues);
-}
-
-// the SInterpolationInfo itself will not be released
-void taosDestoryInterpoInfo(SInterpolationInfo* pInterpoInfo) {
-  if (pInterpoInfo == NULL) {
-    return;
-  }
-
-  tfree(pInterpoInfo->prevValues);
-  tfree(pInterpoInfo->nextValues);
-
-  tfree(pInterpoInfo->pTags);
-}
-
-void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawDataInRows, int32_t type) {
-  if (type == TSDB_INTERPO_NONE) {
-    return;
-  }
-
-  pInterpoInfo->rowIdx = 0;
-  pInterpoInfo->numOfRawDataInRows = numOfRawDataInRows;
-}
-
-TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision) {
-  if (order == TSDB_ORDER_ASC) {
-    return ekey;
-  } else {
-    return taosGetIntervalStartTimestamp(ekey, timeInterval, intervalTimeUnit, precision);
-  }
-}
-
-int32_t taosGetNumOfResultWithInterpo(SInterpolationInfo* pInterpoInfo, TSKEY* pPrimaryKeyArray,
-                                      int32_t numOfRawDataInRows, int64_t nInterval, int64_t ekey,
-                                      int32_t maxNumOfRows) {
-  int32_t numOfRes = taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeyArray, numOfRawDataInRows, nInterval, ekey);
-  return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
-}
-
-int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo* pInterpoInfo, int64_t* pPrimaryKeyArray,
-                                    int32_t numOfAvailRawData, int64_t nInterval, int64_t ekey) {
-  if (numOfAvailRawData > 0) {
-    int32_t finalNumOfResult = 0;
-
-    // get last timestamp, calculate the result size
-    int64_t lastKey = pPrimaryKeyArray[pInterpoInfo->numOfRawDataInRows - 1];
-    finalNumOfResult = (int32_t)(labs(lastKey - pInterpoInfo->startTimestamp) / nInterval) + 1;
-
-    assert(finalNumOfResult >= numOfAvailRawData);
-    return finalNumOfResult;
-  } else {
-    /* reach the end of data */
-    if ((ekey < pInterpoInfo->startTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-        (ekey > pInterpoInfo->startTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
-      return 0;
-    } else {
-      return (int32_t)(labs(ekey - pInterpoInfo->startTimestamp) / nInterval) + 1;
-    }
-  }
-}
-
-bool taosHasRemainsDataForInterpolation(SInterpolationInfo* pInterpoInfo) {
-  return taosNumOfRemainPoints(pInterpoInfo) > 0;
-}
-
-int32_t taosNumOfRemainPoints(SInterpolationInfo* pInterpoInfo) {
-  if (pInterpoInfo->rowIdx == -1 || pInterpoInfo->numOfRawDataInRows == 0) {
-    return 0;
-  }
-
-  return INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? (pInterpoInfo->numOfRawDataInRows - pInterpoInfo->rowIdx)
-                                                : pInterpoInfo->rowIdx + 1;
-}
-
-static double doLinearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
-  return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
-}
-
-int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
-  switch (type) {
-    case TSDB_DATA_TYPE_INT: {
-      *(int32_t*)point->val = doLinearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key,
-                                                        point2->key, point->key);
-      break;
-    }
-    case TSDB_DATA_TYPE_FLOAT: {
-      *(float*)point->val =
-          doLinearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, point1->key, point2->key, point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_DOUBLE: {
-      *(double*)point->val =
-          doLinearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, point1->key, point2->key, point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_BIGINT: {
-      *(int64_t*)point->val = doLinearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key,
-                                                        point2->key, point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_SMALLINT: {
-      *(int16_t*)point->val = doLinearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key,
-                                                        point2->key, point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_TINYINT: {
-      *(int8_t*)point->val =
-          doLinearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key);
-      break;
-    };
-    default: {
-      // TODO: Deal with interpolation with bool and strings and timestamp
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static char* getPos(char* data, int32_t bytes, int32_t index) { return data + index * bytes; }
-
-static void setTagsValueInInterpolation(tFilePage** data, char** pTags, SColumnModel* pModel, int32_t order,
-                                        int32_t start, int32_t capacity, int32_t num) {
-  for (int32_t j = 0, i = start; i < pModel->numOfCols; ++i, ++j) {
-    SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-    char* val1 = getPos(data[i]->data, pSchema->bytes, num);
-    assignVal(val1, pTags[j], pSchema->bytes, pSchema->type);
-  }
-}
-
-static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data,
-                                SColumnModel* pModel, int32_t* num, char** srcData, int64_t nInterval,
-                                int64_t* defaultVal, int64_t currentTimestamp, int32_t capacity, int32_t numOfTags,
-                                char** pTags, bool outOfBound) {
-  char** prevValues = &pInterpoInfo->prevValues;
-  char** nextValues = &pInterpoInfo->nextValues;
-
-  SPoint point1, point2, point;
-
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order);
-
-  char* val = getPos(data[0]->data, TSDB_KEYSIZE, *num);
-  *(TSKEY*)val = pInterpoInfo->startTimestamp;
-
-  int32_t numOfValCols = pModel->numOfCols - numOfTags;
-
-  // set the other values
-  if (interpoType == TSDB_INTERPO_PREV) {
-    char* pInterpolationData = INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? *prevValues : *nextValues;
-    if (pInterpolationData != NULL) {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SSchema* pSchema = getColumnModelSchema(pModel, i);
-        int16_t  offset = getColumnModelOffset(pModel, i);
-
-        char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
-
-        if (isNull(pInterpolationData + offset, pSchema->type)) {
-          setNull(val1, pSchema->type, pSchema->bytes);
-        } else {
-          assignVal(val1, pInterpolationData + offset, pSchema->bytes, pSchema->type);
-        }
-      }
-    } else { /* no prev value yet, set the value for null */
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-        char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
-        setNull(val1, pSchema->type, pSchema->bytes);
-      }
-    }
-
-    setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
-  } else if (interpoType == TSDB_INTERPO_LINEAR) {
-    // TODO : linear interpolation supports NULL value
-    if (*prevValues != NULL && !outOfBound) {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SSchema* pSchema = getColumnModelSchema(pModel, i);
-        int16_t  offset = getColumnModelOffset(pModel, i);
-
-        int16_t type = pSchema->type;
-        char*   val1 = getPos(data[i]->data, pSchema->bytes, *num);
-
-        if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
-          setNull(val1, type, pSchema->bytes);
-          continue;
-        }
-
-        point1 = (SPoint){.key = *(TSKEY*)(*prevValues), .val = *prevValues + offset};
-        point2 = (SPoint){.key = currentTimestamp, .val = srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes};
-        point = (SPoint){.key = pInterpoInfo->startTimestamp, .val = val1};
-        taosDoLinearInterpolation(type, &point1, &point2, &point);
-      }
-
-      setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
-
-    } else {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-        char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
-        setNull(val1, pSchema->type, pSchema->bytes);
-      }
-
-      setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
-    }
-  } else { /* default value interpolation */
-    for (int32_t i = 1; i < numOfValCols; ++i) {
-      SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-      char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
-      assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type);
-    }
-
-    setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, numOfValCols, capacity, *num);
-  }
-
-  pInterpoInfo->startTimestamp += (nInterval * step);
-  pInterpoInfo->numOfCurrentInterpo++;
-
-  (*num) += 1;
-}
-
-static void initBeforeAfterDataBuf(SColumnModel* pModel, char** nextValues) {
-  if (*nextValues != NULL) {
-    return;
-  }
-  
-  *nextValues = calloc(1, pModel->rowSize);
-  for (int i = 1; i < pModel->numOfCols; i++) {
-    int16_t  offset = getColumnModelOffset(pModel, i);
-    SSchema* pSchema = getColumnModelSchema(pModel, i);
-    
-    setNull(*nextValues + offset, pSchema->type, pSchema->bytes);
-  }
-}
-
-int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data,
-                            int32_t numOfRawDataInRows, int32_t outputRows, int64_t nInterval,
-                            const int64_t* pPrimaryKeyArray, SColumnModel* pModel, char** srcData, int64_t* defaultVal,
-                            const int32_t* functionIDs, int32_t bufSize) {
-  int32_t num = 0;
-  pInterpoInfo->numOfCurrentInterpo = 0;
-
-  char** prevValues = &pInterpoInfo->prevValues;
-  char** nextValues = &pInterpoInfo->nextValues;
-
-  int32_t numOfTags = pInterpoInfo->numOfTags;
-  char**  pTags = pInterpoInfo->pTags;
-
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order);
-
-  if (numOfRawDataInRows == 0) {
-    /*
-     * we need to rebuild whole data
-     * NOTE:we need to keep the last saved data, to satisfy the interpolation
-     */
-    while (num < outputRows) {
-      doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal,
-                          pInterpoInfo->startTimestamp, bufSize, numOfTags, pTags, true);
-    }
-    pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
-    return outputRows;
-
-  } else {
-    while (1) {
-      int64_t currentTimestamp = pPrimaryKeyArray[pInterpoInfo->rowIdx];
-
-      if ((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-          (pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
-        /* set the next value for interpolation */
-        initBeforeAfterDataBuf(pModel, nextValues);
-        
-        int32_t offset = pInterpoInfo->rowIdx;
-        for (int32_t tlen = 0, i = 0; i < pModel->numOfCols - numOfTags; ++i) {
-          SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-          memcpy(*nextValues + tlen, srcData[i] + offset * pSchema->bytes, pSchema->bytes);
-          tlen += pSchema->bytes;
-        }
-      }
-
-      if (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-           (pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
-          num < outputRows) {
-        while (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-                (pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
-               num < outputRows) {
-          doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal,
-                              currentTimestamp, bufSize, numOfTags, pTags, false);
-        }
-
-        /* output buffer is full, abort */
-        if ((num == outputRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-            (num < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
-          pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
-          return outputRows;
-        }
-      } else {
-        assert(pInterpoInfo->startTimestamp == currentTimestamp);
-        
-        initBeforeAfterDataBuf(pModel, prevValues);
-        
-        // assign rows to dst buffer
-        int32_t i = 0;
-        for (int32_t tlen = 0; i < pModel->numOfCols - numOfTags; ++i) {
-          int16_t  offset = getColumnModelOffset(pModel, i);
-          SSchema* pSchema = getColumnModelSchema(pModel, i);
-
-          char* val1 = getPos(data[i]->data, pSchema->bytes, num);
-          char* src = srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes;
-          
-          if (i == 0 ||
-              (functionIDs[i] != TSDB_FUNC_COUNT && !isNull(src, pSchema->type)) ||
-              (functionIDs[i] == TSDB_FUNC_COUNT && *(int64_t*)(src) != 0)) {
-            assignVal(val1, src, pSchema->bytes, pSchema->type);
-            memcpy(*prevValues + tlen, src, pSchema->bytes);
-          } else {  // i > 0 and data is null , do interpolation
-            if (interpoType == TSDB_INTERPO_PREV) {
-              assignVal(val1, *prevValues + offset, pSchema->bytes, pSchema->type);
-            } else if (interpoType == TSDB_INTERPO_LINEAR) {
-              assignVal(val1, src, pSchema->bytes, pSchema->type);
-              memcpy(*prevValues + tlen, src, pSchema->bytes);
-            } else {
-              assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type);
-            }
-          }
-          tlen += pSchema->bytes;
-        }
-
-        /* set the tag value for final result */
-        setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, pModel->numOfCols - numOfTags, bufSize,
-                                    num);
-
-        pInterpoInfo->startTimestamp += (nInterval * step);
-        pInterpoInfo->rowIdx += 1;
-        num += 1;
-      }
-
-      if ((pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
-          (pInterpoInfo->rowIdx < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || num >= outputRows) {
-        if (pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows || pInterpoInfo->rowIdx < 0) {
-          pInterpoInfo->rowIdx = -1;
-          pInterpoInfo->numOfRawDataInRows = 0;
-
-          /* the raw data block is exhausted, next value does not exists */
-          tfree(*nextValues);
-        }
-
-        pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
-        return num;
-      }
-    }
-  }
-}

src/query/src/qpercentile.c

@@ -64,26 +64,26 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx,
       for (uint32_t j = 0; j < pFlushInfo->numOfPages; ++j) {
         ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
         UNUSED(ret);
-        assert(pPage->numOfElems > 0);
+        assert(pPage->num > 0);
         
-        tColModelAppend(pDesc->pColumnModel, buffer, pPage->data, 0, pPage->numOfElems, pPage->numOfElems);
+        tColModelAppend(pDesc->pColumnModel, buffer, pPage->data, 0, pPage->num, pPage->num);
-        printf("id: %d  count: %" PRIu64 "\n", j, buffer->numOfElems);
+        printf("id: %d  count: %" PRIu64 "\n", j, buffer->num);
       }
     }
     tfree(pPage);
     
-    assert(buffer->numOfElems == pMemBuffer->fileMeta.numOfElemsInFile);
+    assert(buffer->num == pMemBuffer->fileMeta.numOfElemsInFile);
   }
   
   // load data in pMemBuffer to buffer
   tFilePagesItem *pListItem = pMemBuffer->pHead;
   while (pListItem != NULL) {
-    tColModelAppend(pDesc->pColumnModel, buffer, pListItem->item.data, 0, pListItem->item.numOfElems,
+    tColModelAppend(pDesc->pColumnModel, buffer, pListItem->item.data, 0, pListItem->item.num,
-                    pListItem->item.numOfElems);
+                    pListItem->item.num);
     pListItem = pListItem->pNext;
   }
   
-  tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSDB_ORDER_ASC);
+  tColDataQSort(pDesc, buffer->num, 0, buffer->num - 1, buffer->data, TSDB_ORDER_ASC);
   
   pDesc->pColumnModel->capacity = oldCapacity;  // restore value
   return buffer;
@@ -881,7 +881,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
           for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) {
             ret = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
             UNUSED(ret);
-            tMemBucketPut(pMemBucket, pPage->data, pPage->numOfElems);
+            tMemBucketPut(pMemBucket, pPage->data, pPage->num);
           }
 
           fclose(pMemBuffer->file);

src/query/src/tlosertree.c

@@ -54,7 +54,7 @@ uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* pa
   (*pTree)->numOfEntries = numOfEntries;
   (*pTree)->totalEntries = totalEntries;
   (*pTree)->param = param;
-  (*pTree)->comparaFn = compareFn;
+  (*pTree)->comparFn = compareFn;
 
   // set initial value for loser tree
   tLoserTreeInit(*pTree);
@@ -95,7 +95,7 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
       return;
     }
 
-    int32_t ret = pTree->comparaFn(&pTree->pNode[parentId], &kLeaf, pTree->param);
+    int32_t ret = pTree->comparFn(&pTree->pNode[parentId], &kLeaf, pTree->param);
     if (ret < 0) {
       SLoserTreeNode t = pTree->pNode[parentId];
       pTree->pNode[parentId] = kLeaf;

src/tsdb/src/tsdbMeta.c

@@ -572,9 +572,20 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
   
   char* key = tdGetRowDataOfCol(pTable->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
-  bool ret = tSkipListRemove(pSTable->pIndex, key);
+  SArray* res = tSkipListGet(pSTable->pIndex, key);
+  
+  size_t size = taosArrayGetSize(res);
+  assert(size > 0);
+  
+  for(int32_t i = 0; i < size; ++i) {
+    SSkipListNode* pNode = taosArrayGetP(res, i);
+    
+    STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
+    if (pElem->pTable == pTable) {  // this is the exact what we need
+      tSkipListRemoveNode(pSTable->pIndex, pNode);
+    }
+  }
   
-  assert(ret);
   return 0;
 }
 

src/tsdb/src/tsdbRead.c

@@ -49,6 +49,10 @@ typedef struct SQueryFilePos {
   int32_t slot;
   int32_t pos;
   int64_t lastKey;
+  int32_t rows;
+  bool    mixBlock;
+  bool    blockCompleted;
+  STimeWindow win;
 } SQueryFilePos;
 
 typedef struct SDataBlockLoadInfo {
@@ -61,7 +65,6 @@ typedef struct SDataBlockLoadInfo {
 typedef struct SLoadCompBlockInfo {
   int32_t tid; /* table tid */
   int32_t fileId;
-  int32_t fileListIndex;
 } SLoadCompBlockInfo;
 
 typedef struct STableCheckInfo {
@@ -71,10 +74,13 @@ typedef struct STableCheckInfo {
   int32_t    start;
   SCompInfo* pCompInfo;
   int32_t    compSize;
-  int32_t    numOfBlocks;  // number of qualified data blocks not the original blocks
+  int32_t    numOfBlocks;    // number of qualified data blocks not the original blocks
   SDataCols* pDataCols;
   
-  SSkipListIterator* iter;
+  SSkipListIterator* iter;   // skip list iterator
+  SSkipListIterator* iiter;  // imem iterator
+  
+  bool       hasObtainBuf;   // if we should initialize the in-memory skip list iterator
 } STableCheckInfo;
 
 typedef struct {
@@ -110,6 +116,7 @@ typedef struct STsdbQueryHandle {
   SField**    pFields;
   SArray*     pColumns;    // column list, SColumnInfoData array list
   bool        locateStart;
+  int32_t     outputCapacity;
   int32_t     realNumOfRows;
   SArray*     pTableCheckInfo;  //SArray<STableCheckInfo>
   int32_t     activeIndex;
@@ -134,7 +141,6 @@ static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
 static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
   pCompBlockLoadInfo->tid = -1;
   pCompBlockLoadInfo->fileId = -1;
-  pCompBlockLoadInfo->fileListIndex = -1;
 }
 
 TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
@@ -149,6 +155,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
   tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
 
   pQueryHandle->cur.fid = -1;
+  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
 
   size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
   assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
@@ -181,20 +188,20 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
    * For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place
    * in case of descending timestamp order query.
    */
-  pQueryHandle->checkFiles  = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order);
+  pQueryHandle->checkFiles  = true;//ASCENDING_ORDER_TRAVERSE(pQueryHandle->order);
   pQueryHandle->activeIndex = 0;
 
   // allocate buffer in order to load data blocks from file
   int32_t numOfCols = pCond->numOfCols;
-  size_t  bufferCapacity = 4096;
+  pQueryHandle->outputCapacity = 4096;
-
+  
   pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    SColumnInfoData  pDest = {{0}, 0};
+    SColumnInfoData  colInfo = {{0}, 0};
-
+  
-    pDest.info = pCond->colList[i];
+    colInfo.info = pCond->colList[i];
-    pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].bytes);
+    colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes);
-    taosArrayPush(pQueryHandle->pColumns, &pDest);
+    taosArrayPush(pQueryHandle->pColumns, &colInfo);
   }
 
   tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
@@ -223,6 +230,72 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond*
   return pQueryHandle;
 }
 
+static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) {
+  STable* pTable = pCheckInfo->pTableObj;
+  assert(pTable != NULL);
+  
+  if (pCheckInfo->hasObtainBuf) {
+    return true;
+  }
+  
+  pCheckInfo->hasObtainBuf = true;
+  int32_t order = pHandle->order;
+  
+  // no data in buffer, abort
+  if (pTable->mem == NULL && pTable->imem == NULL) {
+    return false;
+  }
+  
+  assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
+  
+  if (pTable->mem) {
+    pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey,
+                                                  TSDB_DATA_TYPE_TIMESTAMP, order);
+  }
+  
+  if (pTable->imem) {
+    pCheckInfo->iiter = tSkipListCreateIterFromVal(pTable->imem->pData, (const char*) &pCheckInfo->lastKey,
+                                                   TSDB_DATA_TYPE_TIMESTAMP, order);
+  }
+  
+  // both iterators are NULL, no data in buffer right now
+  if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) {
+    return false;
+  }
+  
+  bool memEmpty  = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
+  bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
+  if (memEmpty && imemEmpty) { // buffer is empty
+    return false;
+  }
+  
+  if (!memEmpty) {
+    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
+    assert(node != NULL);
+  
+    SDataRow row = SL_GET_NODE_DATA(node);
+    TSKEY key = dataRowKey(row);  // first timestamp in buffer
+    uTrace("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle,
+           pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo);
+  } else {
+    uTrace("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid);
+  }
+  
+  if (!imemEmpty) {
+    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
+    assert(node != NULL);
+  
+    SDataRow row = SL_GET_NODE_DATA(node);
+    TSKEY key = dataRowKey(row);  // first timestamp in buffer
+    uTrace("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle,
+           pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo);
+  } else {
+    uTrace("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid);
+  }
+  
+  return true;
+}
+
 static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
@@ -245,12 +318,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
     if (pCheckInfo->iter == NULL) {
       return false;
     }
+  
+    if (!tSkipListIterNext(pCheckInfo->iter)) {  // buffer is empty
+      return false;
+    }
   }
   
-  if (!tSkipListIterNext(pCheckInfo->iter)) {  // buffer is empty
-    return false;
-  }
-
   SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
   if (node == NULL) {
     return false;
@@ -270,9 +343,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   return true;
 }
 
-// todo dynamic get the daysperfile
+static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile) {
-static int32_t getFileIdFromKey(TSKEY key) {
+  int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[0]));  // set the starting fileId
-  int64_t fid = (int64_t)(key / (10 * tsMsPerDay[0]));  // set the starting fileId
   if (fid > INT32_MAX) {
     fid = INT32_MAX;
   }
@@ -409,7 +481,7 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
   return pLocalIdList;
 }
 
-static void    filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
+static void    mergeDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
                                      SArray* sa);
 static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
 
@@ -456,22 +528,63 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
         return false;
       }
   
-      SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
-      assert(pCols->numOfPoints == pBlock->numOfPoints);
+      assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfPoints == pBlock->numOfPoints);
       
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
         cur->pos =
-            binarySearchForKey(pCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order);
+            binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = 0;
       }
-
+      
-      filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
+      mergeDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
     } else {  // the whole block is loaded in to buffer
-      pQueryHandle->realNumOfRows = pBlock->numOfPoints;
+      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
+      /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
+
+      TSKEY k1 = TSKEY_INITIAL_VAL, k2 = TSKEY_INITIAL_VAL;
+      if (pCheckInfo->iter != NULL) {
+        SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
+        SDataRow       row = SL_GET_NODE_DATA(node);
+        k1 = dataRowKey(row);
+
+        if (k1 == binfo.window.skey) {
+          if (tSkipListIterNext(pCheckInfo->iter)) {
+            node = tSkipListIterGet(pCheckInfo->iter);
+            row = SL_GET_NODE_DATA(node);
+            k1 = dataRowKey(row);
+          } else {
+            k1 = TSKEY_INITIAL_VAL;
+          }
+        }
+      }
+
+      if (pCheckInfo->iiter != NULL) {
+        SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
+        SDataRow       row = SL_GET_NODE_DATA(node);
+        k2 = dataRowKey(row);
+
+        if (k2 == binfo.window.skey) {
+          if (tSkipListIterNext(pCheckInfo->iiter)) {
+            node = tSkipListIterGet(pCheckInfo->iiter);
+            row = SL_GET_NODE_DATA(node);
+            k2 = dataRowKey(row);
+          } else {
+            k2 = TSKEY_INITIAL_VAL;
+          }
+        }
+      }
+
       cur->pos = 0;
+      if ((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.ekey)) {
+        doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
+        mergeDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
+      } else {
+        pQueryHandle->realNumOfRows = binfo.rows;
+      }
     }
-  } else {
+  } else {  //desc order
     // query ended in current block
     if (pQueryHandle->window.ekey > pBlock->keyFirst) {
       if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
@@ -486,12 +599,55 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
         cur->pos = pBlock->numOfPoints - 1;
       }
       
-      filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
+      mergeDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
     } else {
-      pQueryHandle->realNumOfRows = pBlock->numOfPoints;
+      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
-      cur->pos = pBlock->numOfPoints - 1;
+      /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
+  
+      TSKEY k1 = TSKEY_INITIAL_VAL, k2 = TSKEY_INITIAL_VAL;
+      if (pCheckInfo->iter != NULL) {
+        SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
+        SDataRow       row = SL_GET_NODE_DATA(node);
+        k1 = dataRowKey(row);
+    
+        if (k1 == binfo.window.skey) {
+          if (tSkipListIterNext(pCheckInfo->iter)) {
+            node = tSkipListIterGet(pCheckInfo->iter);
+            row = SL_GET_NODE_DATA(node);
+            k1 = dataRowKey(row);
+          } else {
+            k1 = TSKEY_INITIAL_VAL;
+          }
+        }
+      }
+  
+      if (pCheckInfo->iiter != NULL) {
+        SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
+        SDataRow       row = SL_GET_NODE_DATA(node);
+        k2 = dataRowKey(row);
+    
+        if (k2 == binfo.window.skey) {
+          if (tSkipListIterNext(pCheckInfo->iiter)) {
+            node = tSkipListIterGet(pCheckInfo->iiter);
+            row = SL_GET_NODE_DATA(node);
+            k2 = dataRowKey(row);
+          } else {
+            k2 = TSKEY_INITIAL_VAL;
+          }
+        }
+      }
+  
+      cur->pos = binfo.rows - 1;
+      if ((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.ekey)) {
+        doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
+        mergeDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa);
+      } else {
+        pQueryHandle->realNumOfRows = binfo.rows;
+      }
+    }
+//      pQueryHandle->realNumOfRows = pBlock->numOfPoints;
+//      cur->pos = pBlock->numOfPoints - 1;
     }
-  }
 
   taosArrayDestroy(sa);
   return pQueryHandle->realNumOfRows > 0;
@@ -559,72 +715,39 @@ static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
   return midPos;
 }
 
-// only return the qualified data to client in terms of query time window, data rows in the same block but do not
+static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
-// be included in the query time window will be discarded
+  char* pData = NULL;
-static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
+  int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 1 : -1;
-                                  SArray* sa) {
-  SQueryFilePos* cur = &pQueryHandle->cur;
-  SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
   
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  TSKEY* tsArray = pCols->cols[0].pData;
   
-  int32_t endPos = cur->pos;
+  int32_t num = end - start + 1;
-  if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
+  int32_t reqiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-    endPos = blockInfo.rows - 1;
+  
-    pQueryHandle->realNumOfRows = endPos - cur->pos + 1;
+  //data in buffer has greater timestamp, copy data in file block
-    pCheckInfo->lastKey = blockInfo.window.ekey + 1;
+  for (int32_t i = 0; i < reqiredNumOfCols; ++i) {
-  } else if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) {
+    SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-    endPos = 0;
+    int32_t          bytes = pColInfo->info.bytes;
-    pQueryHandle->realNumOfRows = cur->pos + 1;
+    
-    pCheckInfo->lastKey = blockInfo.window.ekey - 1;
-  } else {
-    int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
-    endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order);
-
     if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
-      if (endPos < cur->pos) {
+      pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
-        pQueryHandle->realNumOfRows = 0;
-        return;
-      } else {
-        pQueryHandle->realNumOfRows = endPos - cur->pos + 1;
-      }
-
-      pCheckInfo->lastKey = ((int64_t*)(pCols->cols[0].pData))[endPos] + 1;
     } else {
-      if (endPos > cur->pos) {
+      pData = pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
-        pQueryHandle->realNumOfRows = 0;
-        return;
-      } else {
-        pQueryHandle->realNumOfRows = cur->pos - endPos + 1;
-      }
     }
-  }
+    
-
+    for (int32_t j = 0; j < pCols->numOfCols; ++j) {  // todo opt performance
-  int32_t start = MIN(cur->pos, endPos);
+      SDataCol* src = &pCols->cols[j];
-  
+      
-//  if (start > 0) {
+      if (pColInfo->info.colId == src->colId) {
-//    tdPopDataColsPoints(pQueryHandle->rhelper.pDataCols[0], start);
+        
-//  }
+        if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
-
+          memmove(pData, src->pData + bytes * start, bytes * num);
-  // move the data block in the front to data block if needed
+        } else {  // handle the var-string
-  int32_t numOfCols = pQueryHandle->rhelper.pDataCols[0]->numOfCols;
+          char* dst = pData;
-  int32_t reqCols = taosArrayGetSize(pQueryHandle->pColumns);
+          
-  
-  for (int32_t i = 0; i < reqCols; ++i) {
-    SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
-    int32_t bytes = pCol->info.bytes;
-
-    for (int32_t j = 0; j < numOfCols; ++j) {  //todo opt performance
-      SDataCol* src = &pQueryHandle->rhelper.pDataCols[0]->cols[j];
-
-      if (pCol->info.colId == src->colId) {
-        if (pCol->info.type != TSDB_DATA_TYPE_BINARY && pCol->info.type != TSDB_DATA_TYPE_NCHAR) {
-          memmove(pCol->pData, src->pData + bytes * start, bytes * pQueryHandle->realNumOfRows);
-        } else { // handle the var-string
-          char* dst = pCol->pData;
-  
           // todo refactor, only copy one-by-one
-          for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) {
+          for (int32_t k = start; k < num + start; ++k) {
             char* p = tdGetColDataOfRow(src, k);
             memcpy(dst, p, varDataTLen(p));
             dst += bytes;
@@ -635,11 +758,268 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
       }
     }
   }
+  
+  pQueryHandle->cur.win.ekey = tsArray[end];
+  pQueryHandle->cur.lastKey = tsArray[end] + step;
+  
+  return numOfRows + num;
+}
 
-  assert(pQueryHandle->realNumOfRows <= blockInfo.rows);
+static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t capacity,
+    int32_t numOfRows, SDataRow row, STSchema* pSchema) {
+  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+  int32_t numOfTableCols = schemaNCols(pSchema);
+  
+  char* pData = NULL;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+    
+    if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+      pData = pColInfo->pData + numOfRows * pColInfo->info.bytes;
+    } else {
+      pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
+    }
+    
+    int32_t offset = 0;
+    for (int32_t j = 0; j < numOfTableCols; ++j) {
+      if (pColInfo->info.colId == pSchema->columns[j].colId) {
+        offset = pSchema->columns[j].offset;
+        break;
+      }
+    }
+    
+    assert(offset != -1);  // todo handle error
+    void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
+    
+    if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+      memcpy(pData, value, varDataTLen(value));
+    } else {
+      memcpy(pData, value, pColInfo->info.bytes);
+    }
+  }
+}
 
-  // forward(backward) the position for cursor
+// only return the qualified data to client in terms of query time window, data rows in the same block but do not
-  cur->pos = endPos;
+// be included in the query time window will be discarded
+static void mergeDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
+                                  SArray* sa) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+  SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
+  
+  initTableMemIterator(pQueryHandle, pCheckInfo);
+  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  
+  int32_t endPos = cur->pos;
+  if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
+    endPos = blockInfo.rows - 1;
+    cur->mixBlock = (cur->pos != 0);
+  } else if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) {
+    endPos = 0;
+    cur->mixBlock = (cur->pos != blockInfo.rows - 1);
+  } else {
+    int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
+    endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order);
+    cur->mixBlock = true;
+  }
+  
+  // compared with the data from in-memory buffer, to generate the correct timestamp array list
+  int32_t pos = cur->pos;
+  
+  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == 0);
+  TSKEY* tsArray = pCols->cols[0].pData;
+  
+  int32_t numOfRows = 0;
+  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
+  int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 1:-1;
+  
+  // no data in buffer, load data from file directly
+  if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
+    int32_t start = cur->pos;
+    int32_t end = endPos;
+    if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+      end = cur->pos;
+      start = endPos;
+    }
+    
+    cur->win.skey = tsArray[start];
+    cur->win.ekey = tsArray[end];
+    
+    // todo opt in case of no data in buffer
+    numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
+    
+      // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+    if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
+      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
+      int32_t reqNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+  
+      for(int32_t i = 0; i < reqNumOfCols; ++i) {
+        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+      }
+    }
+  
+    cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+        ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)));
+    
+    pCheckInfo->lastKey = cur->lastKey;
+    pQueryHandle->realNumOfRows = numOfRows;
+    cur->rows = numOfRows;
+    return;
+  } else if (pCheckInfo->iter != NULL && pCheckInfo->iiter == NULL) {
+    //  } else if (pCheckInfo->iter == NULL && pCheckInfo->iiter != NULL) {
+    //  } else { // iter and iiter are all not NULL, three-way merge data block
+    STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj);
+    SSkipListNode* node = NULL;
+    
+    do {
+      node = tSkipListIterGet(pCheckInfo->iter);
+      if (node == NULL) {
+        break;
+      }
+
+      SDataRow row = SL_GET_NODE_DATA(node);
+      TSKEY    key = dataRowKey(row);
+      if ((key > pQueryHandle->window.ekey && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+          (key < pQueryHandle->window.ekey && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
+        break;
+      }
+
+      if (((tsArray[pos] > pQueryHandle->window.ekey || pos > endPos) && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+          ((tsArray[pos] < pQueryHandle->window.ekey || pos < endPos) && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
+        break;
+      }
+
+      if ((key < tsArray[pos] && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+          (key > tsArray[pos] && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
+        copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
+        numOfRows += 1;
+        if (cur->win.skey == TSKEY_INITIAL_VAL) {
+          cur->win.skey = key;
+        }
+
+        cur->win.ekey = key;
+        cur->lastKey  = key + step;
+        cur->mixBlock = true;
+
+        tSkipListIterNext(pCheckInfo->iter);
+      } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
+        tSkipListIterNext(pCheckInfo->iter);
+      } else if ((key > tsArray[pos] && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+                  (key < tsArray[pos] && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
+        if (cur->win.skey == TSKEY_INITIAL_VAL) {
+          cur->win.skey = tsArray[pos];
+        }
+
+        int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+        int32_t end = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, key, order);
+
+        int32_t start = -1;
+        if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+          int32_t remain = end - pos + 1;
+          if (remain + numOfRows > pQueryHandle->outputCapacity) {
+            end = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
+          }
+
+          start = pos;
+        } else {
+          int32_t remain = (pos - end) + 1;
+          if (remain + numOfRows > pQueryHandle->outputCapacity) {
+            end = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
+          }
+
+          start = end;
+          end = pos;
+        }
+
+        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
+        pos += (end - start + 1) * step;
+      }
+    } while (numOfRows < pQueryHandle->outputCapacity);
+    
+    if (numOfRows < pQueryHandle->outputCapacity) {
+      if (node == NULL ||
+          ((dataRowKey(SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+          ((dataRowKey(SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
+        // no data in cache or data in cache is greater than the ekey of time window, load data from file block
+        if (cur->win.skey == TSKEY_INITIAL_VAL) {
+          cur->win.skey = tsArray[pos];
+        }
+
+        int32_t start = -1;
+        int32_t end = -1;
+
+        // all remain data are qualified, but check the remain capacity in the first place.
+        if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+          int32_t remain = endPos - pos + 1;
+          if (remain + numOfRows > pQueryHandle->outputCapacity) {
+            endPos = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
+          }
+
+          start = pos;
+          end = endPos;
+        } else {
+          int32_t remain = pos + 1;
+          if (remain + numOfRows > pQueryHandle->outputCapacity) {
+            endPos = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
+          }
+
+          start = endPos;
+          end = pos;
+        }
+
+        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
+        pos += (end - start + 1) * step;
+      } else {
+        
+        while(numOfRows < pQueryHandle->outputCapacity && node != NULL &&
+            (((dataRowKey(SL_GET_NODE_DATA(node)) <= pQueryHandle->window.ekey) && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+            ((dataRowKey(SL_GET_NODE_DATA(node)) >= pQueryHandle->window.ekey) && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)))) {
+          SDataRow row = SL_GET_NODE_DATA(node);
+          TSKEY key = dataRowKey(row);
+          
+          copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
+          numOfRows += 1;
+          
+          if (cur->win.skey == TSKEY_INITIAL_VAL) {
+            cur->win.skey = key;
+          }
+  
+          cur->win.ekey = key;
+          cur->lastKey  = key + step;
+          cur->mixBlock = true;
+  
+          tSkipListIterNext(pCheckInfo->iter);
+          node = tSkipListIterGet(pCheckInfo->iter);
+        }
+      }
+    }
+  }
+  
+  cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+      ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)));
+
+  if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+    SWAP(cur->win.skey, cur->win.ekey, TSKEY);
+  
+    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+    if (numOfRows < pQueryHandle->outputCapacity) {
+      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
+  
+      int32_t reqiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+      for(int32_t i = 0; i < reqiredNumOfCols; ++i) {
+        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+      }
+    }
+  }
+  
+  pCheckInfo->lastKey = cur->lastKey;
+  pQueryHandle->realNumOfRows = numOfRows;
+  cur->rows = numOfRows;
+  cur->pos = pos;
+
+  uTrace("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" %p", pQueryHandle, cur->win.skey,
+      cur->win.ekey, cur->rows, pQueryHandle->qinfo);
 }
 
 int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
@@ -879,7 +1259,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) {
   // no data in file anymore
   if (pQueryHandle->numOfBlocks <= 0) {
     assert(pQueryHandle->pFileGroup == NULL);
-    cur->fid = -1;
+    cur->fid = -1;  // denote that there are no data in file anymore
     
     return false;
   }
@@ -888,10 +1268,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) {
   cur->fid = pQueryHandle->pFileGroup->fileId;
   
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
-  STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
+  return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo);
-  SCompBlock*      pBlock = pBlockInfo->pBlock.compBlock;
-  
-  return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
 }
 
 static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
@@ -901,30 +1278,37 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
   // find the start data block in file
   if (!pQueryHandle->locateStart) {
     pQueryHandle->locateStart = true;
-
+    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pQueryHandle->pTsdb->config.daysPerFile);
-    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey);
     
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
 
     return getDataBlocksInFilesImpl(pQueryHandle);
   } else {
-    if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
+    // check if current file block is all consumed
-        (cur->slot == 0 && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) { // all blocks
+    STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
-      
+    STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
-      return getDataBlocksInFilesImpl(pQueryHandle);
+    
-    } else {  // next block of the same file
+    // current block is done, try next
-      int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 1:-1;
+    if (!cur->mixBlock || cur->blockCompleted) {
-      cur->slot += step;
+      if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) ||
-      
+          (cur->slot == 0 && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) {
-      STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
+        // all data blocks in current file has been checked already, try next file if exists
-      if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+        return getDataBlocksInFilesImpl(pQueryHandle);
-        cur->pos = 0;
+      } else {  // next block of the same file
-      } else {
+        int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) ? 1 : -1;
-        cur->pos = pBlockInfo->pBlock.compBlock->numOfPoints - 1;
+        cur->slot += step;
+        
+        cur->mixBlock = false;
+        cur->blockCompleted = false;
+        
+        STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
+        return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo);
       }
-
+    } else {
-      return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo);
+      SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
+      mergeDataInDataBlock(pQueryHandle, pCheckInfo, pBlockInfo->pBlock.compBlock, sa);
+      return true;
     }
   }
 }
@@ -951,28 +1335,18 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) {
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   assert(numOfTables > 0);
   
-  if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) {
+  if (pQueryHandle->checkFiles) {
-    if (pQueryHandle->checkFiles) {
+    if (getDataBlocksInFiles(pQueryHandle)) {
-      if (getDataBlocksInFiles(pQueryHandle)) {
+      return true;
-        return true;
-      }
-
-      pQueryHandle->activeIndex = 0;
-      pQueryHandle->checkFiles  = false;
     }
-    
+  
-    return doHasDataInBuffer(pQueryHandle);
+    pQueryHandle->activeIndex = 0;
-  } else {  // starts from the buffer in case of descending timestamp order check data blocks
+    pQueryHandle->checkFiles  = false;
-    if (!pQueryHandle->checkFiles) {
-      if (doHasDataInBuffer(pQueryHandle)) {
-        return true;
-      }
-      
-      pQueryHandle->checkFiles = true;
-    }
-
-    return getDataBlocksInFiles(pQueryHandle);
   }
+  
+  // TODO: opt by using lastKeyOnFile
+  // TODO: opt by consider the scan order
+  return doHasDataInBuffer(pQueryHandle);
 }
 
 void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
@@ -1022,6 +1396,8 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   taosArrayDestroy(pQueryHandle->pTableCheckInfo);
   
   pQueryHandle->pTableCheckInfo = taosArrayInit(1, sizeof(STableCheckInfo));
+  
+  info.lastKey = key;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
   
   // update the query time window according to the chosen last timestamp
@@ -1032,7 +1408,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY
                                  STsdbQueryHandle* pQueryHandle) {
   int     numOfRows = 0;
   int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-  *skey = INT64_MIN;
+  *skey = TSKEY_INITIAL_VAL;
 
   do {
     SSkipListNode* node = tSkipListIterGet(pIter);
@@ -1117,34 +1493,29 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
 
   STable* pTable = NULL;
-
-  TSKEY   skey = 0, ekey = 0;
   int32_t rows = 0;
 
   int32_t step = ASCENDING_ORDER_TRAVERSE(pHandle->order)? 1:-1;
   
-  // data in file
+  // there are data in file
   if (pHandle->cur.fid >= 0) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
+    STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
 
-    pTable = pBlockInfo->pTableCheckInfo->pTableObj;
+    pTable = pCheckInfo->pTableObj;
 
-    SDataBlockInfo binfo = getTrueDataBlockInfo(pBlockInfo->pTableCheckInfo, pBlockInfo->pBlock.compBlock);
+    if (pHandle->cur.mixBlock) {
-    if (binfo.rows == pHandle->realNumOfRows) {
+      SDataBlockInfo blockInfo = {
-      pBlockInfo->pTableCheckInfo->lastKey = pBlockInfo->pBlock.compBlock->keyLast + 1;
+          .uid = pTable->tableId.uid,
-      return binfo;
+          .tid = pTable->tableId.tid,
+          .rows = pHandle->cur.rows,
+          .window = pHandle->cur.win,
+      };
+  
+      return blockInfo;
     } else {
-      /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the
+      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
-       * block next function
+      return binfo;
-       */
-      SColumnInfoData* pColInfoEx = taosArrayGet(pHandle->pColumns, 0);
-
-      rows = pHandle->realNumOfRows;
-      skey = *(TSKEY*)pColInfoEx->pData;
-      ekey = *(TSKEY*)((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1));
-
-      // update the last key value
-      pBlockInfo->pTableCheckInfo->lastKey = ekey + step;
     }
   } else {
     STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
@@ -1153,21 +1524,28 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
     if (pTable->mem != NULL) {
       // create mem table iterator if it is not created yet
       assert(pCheckInfo->iter != NULL);
-      rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey, 4000, &skey, &ekey, pHandle);
+      STimeWindow* win = &pHandle->cur.win;
+      
+      rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
+          pHandle->outputCapacity, &win->skey, &win->ekey, pHandle);  // todo refactor API
 
       // update the last key value
-      pCheckInfo->lastKey = ekey + step;
+      pCheckInfo->lastKey = win->ekey + step;
     }
+  
+    if (!ASCENDING_ORDER_TRAVERSE(pHandle->order)) {
+      SWAP(pHandle->cur.win.skey, pHandle->cur.win.ekey, TSKEY);
+    }
+    
+    SDataBlockInfo blockInfo = {
+        .uid = pTable->tableId.uid,
+        .tid = pTable->tableId.tid,
+        .rows = rows,
+        .window = pHandle->cur.win,
+    };
+  
+    return blockInfo;
   }
-
-  SDataBlockInfo blockInfo = {
-      .uid = pTable->tableId.uid,
-      .tid = pTable->tableId.tid,
-      .rows = rows,
-      .window = {.skey = MIN(skey, ekey), .ekey = MAX(skey, ekey)}
-  };
-
-  return blockInfo;
 }
 
 // return null for data block in cache
@@ -1189,26 +1567,36 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
     STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot];
     STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo;
 
-    SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock);
+    if (pHandle->cur.mixBlock) {
-    assert(pHandle->realNumOfRows <= binfo.rows);
-
-    if (pHandle->realNumOfRows < binfo.rows) {
       return pHandle->pColumns;
     } else {
+      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock);
+      assert(pHandle->realNumOfRows <= binfo.rows);
+  
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
       
       if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
-      } else {
+      } else {  // only load the file block
         SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock;
         doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
-
+  
-        SArray* sa = getDefaultLoadColumns(pHandle, true);
+        // todo refactor
-        filterDataInDataBlock(pHandle, pCheckInfo, pBlock, sa);
+        int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfPoints - 1);
-        taosArrayDestroy(sa);
+  
-
+        // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+        if (!ASCENDING_ORDER_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
+          int32_t emptySize = pHandle->outputCapacity - numOfRows;
+          int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns);
+    
+          for(int32_t i = 0; i < reqNumOfCols; ++i) {
+            SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
+            memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+          }
+        }
+        
         return pHandle->pColumns;
       }
     }
@@ -1632,7 +2020,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
    size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
    for (int32_t i = 0; i < cols; ++i) {
      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-      tfree(pColInfo->pData);
+     tfree(pColInfo->pData);
    }
 
   taosArrayDestroy(pQueryHandle->pColumns);

src/util/inc/talgo.h

@@ -26,6 +26,8 @@ extern "C" {
 #define TD_GE (TD_EQ | TD_GT)
 #define TD_LE (TD_EQ | TD_LT)
 
+#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
+
 typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void *param);
 
 /**

src/util/src/talgo.c

@@ -23,8 +23,6 @@
   memcpy((__right), (__buf), (__size));\
 } while (0);
 
-#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
-
 static void median(void *src, size_t size, size_t s, size_t e, const void *param, __ext_compar_fn_t comparFn, void* buf) {
   int32_t mid = ((e - s) >> 1u) + s;
   

src/util/src/tskiplist.c

@@ -573,6 +573,7 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
     pthread_rwlock_unlock(pSkipList->lock);
   }
   
+  iter->step += 1;
   return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead);
 }
 

tests/examples/c/demo.c

@@ -74,9 +74,8 @@ int main(int argc, char *argv[]) {
   printf("success to connect to server\n");
   
   doQuery(taos, "create database if not exists test");
-  doQuery(taos, "create database if not exists test");
+  doQuery(taos, "use test");
-//  doQuery(taos, "use test");
+  doQuery(taos, "select count(*) from m1 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:1:59' interval(500a) fill(value, 99)");
-//  doQuery(taos, "select sum(k)*max(k), sum(k), max(k) from tm99");
   
 //  doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))");
 //  for(int32_t i = 0; i< 100000; ++i) {

tests/script/general/parser/create_mt.sim

@@ -45,10 +45,6 @@ sql show stables
 if $rows != 0 then
   return -1
 endi
-print data00 = $data00
-if $data00 != NULL then
-  return -1
-endi
 print case_insensitivity test passed
 
 # case2: illegal_metric_name test

tests/script/general/parser/create_tb.sim

@@ -44,10 +44,6 @@ sql show tables
 if $rows != 0 then
   return -1
 endi
-print data00 = $data00
-if $data00 != NULL then
-  return -1
-endi
 print case_insensitivity test passed
 
 # case2: illegal_table_name test

tests/script/general/parser/limit_tb.sim

@@ -111,18 +111,7 @@ endi
 if $data09 != nchar0 then
   return -1
 endi
-if $data11 != NULL then
+
-  return -1
-endi
-if $data12 != NULL then
-  return -1
-endi
-if $data13 != NULL then
-  return -1
-endi
-if $data14 != NULL then
-  return -1
-endi
 
 ## TBASE-329
 sql select * from $tb where c1 < 9 order by ts desc limit 1 offset 1
@@ -537,7 +526,8 @@ endi
 if $data14 != 8.000000000 then
   return -1
 endi
-if $data21 != NULL then
+if $data21 != null then
+  print expect null, actual: $data21
   return -1
 endi
 
@@ -554,7 +544,7 @@ endi
 if $data21 != 9 then
   return -1
 endi
-if $data31 != NULL then
+if $data31 != null then
   return -1
 endi
 
@@ -574,7 +564,7 @@ endi
 if $data31 != 9 then
   return -1
 endi
-if $data41 != NULL then
+if $data41 != null then
   return -1
 endi
 sql select sum(c1), sum(c2), sum(c3), sum(c4), sum(c5), sum(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 5 offset 1
@@ -590,7 +580,7 @@ endi
 if $data21 != 9 then
   return -1
 endi
-if $data31 != NULL then
+if $data31 != null then
   return -1
 endi
 
@@ -607,7 +597,7 @@ endi
 if $data21 != 7.000000000 then
   return -1
 endi
-if $data31 != NULL then
+if $data31 != null then
   return -1
 endi
 sql select avg(c1), avg(c2), avg(c3), avg(c4), avg(c5), avg(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 3 offset 1
@@ -623,7 +613,7 @@ endi
 if $data21 != 9.000000000 then
   return -1
 endi
-if $data31 != NULL then
+if $data31 != null then
   return -1
 endi
 

tests/script/general/parser/null_char.sim

@@ -84,43 +84,43 @@ endi
 #### case 1: tag NULL, or 'NULL'
 sql create table mt2 (ts timestamp, col1 int, col3 float, col5 binary(8), col6 bool, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag5 bool)
 sql create table st2 using mt2 tags (NULL, 'NULL', 102, 'true')
-sql describe st2
+sql select tag1, tag2, tag3, tag5 from st2
-if $rows != 10 then
+if $rows != 1 then
   return -1
 endi
-if $data63 != NULL then
+if $data00 != NULL then
-  print ==1== expect: NULL, actually: $data63
+  print ==1== expect: NULL, actually: $data00
   return -1
 endi
-if $data73 != NULL then
+if $data01 != NULL then
-  print ==2== expect: NULL, actually: $data73
+  print ==2== expect: NULL, actually: $data01
   return -1
 endi
-if $data83 != 102 then
+if $data02 != 102 then
-  print ==3== expect: NULL, actually: $data83
+  print ==3== expect: NULL, actually: $data02
   return -1
 endi
-if $data93 != true then
+if $data03 != 1 then
-  print ==4== expect: NULL, actually: $data93
+  print ==4== expect: 1, actually: $data03
   return -1
 endi
 
 sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE')
-sql describe st3
+sql select tag1, tag2, tag3, tag5 from st3
-if $rows != 10 then
+if $rows != 1 then
   return -1
 endi
-if $data63 != NULL then
+if $data00 != NULL then
-  print ==5== expect: NULL, actually: $data63
+  print ==5== expect: NULL, actually: $data00
   return -1
 endi
-if $data73 != ABC then
+if $data01 != ABC then
   return -1
 endi
-if $data83 != 103 then
+if $data02 != 103 then
   return -1
 endi
-if $data93 != false then
+if $data03 != 0 then
   return -1
 endi
 
@@ -128,39 +128,39 @@ endi
 sql_error create table stx using mt2 tags ('NULL', '123aBc', 104, '123')
 sql_error create table sty using mt2 tags ('NULL', '123aBc', 104, 'xtz')
 sql create table st4 using mt2 tags ('NULL', '123aBc', 104, 'NULL')
-sql describe st4
+sql select tag1,tag2,tag3,tag5 from st4
-if $rows != 10 then
+if $rows != 1 then
   return -1
 endi
-if $data63 != NULL then
+if $data00 != NULL then
   return -1
 endi
-if $data73 != 123aBc then
+if $data01 != 123aBc then
   return -1
 endi
-if $data83 != 104 then
+if $data02 != 104 then
   return -1
 endi
-if $data93 != NULL then
+if $data03 != NULL then
-  print ==6== expect: NULL, actually: $data93
+  print ==6== expect: NULL, actually: $data03
   return -1
 endi
 
 sql create table st5 using mt2 tags ('NULL', '123aBc', 105, NULL)
-sql describe st5
+sql select tag1,tag2,tag3,tag5 from st5
-if $rows != 10 then
+if $rows != 1 then
   return -1
 endi
-if $data63 != NULL then
+if $data00 != NULL then
   return -1
 endi
-if $data73 != 123aBc then
+if $data01 != 123aBc then
   return -1
 endi
-if $data83 != 105 then
+if $data02 != 105 then
   return -1
 endi
-if $data93 != NULL then
+if $data03 != NULL then
   return -1
 endi
 
@@ -177,28 +177,29 @@ sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL)   values
 #### case 3: set tag value
 sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double)
 sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789)
-sql describe st41
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double st41
-if $rows != 8 then
+if $rows != 1 then
   return -1
 endi
-if $data23 != beijing then
+if $data00 != beijing then
   return -1
 endi
-if $data33 != nchar_tag then
+if $data01 != nchar_tag then
   return -1
 endi
-if $data43 != 100 then
+if $data02 != 100 then
   return -1
 endi
-if $data53 != false then
+if $data03 != false then
   return -1
 endi
-if $data63 != 9.123450 then
+if $dat04 != 9.123450 then
   return -1
 endi
-if $data73 != 7.123457 then
+if $data05 != 7.123457 then
   return -1
 endi
+
 ################# binary
 sql alter table st41 set tag tag_binary = "shanghai"
 sql describe st41

tests/script/general/parser/selectResNum.sim

@@ -23,7 +23,7 @@ $stb = $stbPrefix . $i
 
 sql drop database $db -x step1
 step1:
-sql create database $db cache 2048 tables 200
+sql create database $db cache 16 maxtables 200
 print ====== create tables
 sql use $db
 sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int)

tests/script/general/parser/single_row_in_tb.sim

@@ -15,7 +15,7 @@ $db = $dbPrefix
 $stb = $stbPrefix
 
 sql drop database if exists $db
-sql create database $db rows 200 maxTables 4
+sql create database $db maxrows 200 maxTables 4
 print ====== create tables
 sql use $db
 sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 bool, c6 binary(10), c7 nchar(10)) tags(t1 int)

tests/script/general/parser/single_row_in_tb_query.sim

@@ -130,6 +130,7 @@ if $data03 != 1 then
   return -1
 endi
 if $data04 != 0.000000000 then
+  print expect: 0.00000000 , actual: $data04
   return -1
 endi
 if $data05 != 1 then