From 54890e481558fcdb17e3d05ac25353b3cff14127 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 22 May 2020 22:45:47 +0800
Subject: [PATCH 01/18] [td-225] fix bug in test script

---
 tests/script/general/parser/slimit.sim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/script/general/parser/slimit.sim b/tests/script/general/parser/slimit.sim
index 161463a7c1..be63f91803 100644
--- a/tests/script/general/parser/slimit.sim
+++ b/tests/script/general/parser/slimit.sim
@@ -22,7 +22,7 @@ $db = $dbPrefix . $i
 $stb = $stbPrefix . $i
 
 sql drop database if exists $db
-sql create database $db maxrows 200 cache 1024 tblocks 200 maxTables 4
+sql create database $db maxrows 200 cache 16 maxTables 4
 print ====== create tables
 sql use $db
 sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 binary(15), t2 int, t3 bigint, t4 nchar(10), t5 double, t6 bool)
@@ -64,7 +64,7 @@ print ====== $db tables created
 
 $db = $dbPrefix . 1
 sql drop database if exists $db
-sql create database $db maxrows 200 cache 1024
+sql create database $db maxrows 200 cache 16
 sql use $db
 sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 binary(15), t2 int, t3 bigint, t4 nchar(10), t5 double, t6 bool)
 

From 95c97a5df812fa9c81ca35550b2ce8d2dffb4905 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 22 May 2020 22:46:11 +0800
Subject: [PATCH 02/18] [td-225]refactor code: remove useless codes

---
 src/client/src/tscLocal.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index 79fa1e28cc..5ffdcd2167 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -384,13 +384,10 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
 
   // keep the code in local variable in order to avoid invalid read in case of async query
   int32_t code = pSql->res.code;
-
-  if (pSql->fp != NULL) {  // callback function
-    if (code == 0) {
-      (*pSql->fp)(pSql->param, pSql, 0);
-    } else {
-      tscQueueAsyncRes(pSql);
-    }
+  if (code == TSDB_CODE_SUCCESS) {
+    (*pSql->fp)(pSql->param, pSql, 0);
+  } else {
+    tscQueueAsyncRes(pSql);
   }
 
   return code;

From 7f514df778ce4545eec6e317273c905a0b8cf935 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 11:40:52 +0800
Subject: [PATCH 03/18] [td-338]

---
 src/client/src/tscSQLParser.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index a134410bc4..d68290653a 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5606,7 +5606,8 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
   const char* msg3 = "fill only available for interval query";
   const char* msg4 = "fill option not supported in stream computing";
   const char* msg5 = "sql too long";  // todo ADD support
-
+  const char* msg6 = "from missing in subclause";
+  
   SSqlCmd*    pCmd = &pSql->cmd;
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   assert(pQueryInfo->numOfTables == 1);
@@ -5621,10 +5622,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
   if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
-
+  
   tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
-  tVariant*     pVar = &pSrcMeterName->a[0].pVar;
-
+  if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
+  }
+  
+  tVariant* pVar = &pSrcMeterName->a[0].pVar;
   SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
   if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(pQueryInfo->msg, msg1);

From 7221f5a62497652d445182275b0a5c9d952e9943 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 16:10:44 +0800
Subject: [PATCH 04/18] [td-227]

---
 src/inc/taosdef.h         |   3 +-
 src/query/src/qExecutor.c |  40 +++---
 src/tsdb/src/tsdbRead.c   | 266 ++++++++++++++++++++------------------
 3 files changed, 159 insertions(+), 150 deletions(-)

diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index b6a37d85a2..74de811c59 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -326,8 +326,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 
 #define TSDB_QUERY_TYPE_TAG_FILTER_QUERY              0x400u
 #define TSDB_QUERY_TYPE_INSERT                        0x100u    // insert type
-#define TSDB_QUERY_TYPE_IMPORT                        0x200u    // import data
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY              0x800u
+#define TSDB_QUERY_TYPE_MULTITABLE_QUERY              0x200u
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)         (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)         ((x) |= (_type))
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 052273a90c..a8158a386d 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -403,23 +403,24 @@ static bool isTopBottomQuery(SQuery *pQuery) {
   return false;
 }
 
-static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) {
+static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, int32_t numOfCols, int32_t index) {
   // for a tag column, no corresponding field info
-  SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].base.colInfo;
-  if (TSDB_COL_IS_TAG(pColIndexEx->flag)) {
+  SColIndex *pColIndex = &pQuery->pSelectExpr[index].base.colInfo;
+  if (TSDB_COL_IS_TAG(pColIndex->flag)) {
     return NULL;
   }
-
+  
   /*
    * Choose the right column field info by field id, since the file block may be out of date,
    * which means the newest table schema is not equalled to the schema of this block.
+   * TODO: speedup by using bsearch
    */
-  for (int32_t i = 0; i < pDataBlockInfo->numOfCols; ++i) {
-    if (pColIndexEx->colId == pStatis[i].colId) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    if (pColIndex->colId == pStatis[i].colId) {
       return &pStatis[i];
     }
   }
-
+  
   return NULL;
 }
 
@@ -431,8 +432,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo
  * @param pColStatis
  * @return
  */
-static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis,
-                         SDataStatis **pColStatis) {
+static bool hasNullValue(SQuery *pQuery, int32_t col, int32_t numOfCols, SDataStatis *pStatis, SDataStatis **pColStatis) {
   SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo;
   if (TSDB_COL_IS_TAG(pColIndex->flag)) {
     return false;
@@ -444,7 +444,7 @@ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlock
   }
 
   if (pStatis != NULL) {
-    *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col);
+    *pColStatis = getStatisInfo(pQuery, pStatis, numOfCols, col);
   } else {
     *pColStatis = NULL;
   }
@@ -936,7 +936,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
 
     SDataStatis *tpField = NULL;
 
-    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField);
+    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
     setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull,
@@ -1157,7 +1157,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
     SDataStatis *pColStatis = NULL;
 
-    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis);
+    bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
     setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull,
@@ -2455,9 +2455,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     }
 
     SDataStatis *pStatis = NULL;
-    SArray *     pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
-
     pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
+    
+    SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
     int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
 
     qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
@@ -5610,18 +5610,18 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
       continue;
     }
 
-    SColIndex *pColIndexEx = &pSqlExprMsg->colInfo;
-    if (!TSDB_COL_IS_TAG(pColIndexEx->flag)) {
+    SColIndex *pColIndex = &pSqlExprMsg->colInfo;
+    if (!TSDB_COL_IS_TAG(pColIndex->flag)) {
       for (int32_t f = 0; f < pQuery->numOfCols; ++f) {
-        if (pColIndexEx->colId == pQuery->colList[f].colId) {
-          pColIndexEx->colIndex = f;
+        if (pColIndex->colId == pQuery->colList[f].colId) {
+          pColIndex->colIndex = f;
           break;
         }
       }
     } else {
       for (int32_t f = 0; f < pQuery->numOfTags; ++f) {
-        if (pColIndexEx->colId == pQuery->tagColList[f].colId) {
-          pColIndexEx->colIndex = f;
+        if (pColIndex->colId == pQuery->tagColList[f].colId) {
+          pColIndex->colIndex = f;
           break;
         }
       }
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index b5340d4ba9..cc191f3900 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -40,10 +40,6 @@ enum {
   TSDB_QUERY_TYPE_EXTERNAL = 3,
 };
 
-typedef struct SField {
-  // todo need the definition
-} SField;
-
 typedef struct SQueryFilePos {
   int32_t fid;
   int32_t slot;
@@ -68,66 +64,56 @@ typedef struct SLoadCompBlockInfo {
 } SLoadCompBlockInfo;
 
 typedef struct STableCheckInfo {
-  STableId   tableId;
-  TSKEY      lastKey;
-  STable*    pTableObj;
-  int32_t    start;
-  SCompInfo* pCompInfo;
-  int32_t    compSize;
-  int32_t    numOfBlocks;    // number of qualified data blocks not the original blocks
-  SDataCols* pDataCols;
-  
-  SSkipListIterator* iter;   // skip list iterator
-  SSkipListIterator* iiter;  // imem iterator
-  
-  bool       initBuf;   // if we should initialize the in-memory skip list iterator
+  STableId      tableId;
+  TSKEY         lastKey;
+  STable*       pTableObj;
+  SCompInfo*    pCompInfo;
+  int32_t       compSize;
+  int32_t       numOfBlocks;    // number of qualified data blocks not the original blocks
+  SDataCols*    pDataCols;
+  bool          initBuf;        // whether to initialize the in-memory skip list iterator or not
+  SSkipListIterator* iter;      // mem buffer skip list iterator
+  SSkipListIterator* iiter;     // imem buffer skip list iterator
 } STableCheckInfo;
 
-typedef struct {
-  SCompBlock* compBlock;
-  SField*     fields;
-} SCompBlockFields;
-
 typedef struct STableBlockInfo {
-  SCompBlockFields pBlock;
-  STableCheckInfo* pTableCheckInfo;
-  int32_t          blockIndex;
-  int32_t          groupIdx; /* number of group is less than the total number of tables */
+  SCompBlock*        compBlock;
+  STableCheckInfo*   pTableCheckInfo;
+//  int32_t            blockIndex;
+//  int32_t            groupIdx; /* number of group is less than the total number of tables */
 } STableBlockInfo;
 
 typedef struct SBlockOrderSupporter {
   int32_t             numOfTables;
-  STableBlockInfo** pDataBlockInfo;
+  STableBlockInfo**   pDataBlockInfo;
   int32_t*            blockIndexArray;
   int32_t*            numOfBlocksPerTable;
 } SBlockOrderSupporter;
 
 typedef struct STsdbQueryHandle {
-  STsdbRepo*    pTsdb;
-  SQueryFilePos cur;    // current position
-
-  SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
-  SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
-
-  int16_t     order;
-  STimeWindow window;  // the primary query time window that applies to all queries
-  SCompBlock* pBlock;
-  int32_t     numOfBlocks;
-  SField**    pFields;
-  SArray*     pColumns;    // column list, SColumnInfoData array list
-  bool        locateStart;
-  int32_t     outputCapacity;
-  int32_t     realNumOfRows;
-  SArray*     pTableCheckInfo;  //SArray<STableCheckInfo>
-  int32_t     activeIndex;
-  bool        checkFiles;  // check file stage
-  void*       qinfo;       // query info handle, for debug purpose
-  int32_t     type;        // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
-  STableBlockInfo* pDataBlockInfo;
-
+  STsdbRepo*     pTsdb;
+  SQueryFilePos  cur;              // current position
+  int16_t        order;
+  STimeWindow    window;           // the primary query time window that applies to all queries
+  SCompBlock*    pBlock;
+  SDataStatis*   statis;           // query level statistics, only one table block statistics info exists at any time
+  int32_t        numOfBlocks;
+  SArray*        pColumns;         // column list, SColumnInfoData array list
+  bool           locateStart;
+  int32_t        outputCapacity;
+  int32_t        realNumOfRows;
+  SArray*        pTableCheckInfo;  //SArray<STableCheckInfo>
+  int32_t        activeIndex;
+  bool           checkFiles;       // check file stage
+  void*          qinfo;            // query info handle, for debug purpose
+  int32_t        type;             // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
   SFileGroup*    pFileGroup;
   SFileGroupIter fileIter;
   SRWHelper      rhelper;
+  STableBlockInfo* pDataBlockInfo;
+  
+  SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
+  SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
 } STsdbQueryHandle;
 
 static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle);
@@ -148,23 +134,43 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
   // todo 2. add the reference count for each table that is involved in query
 
   STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
-  pQueryHandle->order  = pCond->order;
-  pQueryHandle->window = pCond->twindow;
-  pQueryHandle->pTsdb  = tsdb;
-  pQueryHandle->type   = TSDB_QUERY_TYPE_ALL;
+  pQueryHandle->order       = pCond->order;
+  pQueryHandle->window      = pCond->twindow;
+  pQueryHandle->pTsdb       = tsdb;
+  pQueryHandle->type        = TSDB_QUERY_TYPE_ALL;
+  pQueryHandle->cur.fid     = -1;
+  pQueryHandle->cur.win     = TSWINDOW_INITIALIZER;
+  pQueryHandle->checkFiles  = true;//ASCENDING_TRAVERSE(pQueryHandle->order);
+  pQueryHandle->activeIndex = 0;   // current active table index
+  pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
+  
   tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
 
-  pQueryHandle->cur.fid = -1;
-  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
-
   size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
   assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
 
+  // allocate buffer in order to load data blocks from file
+  int32_t numOfCols = pCond->numOfCols;
+  
+  pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
+  pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));  // todo: use list instead of array?
+  
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData  colInfo = {{0}, 0};
+  
+    colInfo.info = pCond->colList[i];
+    colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes);
+    taosArrayPush(pQueryHandle->pColumns, &colInfo);
+    pQueryHandle->statis[i].colId = colInfo.info.colId;
+  }
+  
   pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
+  STsdbMeta* pMeta = tsdbGetMeta(tsdb);
+  assert(pMeta != NULL);
   
   for (int32_t i = 0; i < sizeOfGroup; ++i) {
     SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
-
+    
     size_t gsize = taosArrayGetSize(group);
     assert(gsize > 0);
     
@@ -174,35 +180,18 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
       STableCheckInfo info = {
           .lastKey = pQueryHandle->window.skey,
           .tableId = *id,
-          .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid),
+          .pTableObj = tsdbGetTableByUid(pMeta, id->uid),
       };
-
+      
       assert(info.pTableObj != NULL && info.pTableObj->tableId.tid == id->tid);
       taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
     }
   }
-
-  uTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
   
-  /*
-   * For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place
-   * in case of descending timestamp order query.
-   */
-  pQueryHandle->checkFiles  = true;//ASCENDING_TRAVERSE(pQueryHandle->order);
-  pQueryHandle->activeIndex = 0;
-
-  // allocate buffer in order to load data blocks from file
-  int32_t numOfCols = pCond->numOfCols;
-  pQueryHandle->outputCapacity = 4096;
-  
-  pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
-  for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    SColumnInfoData  colInfo = {{0}, 0};
-  
-    colInfo.info = pCond->colList[i];
-    colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes);
-    taosArrayPush(pQueryHandle->pColumns, &colInfo);
+  for(int32_t i = 0; i < numOfCols; ++i) {
   }
+  
+  uTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
 
   tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
   tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
@@ -499,8 +488,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
   bool    blockLoaded = false;
   SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
 
-  if (pCheckInfo->pDataCols == NULL) { // todo: why not the real data?
-    pCheckInfo->pDataCols = tdNewDataCols(pRepo->tsdbMeta->maxRowBytes, pRepo->tsdbMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
+  if (pCheckInfo->pDataCols == NULL) {
+    STsdbMeta* pMeta = tsdbGetMeta(pRepo);
+    pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
   }
 
   tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj));
@@ -522,8 +512,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 }
 
 static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
-  SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
-  
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
   /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
@@ -591,9 +579,12 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
       cur->blockCompleted = false;
       return;
     }
-    
+  
+    SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
     doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
     doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa);
+    taosArrayDestroy(sa);
+    
   } else {
     pQueryHandle->realNumOfRows = binfo.rows;
     
@@ -1074,7 +1065,9 @@ static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t n
   tfree(pSupporter->blockIndexArray);
 
   for (int32_t i = 0; i < numOfTables; ++i) {
-    tfree(pSupporter->pDataBlockInfo[i]);
+    STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
+//    tfree(pBlockInfo->statis);
+    tfree(pBlockInfo);
   }
 
   tfree(pSupporter->pDataBlockInfo);
@@ -1100,14 +1093,14 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
   STableBlockInfo* pLeftBlockInfoEx = &pSupporter->pDataBlockInfo[leftTableIndex][leftTableBlockIndex];
   STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex];
 
-  //    assert(pLeftBlockInfoEx->pBlock.compBlock->offset != pRightBlockInfoEx->pBlock.compBlock->offset);
-  if (pLeftBlockInfoEx->pBlock.compBlock->offset == pRightBlockInfoEx->pBlock.compBlock->offset &&
-      pLeftBlockInfoEx->pBlock.compBlock->last == pRightBlockInfoEx->pBlock.compBlock->last) {
+  //    assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset);
+  if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset &&
+      pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
     // todo add more information
-    uError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->pBlock.compBlock->offset);
+    uError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->compBlock->offset);
   }
 
-  return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1;
+  return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1;
 }
 
 static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) {
@@ -1116,7 +1109,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
 
-  pQueryHandle->pDataBlockInfo = (STableBlockInfo*)tmp;
+  pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp;
   memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks);
   *numOfAllocBlocks = numOfBlocks;
 
@@ -1132,9 +1125,10 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     cleanBlockOrderSupporter(&sup, 0);
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
-
+  
   int32_t cnt = 0;
   int32_t numOfQualTables = 0;
+  
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j);
     if (pTableCheck->numOfBlocks <= 0) {
@@ -1153,14 +1147,12 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     sup.pDataBlockInfo[numOfQualTables] = (STableBlockInfo*)buf;
 
     for (int32_t k = 0; k < pTableCheck->numOfBlocks; ++k) {
-      STableBlockInfo* pBlockInfoEx = &sup.pDataBlockInfo[numOfQualTables][k];
+      STableBlockInfo* pBlockInfo = &sup.pDataBlockInfo[numOfQualTables][k];
 
-      pBlockInfoEx->pBlock.compBlock = &pBlock[k];
-      pBlockInfoEx->pBlock.fields = NULL;
-
-      pBlockInfoEx->pTableCheckInfo = pTableCheck;
-      //      pBlockInfoEx->groupIdx = pTableCheckInfo[j]->groupIdx;     // set the group index
-      //      pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k;    // set the block index in original table
+      pBlockInfo->compBlock = &pBlock[k];
+      pBlockInfo->pTableCheckInfo = pTableCheck;
+      //      pBlockInfo->groupIdx = pTableCheckInfo[j]->groupIdx;     // set the group index
+      //      pBlockInfo->blockIndex = pTableCheckInfo[j]->start + k;    // set the block index in original table
       cnt++;
     }
 
@@ -1185,8 +1177,8 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     int32_t pos = pTree->pNode[0].index;
     int32_t index = sup.blockIndexArray[pos]++;
 
-    STableBlockInfo* pBlocksInfoEx = sup.pDataBlockInfo[pos];
-    pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfoEx[index];
+    STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos];
+    pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index];
 
     // set data block index overflow, in order to disable the offset comparator
     if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) {
@@ -1199,7 +1191,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
   /*
    * available when no import exists
    * for(int32_t i = 0; i < cnt - 1; ++i) {
-   *   assert((*pDataBlockInfo)[i].pBlock.compBlock->offset < (*pDataBlockInfo)[i+1].pBlock.compBlock->offset);
+   *   assert((*pDataBlockInfo)[i].compBlock->offset < (*pDataBlockInfo)[i+1].compBlock->offset);
    * }
    */
 
@@ -1255,7 +1247,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) {
   cur->fid = pQueryHandle->pFileGroup->fileId;
   
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
-  return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo);
+  return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo);
 }
 
 static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
@@ -1291,10 +1283,10 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
         cur->blockCompleted = false;
         
         STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
-        return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo);
+        return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo);
       }
     } else {
-      handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->pBlock.compBlock, pCheckInfo);
+      handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo);
       return pQueryHandle->realNumOfRows > 0;
     }
   }
@@ -1484,35 +1476,33 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
   // there are data in file
   if (pHandle->cur.fid >= 0) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
-    STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
-  
-    STable* pTable = pCheckInfo->pTableObj;
+    STable* pTable = pBlockInfo->pTableCheckInfo->pTableObj;
 
-    if (pHandle->cur.mixBlock) {
-      SDataBlockInfo blockInfo = {
-          .uid = pTable->tableId.uid,
-          .tid = pTable->tableId.tid,
-          .rows = pHandle->cur.rows,
-          .window = pHandle->cur.win,
-      };
-  
-      return blockInfo;
-    } else {
-      return getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
-    }
+    SDataBlockInfo blockInfo = {
+        .uid = pTable->tableId.uid,
+        .tid = pTable->tableId.tid,
+        .rows = pHandle->cur.rows,
+        .window = pHandle->cur.win,
+        .numOfCols = QH_GET_NUM_OF_COLS(pHandle),
+    };
+
+    return blockInfo;
   } else {
     STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
+    SQueryFilePos* cur = &pHandle->cur;
     
     STable* pTable = pCheckInfo->pTableObj;
     if (pTable->mem != NULL) { // create mem table iterator if it is not created yet
       assert(pCheckInfo->iter != NULL);
-      STimeWindow* win = &pHandle->cur.win;
+      STimeWindow* win = &cur->win;
       
       pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
           pHandle->outputCapacity, &win->skey, &win->ekey, pHandle);  // todo refactor API
 
       // update the last key value
       pCheckInfo->lastKey = win->ekey + step;
+      cur->lastKey = win->ekey + step;
+      cur->mixBlock = true;
     }
   
     if (!ASCENDING_TRAVERSE(pHandle->order)) {
@@ -1524,15 +1514,34 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
         .tid = pTable->tableId.tid,
         .rows = pHandle->cur.rows,
         .window = pHandle->cur.win,
+        .numOfCols = QH_GET_NUM_OF_COLS(pHandle),
     };
   
     return blockInfo;
   }
 }
 
-// return null for data block in cache
+/*
+ * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL
+ */
 int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
-  *pBlockStatis = NULL;
+  STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
+  
+  SQueryFilePos* cur = &pHandle->cur;
+  if (cur->mixBlock) {
+    *pBlockStatis = NULL;
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  assert((cur->slot >= 0 && cur->slot < pHandle->numOfBlocks) ||
+      ((cur->slot == pHandle->numOfBlocks) && (cur->slot == 0)));
+  
+  STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
+  tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
+  
+  tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, QH_GET_NUM_OF_COLS(pHandle));
+  *pBlockStatis = pHandle->statis;
+  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1546,13 +1555,13 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
   if (pHandle->cur.fid < 0) {
     return pHandle->pColumns;
   } else {
-    STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot];
-    STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo;
+    STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
+    STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
 
     if (pHandle->cur.mixBlock) {
       return pHandle->pColumns;
     } else {
-      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock);
+      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->compBlock);
       assert(pHandle->realNumOfRows <= binfo.rows);
   
       // data block has been loaded, todo extract method
@@ -1562,7 +1571,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
-        SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock;
+        SCompBlock* pBlock = pBlockInfo->compBlock;
         doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
   
         // todo refactor
@@ -2006,8 +2015,9 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
    }
 
   taosArrayDestroy(pQueryHandle->pColumns);
-  
   tfree(pQueryHandle->pDataBlockInfo);
+  tfree(pQueryHandle->statis);
+  
   tsdbDestroyHelper(&pQueryHandle->rhelper);
   
   tfree(pQueryHandle);

From e46896bfcd7d4596d7000685a247875a8895b26d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 16:12:18 +0800
Subject: [PATCH 05/18] [td-225] support hex/bin numeric data to insert

---
 src/client/src/tscAsync.c        |  2 +-
 src/client/src/tscFunctionImpl.c | 23 ++++++++-------
 src/client/src/tscParseInsert.c  | 50 ++++++++++++++++++--------------
 3 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 26de2a51a8..6fba0efd97 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -48,7 +48,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   pSql->param     = param;
   pSql->pTscObj   = pObj;
   pSql->maxRetry  = TSDB_MAX_REPLICA_NUM;
-  pSql->fp = fp;
+  pSql->fp        = fp;
   
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     tscError("failed to malloc payload");
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index f3e24e43a9..168863d322 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -947,18 +947,21 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
       index = pCtx->preAggVals.statis.maxIndex;
     }
     
-    /**
-     * NOTE: work around the bug caused by invalid pre-calculated function.
-     * Here the selectivity + ts will not return correct value.
-     *
-     * The following codes of 3 lines will be removed later.
-     */
-    if (index < 0 || index >= pCtx->size + pCtx->startOffset) {
-      index = 0;
+    TSKEY key = TSKEY_INITIAL_VAL;
+    if (pCtx->ptsList != NULL) {
+      /**
+       * NOTE: work around the bug caused by invalid pre-calculated function.
+       * Here the selectivity + ts will not return correct value.
+       *
+       * The following codes of 3 lines will be removed later.
+       */
+      if (index < 0 || index >= pCtx->size + pCtx->startOffset) {
+        index = 0;
+      }
+      
+      key = pCtx->ptsList[index];
     }
     
-    TSKEY key = pCtx->ptsList[index];
-    
     if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
       int64_t val = GET_INT64_VAL(tval);
       if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 62f27c0960..3e1f0787c3 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -42,35 +42,42 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 
 static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
-//  int32_t numType = isValidNumber(pToken);
-//  if (TK_ILLEGAL == numType) {
-//    return numType;
-//  }
-
+  if (pToken->n == 0) {
+    return TK_ILLEGAL;
+  }
+  
   int32_t radix = 10;
-  if (pToken->type == TK_HEX) {
-    radix = 16;
-  } else if (pToken->type == TK_OCT) {
-    radix = 8;
-  } else if (pToken->type == TK_BIN) {
-    radix = 2;
+  
+  int32_t radixList[3] = {16, 8, 2};
+  if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
+    radix = radixList[pToken->type - TK_HEX];
   }
 
   errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
+  
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  }
 
   return pToken->type;
 }
 
 static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
-//  int32_t numType = isValidNumber(pToken);
-//  if (TK_ILLEGAL == numType) {
-//    return numType;
-//  }
-
+  if (pToken->n == 0) {
+    return TK_ILLEGAL;
+  }
+  
   errno = 0;
   *value = strtod(pToken->z, endPtr);
-  return pToken->type;
+  
+  // not a valid integer number, return error
+  if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
+    return TK_ILLEGAL;
+  } else {
+    return pToken->type;
+  }
 }
 
 int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@@ -422,9 +429,9 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
       return -1;
     }
 
-    if (((sToken.type != TK_NOW) && (sToken.type != TK_INTEGER) && (sToken.type != TK_STRING) &&
-         (sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) ||
-        (sToken.n == 0) || (sToken.type == TK_RP)) {
+    int16_t type = sToken.type;
+    if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
+         type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) {
       tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z);
       *code = TSDB_CODE_INVALID_SQL;
       return -1;
@@ -1306,8 +1313,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = NULL;
   tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
 
-  uint16_t type = (sToken.type == TK_INSERT)? TSDB_QUERY_TYPE_INSERT:TSDB_QUERY_TYPE_IMPORT;
-  TSDB_QUERY_SET_TYPE(pQueryInfo->type, type);
+  TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
 
   sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
   if (sToken.type != TK_INTO) {

From c6cb40bf5728a31db550995939cef8bb064a0e55 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 17:34:15 +0800
Subject: [PATCH 06/18] [td-225] fix bugs in tbname in query

---
 src/client/src/tscSQLParser.c | 15 +++++++++------
 src/client/src/tscSub.c       |  6 +++---
 src/query/src/qExecutor.c     |  2 +-
 src/query/src/qast.c          | 26 ++++++++++++++------------
 src/tsdb/src/tsdbRead.c       |  2 +-
 src/util/inc/tarray.h         |  6 +++---
 src/util/inc/tcompare.h       |  8 +++++---
 src/util/src/tarray.c         | 21 +++++++--------------
 src/util/src/tcompare.c       | 19 +++++++++++++------
 9 files changed, 56 insertions(+), 49 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index d68290653a..f338bbc644 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -3290,14 +3290,15 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg
 
 static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SCondExpr* pCondExpr, int32_t* type,
                                      int32_t parentOptr) {
-  const char* msg1 = "meter query cannot use tags filter";
+  const char* msg1 = "table query cannot use tags filter";
   const char* msg2 = "illegal column name";
   const char* msg3 = "only one query time range allowed";
   const char* msg4 = "only one join condition allowed";
   const char* msg5 = "not support ordinary column join";
   const char* msg6 = "only one query condition on tbname allowed";
   const char* msg7 = "only in/like allowed in filter table name";
-
+  const char* msg8 = "wildcard string should be less than 20 characters";
+  
   tSQLExpr* pLeft = (*pExpr)->pLeft;
   tSQLExpr* pRight = (*pExpr)->pRight;
 
@@ -3344,7 +3345,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
     // check for like expression
     if ((*pExpr)->nSQLOptr == TK_LIKE) {
       if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) {
-        return TSDB_CODE_INVALID_SQL;
+        return invalidSqlErrMsg(pQueryInfo->msg, msg8);
       }
 
       SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
@@ -3360,6 +3361,10 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
       if (!validTableNameOptr(*pExpr)) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg7);
       }
+  
+      if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg1);
+      }
 
       if (pCondExpr->pTableCond == NULL) {
         pCondExpr->pTableCond = *pExpr;
@@ -3808,9 +3813,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
   const char* msg2 = "invalid filter expression";
 
   int32_t ret = TSDB_CODE_SUCCESS;
-
-  pQueryInfo->window.skey = 0;
-  pQueryInfo->window.ekey = INT64_MAX;
+  pQueryInfo->window = TSWINDOW_INITIALIZER;
 
   // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
   SStringBuilder sb; memset(&sb, 0, sizeof(sb));
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 9092fdd0b3..5831ddad4a 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
   if (p == NULL) {
     return dflt;
   }
@@ -74,7 +74,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
   if (p != NULL) {
     p->key = ts;
   }
@@ -211,7 +211,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index a8158a386d..e9f1ffb49f 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -5745,7 +5745,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
       SGroupItem item = { .id = id };
       // NOTE: compare STableIdInfo with STableId
       // not a problem at present because we only use their 1st int64_t field
-      STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
+      STableIdInfo* pTableId = taosArraySearch( pTableIdList, &id, compareTableIdInfo);
       if (pTableId != NULL ) {
         window.skey = pTableId->key;
       } else {
diff --git a/src/query/src/qast.c b/src/query/src/qast.c
index d784fa4102..f35f4d0184 100644
--- a/src/query/src/qast.c
+++ b/src/query/src/qast.c
@@ -773,9 +773,6 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
 
     // todo refactor:
     tstr *name = ((STableIndexElem *)pData)->pTable->name;
-    //    char* name = NULL;
-//        tsdbGetTableName(pQueryInfo->, pTable, &name);
-
     // todo speed up by using hash
     if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
@@ -1051,7 +1048,7 @@ static void* exception_malloc(size_t size) {
   return p;
 }
 
-static char* exception_strdup(const char* str) {
+static UNUSED_FUNC char* exception_strdup(const char* str) {
   char* p = strdup(str);
   if (p == NULL) {
     THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
@@ -1154,28 +1151,33 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     tVariant* pVal = exception_calloc(1, sizeof(tVariant));
     right->pVal = pVal;
     pVal->nType = TSDB_DATA_TYPE_ARRAY;
-    pVal->arr = taosArrayInit(2, sizeof(char*));
+    pVal->arr = taosArrayInit(2, POINTER_BYTES);
 
     const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
     for (const char *e = cond; *e != 0; e++) {
       if (*e == TS_PATH_DELIMITER[0]) {
         cond = e + 1;
       } else if (*e == ',') {
-        size_t len = e - cond + 1;
-        char* p = exception_malloc( len );
-        memcpy(p, cond, len);
-        p[len - 1] = 0;
+        size_t len = e - cond + VARSTR_HEADER_SIZE;
+        char* p = exception_malloc(len);
+        varDataSetLen(p, len - VARSTR_HEADER_SIZE);
+        memcpy(varDataVal(p), cond, len);
         cond += len;
         taosArrayPush(pVal->arr, &p);
       }
     }
 
     if (*cond != 0) {
-        char* p = exception_strdup( cond );
-        taosArrayPush(pVal->arr, &p);
+      size_t len = strlen(cond) + VARSTR_HEADER_SIZE;
+      
+      char* p = exception_malloc(len);
+      varDataSetLen(p, len - VARSTR_HEADER_SIZE);
+      memcpy(varDataVal(p), cond, len);
+      
+      taosArrayPush(pVal->arr, &p);
     }
 
-    taosArraySortString(pVal->arr);
+    taosArraySortString(pVal->arr, taosArrayCompareString);
   }
 
   CLEANUP_EXECUTE_TO(anchor, false);
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index cc191f3900..7a70312259 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -1678,7 +1678,7 @@ void filterPrepare(void* expr, void* param) {
   tVariant*   pCond = pExpr->_node.pRight->pVal;
   SSchema*    pSchema = pExpr->_node.pLeft->pSchema;
 
-  // todo : if current super table does not change schema yet, this function may failed, add test case
+  // todo : if current super table does not change schema yet, this function may fail to get correct schema, test case
   int32_t index = getTagColumnIndex(pTSSchema, pSchema);
   assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX));
 
diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h
index 866bde0938..4d44e82b1b 100644
--- a/src/util/inc/tarray.h
+++ b/src/util/inc/tarray.h
@@ -129,7 +129,7 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*));
  * sort string array
  * @param pArray
  */
-void taosArraySortString(SArray* pArray);
+void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
 
 /**
  * search the array
@@ -137,14 +137,14 @@ void taosArraySortString(SArray* pArray);
  * @param compar
  * @param key
  */
-void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key);
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn);
 
 /**
  * search the array
  * @param pArray
  * @param key
  */
-char* taosArraySearchString(const SArray* pArray, const char* key);
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn);
 
 #ifdef __cplusplus
 }
diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h
index 8aaa39e483..5e951923c1 100644
--- a/src/util/inc/tcompare.h
+++ b/src/util/inc/tcompare.h
@@ -34,16 +34,18 @@ typedef struct SPatternCompareInfo {
   char matchOne;  // symbol for match one wildcard, default: '_'
 } SPatternCompareInfo;
 
-int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo);
+int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo);
+int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size);
+int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
 
 __compar_fn_t getKeyComparFunc(int32_t keyType);
 
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 
+int32_t taosArrayCompareString(const void* a, const void* b);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c
index 5198597ff7..8908dc2e65 100755
--- a/src/util/src/tarray.c
+++ b/src/util/src/tarray.c
@@ -197,30 +197,23 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
-void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) {
-  assert(pArray != NULL);
-  assert(compar != NULL);
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) {
+  assert(pArray != NULL && comparFn != NULL);
   assert(key != NULL);
 
-  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar);
+  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-static int taosArrayCompareString(const void* a, const void* b) {
-  const char* x = *(const char**)a;
-  const char* y = *(const char**)b;
-  return strcmp(x, y);
-}
-
-void taosArraySortString(SArray* pArray) {
+void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
   assert(pArray != NULL);
-  qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
+  qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-char* taosArraySearchString(const SArray* pArray, const char* key) {
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) {
   assert(pArray != NULL);
   assert(key != NULL);
 
-  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
+  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
   if (p == NULL) {
     return NULL;
   }
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index 751bd0fc34..cb9f339f6a 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -227,9 +227,16 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
+int32_t taosArrayCompareString(const void* a, const void* b) {
+  const char* x = *(const char**)a;
+  const char* y = *(const char**)b;
+  
+  return compareLenPrefixedStr(x, y);
+}
+
 static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
   const SArray* arr = (const SArray*) pRight;
-  return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1;
+  return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1;
 }
 
 static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
@@ -248,25 +255,25 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
   
   switch (type) {
     case TSDB_DATA_TYPE_SMALLINT: {
-      comparFn = compareInt16Val; break;
+      comparFn = compareInt16Val;  break;
     }
     
     case TSDB_DATA_TYPE_INT: {
-      comparFn = compareInt32Val; break;
+      comparFn = compareInt32Val;  break;
     }
     
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      comparFn = compareInt64Val; break;
+      comparFn = compareInt64Val;  break;
     }
 
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:{
-      comparFn = compareInt8Val; break;
+      comparFn = compareInt8Val;   break;
     }
 
     case TSDB_DATA_TYPE_FLOAT: {
-      comparFn = compareFloatVal; break;
+      comparFn = compareFloatVal;  break;
     }
     
     case TSDB_DATA_TYPE_DOUBLE: {

From c44bf0ff17afa33a4c7bb08648564d3e245d4483 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 17:36:19 +0800
Subject: [PATCH 07/18] [td-225] update the script

---
 tests/script/general/parser/testSuite.sim | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index 3d5f33842e..ac867c9f7f 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -56,15 +56,17 @@ sleep 2000
 run general/parser/limit1_tblocks100.sim
 sleep 2000
 run general/parser/select_across_vnodes.sim
+sleep 2000
+run general/parser/slimit1.sim
+sleep 2000
+run general/parser/tbnameIn.sim
+sleep 2000
+run general/parser/projection_limit_offset.sim
 
 sleep 2000
 run general/parser/limit2.sim
 sleep 2000
-run general/parser/tbnameIn.sim
-sleep 2000
 run general/parser/slimit.sim
-sleep 2000
-run general/parser/slimit1.sim
 
 sleep 2000
 run general/parser/fill.sim
@@ -94,8 +96,7 @@ sleep 2000
 run general/parser/join.sim
 sleep 2000
 run general/parser/join_multivnode.sim
-sleep 2000
-run general/parser/projection_limit_offset.sim
+
 sleep 2000
 run general/parser/select_with_tags.sim
 sleep 2000

From 4737c8cd44765bd5330f53e5cd3357004d4e852d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 17:36:38 +0800
Subject: [PATCH 08/18] [td-225]

---
 tests/script/jenkins/basic.txt | 44 +++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 5d76e69e10..e68707e710 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -109,9 +109,9 @@ cd ../../../debug; make
 ./test.sh -f general/parser/auto_create_tb.sim
 ./test.sh -f general/parser/auto_create_tb_drop_tb.sim
 ./test.sh -f general/parser/col_arithmetic_operation.sim
-#/test.sh -f general/parser/columnValue.sim
+./test.sh -f general/parser/columnValue.sim
 ./test.sh -f general/parser/commit.sim
-# ./test.sh -f general/parser/create_db.sim
+./test.sh -f general/parser/create_db.sim
 ./test.sh -f general/parser/create_mt.sim
 ./test.sh -f general/parser/create_tb.sim
 ./test.sh -f general/parser/dbtbnameValidate.sim
@@ -121,39 +121,39 @@ cd ../../../debug; make
 ./test.sh -f general/parser/insert_tb.sim
 ./test.sh -f general/parser/first_last.sim
 # ./test.sh -f general/parser/import_file.sim
-# ./test.sh -f general/parser/lastrow.sim
-# ./test.sh -f general/parser/nchar.sim
-# ./test.sh -f general/parser/null_char.sim
-# ./test.sh -f general/parser/single_row_in_tb.sim
+./test.sh -f general/parser/lastrow.sim
+./test.sh -f general/parser/nchar.sim
+./test.sh -f general/parser/null_char.sim
+./test.sh -f general/parser/single_row_in_tb.sim
 ./test.sh -f general/parser/select_from_cache_disk.sim
-# ./test.sh -f general/parser/limit.sim
+./test.sh -f general/parser/limit.sim
+./test.sh -f general/parser/limit1.sim
+./test.sh -f general/parser/limit1_tblocks100.sim
+./test.sh -f general/parser/mixed_blocks.sim
+./test.sh -f general/parser/selectResNum.sim
+./test.sh -f general/parser/select_across_vnodes.sim
+./test.sh -f general/parser/slimit1.sim
+./test.sh -f general/parser/tbnameIn.sim
+./test.sh -f general/parser/binary_escapeCharacter.sim
+# ./test.sh -f general/parser/limit2.sim
+# ./test.sh -f general/parser/set_tag_vals.sim
+# ./test.sh -f general/parser/slimit.sim
 # ./test.sh -f general/parser/fill.sim
 # ./test.sh -f general/parser/fill_stb.sim
 # ./test.sh -f general/parser/tags_dynamically_specifiy.sim
 # ./test.sh -f general/parser/interp.sim
-# ./test.sh -f general/parser/limit1.sim
-# ./test.sh -f general/parser/limit1_tblocks100.sim
-# ./test.sh -f general/parser/limit2.sim
-# ./test.sh -f general/parser/mixed_blocks.sim
-./test.sh -f general/parser/selectResNum.sim
-# ./test.sh -f general/parser/select_across_vnodes.sim
-# ./test.sh -f general/parser/set_tag_vals.sim
-# ./test.sh -f general/parser/slimit.sim
-./test.sh -f general/parser/slimit1.sim
-#unsupport ./test.sh -f general/parser/slimit_alter_tags.sim
-#unsupport ./test.sh -f general/parser/stream_on_sys.sim
-#unsupport ./test.sh -f general/parser/stream.sim
-# ./test.sh -f general/parser/tbnameIn.sim
 # ./test.sh -f general/parser/where.sim
 # ./test.sh -f general/parser/repeatAlter.sim
-#unsupport ./test.sh -f general/parser/repeatStream.sim
 # ./test.sh -f general/parser/join.sim
 # ./test.sh -f general/parser/join_multivnode.sim
 # ./test.sh -f general/parser/projection_limit_offset.sim
 # ./test.sh -f general/parser/select_with_tags.sim
 # ./test.sh -f general/parser/groupby.sim
-./test.sh -f general/parser/binary_escapeCharacter.sim
 #./test.sh -f general/parser/bug.sim
+#unsupport ./test.sh -f general/parser/slimit_alter_tags.sim
+#unsupport ./test.sh -f general/parser/stream_on_sys.sim
+#unsupport ./test.sh -f general/parser/stream.sim
+#unsupport ./test.sh -f general/parser/repeatStream.sim
 
 ./test.sh -f general/stable/disk.sim
 ./test.sh -f general/stable/dnode3.sim

From 5bc556381609d957326dc3ba8cde1352fc5c8cf1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 18:12:51 +0800
Subject: [PATCH 09/18] [td-225]

---
 .../general/parser/projection_limit_offset.sim  | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim
index 52be50c032..0091bfa59e 100644
--- a/tests/script/general/parser/projection_limit_offset.sim
+++ b/tests/script/general/parser/projection_limit_offset.sim
@@ -133,7 +133,7 @@ if $rows != 4007 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.499@ then
+if $data00 != @70-01-01 08:01:43.500@ then
    return -1
 endi
 
@@ -143,7 +143,7 @@ if $rows != 3907 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.399@ then
+if $data00 != @70-01-01 08:01:43.488@ then
    return -1
 endi
 
@@ -152,7 +152,7 @@ if $rows != 3106 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.099@ then
+if $data00 != @70-01-01 08:01:43.388@ then
    return -1
 endi
 
@@ -161,7 +161,7 @@ if $rows != 3608 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:01:43.100@ then
+if $data00 != @70-01-01 08:01:43.450@ then
    return -1
 endi
 
@@ -344,11 +344,11 @@ if $data11 != 1 then
    return -1
 endi
 
-if $data12 != NULL then
+if $data12 != null then
    return -1
 endi
 
-if $data13 != NULL then
+if $data13 != null then
    return -1
 endi
 
@@ -371,8 +371,11 @@ sql_error select 1 interval(1h);
 sql_error select count(*);
 sql_error select sum(k);
 sql_error select 'abc';
+sql_error select k+1,sum(k) from tm0;
+sql_error select k, sum(k) from tm0;
+sql_error select k, sum(k)+1 from tm0;
 
 #=============================tbase-1205
 sql select count(*) from tm1 where ts<now and ts>= now -1d interval(1h) fill(NULL);
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+

From afc8b5a3cd00bc3aa8ffa6f05b472dcb61b8aaba Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 23 May 2020 18:27:29 +0800
Subject: [PATCH 10/18] [td-225]refactor codes

---
 src/client/src/tscFunctionImpl.c | 16 ++++++----------
 src/query/inc/tsqlfunction.h     |  2 --
 src/query/src/qExecutor.c        |  5 +++--
 3 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 168863d322..fff8823a83 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -396,10 +396,6 @@ static void function_finalizer(SQLFunctionCtx *pCtx) {
   doFinalizer(pCtx);
 }
 
-static bool usePreVal(SQLFunctionCtx *pCtx) {
-  return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size;
-}
-
 /*
  * count function does need the finalize, if data is missing, the default value, which is 0, is used
  * count function does not use the pCtx->interResBuf to keep the intermediate buffer
@@ -412,7 +408,7 @@ static void count_function(SQLFunctionCtx *pCtx) {
    * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true;
    * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false;
    */
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull;
   } else {
     if (pCtx->hasNull) {
@@ -537,7 +533,7 @@ static void do_sum(SQLFunctionCtx *pCtx) {
   int32_t notNullElems = 0;
   
   // Only the pre-computing information loaded and actual data does not loaded
-  if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) {
+  if (pCtx->preAggVals.isSet) {
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull);
     
@@ -768,7 +764,7 @@ static void avg_function(SQLFunctionCtx *pCtx) {
   SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
   double *  pVal = &pAvgInfo->sum;
   
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     // Pre-aggregation
     notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(notNullElems >= 0);
@@ -932,7 +928,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) {
 
 static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) {
   // data in current data block are qualified to the query
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     assert(*notNullElems >= 0);
     
@@ -2916,7 +2912,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == REVERSE_SCAN) {
+  if (pCtx->scanFlag == REVERSE_SCAN) {  // todo : remove it
     return;
   }
   
@@ -3423,7 +3419,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
   
   // todo : opt with pre-calculated result
   // column missing cause the hasNull to be true
-  if (usePreVal(pCtx)) {
+  if (pCtx->preAggVals.isSet) {
     numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
     
     // all data are null in current data block, ignore current data block
diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index a11c03f2c0..6e591b28d2 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -126,7 +126,6 @@ typedef struct SArithmeticSupport {
 
 typedef struct SQLPreAggVal {
   bool    isSet;
-  int32_t size;
   SDataStatis statis;
 } SQLPreAggVal;
 
@@ -174,7 +173,6 @@ typedef struct SQLFunctionCtx {
   int16_t  outputBytes;  // size of results, determined by function and input column data type
   bool     hasNull;      // null value exist in current block
   int16_t  functionId;   // function id
-  int32_t  blockStatus;  // Indicate if data is loaded, it is first/last/internal block. Only for file blocks
   void *   aInputElemBuf;
   char *   aOutputBuf;            // final result output buffer, point to sdata->data
   uint8_t  currentStage;          // record current running step, default: 0
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index e9f1ffb49f..5ef2f128ad 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1342,14 +1342,13 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
 
   if (pStatis != NULL) {
     pCtx->preAggVals.isSet = true;
-    pCtx->preAggVals.size = size;
     pCtx->preAggVals.statis = *pStatis;
   } else {
     pCtx->preAggVals.isSet = false;
   }
 
   pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0;
-  pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1;
+  pCtx->size = size;
 
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
@@ -1377,6 +1376,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
 
   } else if (functionId == TSDB_FUNC_ARITHM) {
     pCtx->param[1].pz = param;
+  } else if (functionId == TSDB_FUNC_SPREAD) {
+    pCtx->
   }
 
 #if defined(_DEBUG_VIEW)

From a585b1d9be7f8b98f0d8f7f26d4c54ce6cd1cbc3 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 24 May 2020 20:58:54 +0800
Subject: [PATCH 11/18] [TD-413] dnode may be null if mnode list changed

---
 src/mnode/inc/mgmtDef.h   |  1 -
 src/mnode/src/mgmtMnode.c | 40 +++++++++++++++++++++++++--------------
 src/mnode/src/mgmtSdb.c   | 12 +++++++++---
 3 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h
index 58d16ce1b3..9d3e46205d 100644
--- a/src/mnode/inc/mgmtDef.h
+++ b/src/mnode/inc/mgmtDef.h
@@ -63,7 +63,6 @@ typedef struct SMnodeObj {
   int8_t     updateEnd[1];
   int32_t    refCount;
   int8_t     role;
-  SDnodeObj *pDnode;
 } SMnodeObj;
 
 typedef struct {
diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c
index e9d14dc6e7..6471b7f182 100644
--- a/src/mnode/src/mgmtMnode.c
+++ b/src/mnode/src/mgmtMnode.c
@@ -65,7 +65,6 @@ static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
   SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
 
-  pMnode->pDnode = pDnode;
   pDnode->isMgmt = true;
   mgmtDecDnodeRef(pDnode);
   
@@ -220,22 +219,27 @@ void mgmtUpdateMnodeIpSet() {
     pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
-    strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
-    ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
+    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+    if (pDnode != NULL) {
+      strcpy(ipSet->fqdn[ipSet->numOfIps], pDnode->dnodeFqdn);
+      ipSet->port[ipSet->numOfIps] = htons(pDnode->dnodePort);
 
-    mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
-    strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
+      mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
+      strcpy(mnodes->nodeInfos[index].nodeEp, pDnode->dnodeEp);
 
-    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
-      ipSet->inUse = ipSet->numOfIps;
-      mnodes->inUse = index;
+      if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+        ipSet->inUse = ipSet->numOfIps;
+        mnodes->inUse = index;
+      }
+
+      mPrint("mnode:%d, ep:%s %s", index, pDnode->dnodeEp,
+             pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
+
+      ipSet->numOfIps++;
+      index++;
     }
 
-    mPrint("mnode:%d, ep:%s %s", index, pMnode->pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
-
-    ipSet->numOfIps++;
-    index++;
-    
+    mgmtDecDnodeRef(pDnode);
     mgmtDecMnodeRef(pMnode);
   }
 
@@ -385,7 +389,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    
+    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+    if (pDnode != NULL) {
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    } else {
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
+    }
+    mgmtDecDnodeRef(pDnode);
+
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c
index 3d4e6fcab1..237d2ca499 100644
--- a/src/mnode/src/mgmtSdb.c
+++ b/src/mnode/src/mgmtSdb.c
@@ -28,6 +28,7 @@
 #include "mgmtDef.h"
 #include "mgmtInt.h"
 #include "mgmtMnode.h"
+#include "mgmtDnode.h"
 #include "mgmtSdb.h"
 
 typedef enum {
@@ -259,10 +260,15 @@ void sdbUpdateSync() {
       if (pMnode == NULL) break;
 
       syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
-      syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
-      strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
-      index++;
 
+      SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+      if (pDnode != NULL) {
+        syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC;
+        strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp);
+        index++;
+      }
+
+      mgmtDecDnodeRef(pDnode);
       mgmtDecMnodeRef(pMnode);
     }
     sdbFreeIter(pIter);

From 54d0511ea57d52803763a8239055c09b65a35cb5 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 24 May 2020 21:07:24 +0800
Subject: [PATCH 12/18] script

---
 tests/script/tmp/prepare.sim | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim
index 31d7839566..1db643c5c9 100644
--- a/tests/script/tmp/prepare.sim
+++ b/tests/script/tmp/prepare.sim
@@ -1,14 +1,4 @@
 system sh/stop_dnodes.sh
-system sh/deploy.sh -n dnode1 -i 1
-system sh/deploy.sh -n dnode2 -i 2
-system sh/deploy.sh -n dnode3 -i 3
-
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 2
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2
-
-return
-system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
@@ -33,4 +23,8 @@ system sh/cfg.sh -n dnode4 -c mgmtEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
\ No newline at end of file
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode2 -c http -v 1
+system sh/cfg.sh -n dnode3 -c http -v 1
\ No newline at end of file

From be671585380a4af0154170521589d07d3d252b74 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 24 May 2020 22:40:07 +0800
Subject: [PATCH 13/18] [td-225] update the sim script

---
 src/query/src/qExecutor.c                     | 76 ++++++++++---------
 tests/script/general/parser/create_db.sim     |  4 +-
 .../general/parser/first_last_query.sim       |  7 --
 tests/script/general/parser/null_char.sim     | 12 ++-
 .../parser/projection_limit_offset.sim        | 13 +++-
 5 files changed, 63 insertions(+), 49 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 5ef2f128ad..0609ed1870 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -110,7 +110,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf
 static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
 static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow);
 
-static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
+static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* pData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
                           int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag);
 static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols);
@@ -733,7 +733,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
 }
 
 static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin,
-                                      int32_t startPos, int32_t forwardStep, TSKEY *tsBuf) {
+                                      int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
@@ -743,12 +743,17 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat
 
       pCtx[k].nStartQueryTimestamp = pWin->skey;
       pCtx[k].size = forwardStep;
-      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1);
+      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
 
       if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-        pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
+        pCtx[k].ptsList = tsBuf;
       }
 
+      // not a whole block involved in query processing, statistics data can not be used
+      if (forwardStep != numOfTotal) {
+        pCtx[k].preAggVals.isSet = false;
+      }
+      
       if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
         aAggs[functionId].xFunction(&pCtx[k]);
       }
@@ -890,7 +895,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
       }
       
       assert(dataBlock != NULL);
-      sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes;  // start from the offset
+      sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/;  // start from the offset
     }
 
   } else {  // other type of query function
@@ -933,14 +938,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
 
     SDataStatis *tpField = NULL;
 
     bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, tpField, hasNull,
+                  &sasArray[k], colId);
   }
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@@ -958,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
         getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true);
 
     SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol);
+    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
 
     int32_t     index = pWindowResInfo->curIndex;
     STimeWindow nextWin = win;
@@ -978,7 +984,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
       forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true);
 
       pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol);
+      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
     }
 
     pWindowResInfo->curIndex = index;
@@ -1154,14 +1160,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
-
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
+    
     SDataStatis *pColStatis = NULL;
 
     bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, pColStatis, hasNull,
+                  &sasArray[k], colId);
   }
 
   // set the input column data
@@ -1214,7 +1221,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       }
 
       // all startOffset are identical
-      offset -= pCtx[0].startOffset;
+//      offset -= pCtx[0].startOffset;
 
       SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
       doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset);
@@ -1255,9 +1262,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
       }
 
-      // all startOffset are identical
-      offset -= pCtx[0].startOffset;
-
       for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
         int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
         if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
@@ -1333,26 +1337,28 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
   return numOfRes;
 }
 
-void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
-                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) {
-  pCtx->scanFlag = scanFlag;
-
+void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
+                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t colId) {
+  pCtx->hasNull  = hasNull;
   pCtx->aInputElemBuf = inputData;
-  pCtx->hasNull = hasNull;
 
   if (pStatis != NULL) {
-    pCtx->preAggVals.isSet = true;
+    pCtx->preAggVals.isSet  = true;
     pCtx->preAggVals.statis = *pStatis;
+    if (pCtx->preAggVals.statis.numOfNull == -1) {
+      pCtx->preAggVals.statis.numOfNull = pBlockInfo->rows;  // todo :can not be -1
+    }
   } else {
     pCtx->preAggVals.isSet = false;
   }
 
-  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0;
-  pCtx->size = size;
+  // limit/offset query will affect this value
+  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
+  pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
 
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
-    pCtx->ptsList = &tsCol[pCtx->startOffset];
+    pCtx->ptsList = tsCol;
   }
 
   if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {
@@ -1362,7 +1368,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
   } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA ||
              functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
     /*
-     * leastsquares function needs two columns of input, currently, the x value of linear equation is set to
+     * least squares function needs two columns of input, currently, the x value of linear equation is set to
      * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer
      *
      * top/bottom function needs timestamp to indicate when the
@@ -1376,8 +1382,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
 
   } else if (functionId == TSDB_FUNC_ARITHM) {
     pCtx->param[1].pz = param;
-  } else if (functionId == TSDB_FUNC_SPREAD) {
-    pCtx->
+  } else if (functionId == TSDB_FUNC_SPREAD) {  // set the statistics data for primary time stamp column
+    if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      pCtx->preAggVals.isSet  = true;
+      pCtx->preAggVals.statis.min = pBlockInfo->window.skey;
+      pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
+    }
   }
 
 #if defined(_DEBUG_VIEW)
@@ -3634,12 +3644,14 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     STimeWindow     w = {0};
     SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
 
-    getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
+    TSKEY sk = MIN(win.skey, win.ekey);
+    TSKEY ek = MAX(win.skey, win.ekey);
+    getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w);
     pWindowResInfo->startTime = pTableQueryInfo->win.skey;  // windowSKey may be 0 in case of 1970 timestamp
 
     if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
       if (!QUERY_IS_ASC_QUERY(pQuery)) {
-        assert(win.ekey == pQuery->window.skey);
+        assert(win.ekey == pQuery->window.ekey);
       }
       
       pWindowResInfo->prevSKey = w.skey;
@@ -3674,10 +3686,6 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
   return loadPrimaryTS;
 }
 
-bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) {
-  return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery)));
-}
-
 static int32_t getNumOfSubset(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim
index 817d712aa6..7b08d942fd 100644
--- a/tests/script/general/parser/create_db.sim
+++ b/tests/script/general/parser/create_db.sim
@@ -104,14 +104,14 @@ $replica = 1 # max=3
 $days = 10
 $keep = 365
 $rows_db = 1000
-$cache = 4096 # 4 kb
+$cache = 16 # 16MB
 $ablocks = 100 
 $tblocks = 32 # max=512, automatically trimmed when exceeding
 $ctime = 36000 # 10 hours
 $wal = 0 # valid value is 0, 1, 2
 $comp = 1 # max=32, automatically trimmed when exceeding
 
-sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ablocks $ablocks tblocks $tblocks ctime $ctime wal $wal comp $comp
+sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ctime $ctime wal $wal comp $comp
 sql show databases
 if $rows != 1 then
   return -1
diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim
index c6237198c5..fa5ed8b4ad 100644
--- a/tests/script/general/parser/first_last_query.sim
+++ b/tests/script/general/parser/first_last_query.sim
@@ -27,32 +27,25 @@ endi
 if $data00 != @18-09-17 08:59:00.000@ then
   return -1
 endi
-#if $data01 != NULL then
 if $data01 != 0 then
   return -1   
 endi  
-#if $data02 != NULL then
 if $data02 != 0 then
   return -1  
 endi 
-#if $data03 != NULL then
 print data03 = $data03
 if $data03 != 0.00000 then
   return -1  
 endi    
-#if $data04 != NULL then
 if $data04 != 0.000000000 then
   return -1
 endi
-#if $data05 != NULL then
 if $data05 != 0 then
   return -1
 endi 
-#if $data06 != NULL then
 if $data06 != 0 then
   return -1
 endi
-#if $data07 != NULL then
 if $data07 != 1 then
   return -1
 endi
diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim
index 7a6c40c1a3..6da419cd4c 100644
--- a/tests/script/general/parser/null_char.sim
+++ b/tests/script/general/parser/null_char.sim
@@ -177,7 +177,7 @@ sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL)   values
 #### case 3: set tag value
 sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double)
 sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789)
-sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double st41
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
 if $rows != 1 then
   return -1
 endi
@@ -190,13 +190,17 @@ endi
 if $data02 != 100 then
   return -1
 endi
-if $data03 != false then
+if $data03 != 0 then
   return -1
 endi
-if $dat04 != 9.123450 then
+
+if $data04 != 9.12345 then
+  print expect 9.12345 , actual: $data04
   return -1
 endi
-if $data05 != 7.123457 then
+
+if $data05 != 7.123456789 then
+  print expect 7.123456789 , actual: $data05
   return -1
 endi
 
diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim
index 0091bfa59e..5f006d0eb7 100644
--- a/tests/script/general/parser/projection_limit_offset.sim
+++ b/tests/script/general/parser/projection_limit_offset.sim
@@ -344,11 +344,11 @@ if $data11 != 1 then
    return -1
 endi
 
-if $data12 != null then
+if $data12 != NULL then
    return -1
 endi
 
-if $data13 != null then
+if $data13 != NULL then
    return -1
 endi
 
@@ -358,6 +358,15 @@ if $row != 8 then
    return -1
 endi
 
+sql select diff(k) from tm0
+if $row != 3 then
+   return -1
+endi
+
+if $data21 != -1 then
+   return -1
+endi
+
 #error sql
 sql_error select * from 1;
 sql_error select 1;

From c4fe461f72f95846a539ca12bac3e3d0c139380a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 24 May 2020 22:41:01 +0800
Subject: [PATCH 14/18] [td-225] fix bugs in diff and spread

---
 src/client/src/tscFunctionImpl.c | 26 ++++++++------------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index fff8823a83..52d904d314 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2912,10 +2912,6 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 static void date_col_output_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->scanFlag == REVERSE_SCAN) {  // todo : remove it
-    return;
-  }
-  
   SET_VAL(pCtx, pCtx->size, 1);
   *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp;
 }
@@ -3080,7 +3076,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;  // direct previous may be null
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3112,7 +3108,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3143,7 +3139,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           pOutput += 1;
           pTimestamp += 1;
@@ -3174,7 +3170,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3206,7 +3202,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3238,7 +3234,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
           pOutput += 1;
           pTimestamp += 1;
         } else {
-          *pOutput = pData[i] - pData[i - step];
+          *pOutput = pData[i] - pCtx->param[1].i64Key;
           *pTimestamp = pCtx->ptsList[i];
           
           pOutput += 1;
@@ -3445,14 +3441,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
         pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max));
       }
     }
-  } else {
-//    if (pInfo->min > pCtx->param[1].dKey) {
-//      pInfo->min = pCtx->param[1].dKey;
-//    }
-//
-//    if (pInfo->max < pCtx->param[2].dKey) {
-//      pInfo->max = pCtx->param[2].dKey;
-//    }
+    
+    goto _spread_over;
   }
   
   void *pData = GET_INPUT_CHAR(pCtx);

From f8564866c90358b51559a6b24a2ffcfe5717288c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 24 May 2020 22:41:29 +0800
Subject: [PATCH 15/18] [td-225] fix bugs in data block statistics generation

---
 src/common/src/ttypes.c | 39 +++++++++------------------------------
 1 file changed, 9 insertions(+), 30 deletions(-)

diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c
index 265e63222c..d45ceb2a4b 100644
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@@ -42,9 +42,6 @@ static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t nu
   
   ASSERT(numOfRow <= INT16_MAX);
   
-  //  int64_t lastKey = 0;
-  //  int8_t  lastVal = TSDB_DATA_TINYINT_NULL;
-  
   for (int32_t i = 0; i < numOfRow; ++i) {
     if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) {
       (*numOfNull) += 1;
@@ -213,15 +210,6 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
       fmax = fv;
       *maxIndex = i;
     }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
   }
   
   double csum = 0;
@@ -232,9 +220,9 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
   SET_DOUBLE_VAL_ALIGN(max, &fmax);
   SET_DOUBLE_VAL_ALIGN(min, &fmin);
 #else
-  *sum = csum;
-  *max = fmax;
-  *min = fmin;
+  *(double*)sum = csum;
+  *(double*)max = fmax;
+  *(double*)min = fmin;
 #endif
 }
 
@@ -267,15 +255,6 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
       dmax = dv;
       *maxIndex = i;
     }
-    
-    //    if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    } else {
-    //      *wsum = lastVal * (primaryKey[i] - lastKey);
-    //      lastKey = primaryKey[i];
-    //      lastVal = data[i];
-    //    }
   }
   
   double csum = 0;
@@ -285,18 +264,18 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
 
 #ifdef _TD_ARM_32_
   SET_DOUBLE_VAL_ALIGN(sum, &csum);
-    SET_DOUBLE_VAL_ALIGN(max, &dmax);
-    SET_DOUBLE_VAL_ALIGN(min, &dmin);
+  SET_DOUBLE_VAL_ALIGN(max, &dmax);
+  SET_DOUBLE_VAL_ALIGN(min, &dmin);
 #else
-  *sum = csum;
-  *max = dmax;
-  *min = dmin;
+  *(double*) sum = csum;
+  *(double*) max = dmax;
+  *(double*) min = dmin;
 #endif
 }
 
 tDataTypeDescriptor tDataTypeDesc[11] = {
   {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL,                  NULL},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      NULL},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool,      getStatics_i8},
   {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
   {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
   {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt,       getStatics_i32},

From 5f3fce3577a9ccf1e5ee2a067c72004b9d2210c3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 24 May 2020 22:44:21 +0800
Subject: [PATCH 16/18] [td-225]fix crash in parsing the following sql: create
 table t1 as select count(*);

---
 src/client/src/tscSQLParser.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index f338bbc644..d23d0e1860 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5337,13 +5337,6 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
     }
   }
 
-  SColumnIndex ind = {0};
-  SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
-      tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
-  
-  const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
-  strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
-
   switch (index) {
     case 0:
       pQueryInfo->command = TSDB_SQL_CURRENT_DB;
@@ -5362,6 +5355,13 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
       return TSDB_CODE_SUCCESS;
     default: { return invalidSqlErrMsg(pQueryInfo->msg, msg3); }
   }
+  
+  SColumnIndex ind = {0};
+  SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
+                                      tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
+  
+  const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
+  strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
 }
 
 // can only perform the parameters based on the macro definitation

From 569bde3044286181576988e04ec3c22d2497ae41 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 24 May 2020 22:54:06 +0800
Subject: [PATCH 17/18] [td-225]update the basic.txt

---
 tests/script/jenkins/basic.txt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index e68707e710..dc8c564fd9 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -111,7 +111,7 @@ cd ../../../debug; make
 ./test.sh -f general/parser/col_arithmetic_operation.sim
 ./test.sh -f general/parser/columnValue.sim
 ./test.sh -f general/parser/commit.sim
-./test.sh -f general/parser/create_db.sim
+#./test.sh -f general/parser/create_db.sim   #there are bugs in this sim script
 ./test.sh -f general/parser/create_mt.sim
 ./test.sh -f general/parser/create_tb.sim
 ./test.sh -f general/parser/dbtbnameValidate.sim
@@ -135,21 +135,21 @@ cd ../../../debug; make
 ./test.sh -f general/parser/slimit1.sim
 ./test.sh -f general/parser/tbnameIn.sim
 ./test.sh -f general/parser/binary_escapeCharacter.sim
+./test.sh -f general/parser/projection_limit_offset.sim
 # ./test.sh -f general/parser/limit2.sim
-# ./test.sh -f general/parser/set_tag_vals.sim
 # ./test.sh -f general/parser/slimit.sim
 # ./test.sh -f general/parser/fill.sim
 # ./test.sh -f general/parser/fill_stb.sim
-# ./test.sh -f general/parser/tags_dynamically_specifiy.sim
 # ./test.sh -f general/parser/interp.sim
 # ./test.sh -f general/parser/where.sim
-# ./test.sh -f general/parser/repeatAlter.sim
 # ./test.sh -f general/parser/join.sim
 # ./test.sh -f general/parser/join_multivnode.sim
-# ./test.sh -f general/parser/projection_limit_offset.sim
 # ./test.sh -f general/parser/select_with_tags.sim
 # ./test.sh -f general/parser/groupby.sim
-#./test.sh -f general/parser/bug.sim
+# ./test.sh -f general/parser/bug.sim
+#unsupport ./test.sh -f general/parser/tags_dynamically_specifiy.sim
+#unsupport ./test.sh -f general/parser/set_tag_vals.sim
+#unsupport ./test.sh -f general/parser/repeatAlter.sim
 #unsupport ./test.sh -f general/parser/slimit_alter_tags.sim
 #unsupport ./test.sh -f general/parser/stream_on_sys.sim
 #unsupport ./test.sh -f general/parser/stream.sim

From 77ef030ef1d8e6cf780b79763736d64b24714e44 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Mon, 25 May 2020 01:41:05 +0000
Subject: [PATCH 18/18] getFileInfo bug

---
 src/common/src/tglobal.c | 2 +-
 src/tsdb/src/tsdbMain.c  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 250b79febe..faf15c4215 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -142,7 +142,7 @@ int32_t rpcDebugFlag = 135;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
 int32_t sDebugFlag = 135;
-int32_t tsdbDebugFlag = 131;
+int32_t tsdbDebugFlag = 135;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index a9f0d9b6ec..7db61db429 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -1196,7 +1196,7 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *
     // Map index to the file name
     int fid = (*index) / 3;
 
-    if (fid > pFileH->numOfFGroups) {
+    if (fid >= pFileH->numOfFGroups) {
       // return meta data file
       if ((*index) % 3 > 0) { // it is finished
         tfree(spath);