From 65fe54c4f62e9865580d1b4657b94421f2d7aee5 Mon Sep 17 00:00:00 2001
From: dengyihao <luomoxyz@126.com>
Date: Fri, 12 Jun 2020 19:39:17 +0800
Subject: [PATCH 01/25] fixbug TD-327

---
 src/client/src/tscParseInsert.c |  2 +-
 src/client/src/tscSQLParser.c   |  4 ++--
 src/common/inc/tglobal.h        |  1 +
 src/common/src/tglobal.c        |  1 +
 src/common/src/ttimezone.c      |  1 +
 src/util/inc/ttime.h            |  2 +-
 src/util/src/ttime.c            | 40 ++++++++++++++++++++++++++++-----
 7 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index c4749e4611..afd1a3e8c8 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -97,7 +97,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
     useconds = str2int64(pToken->z);
   } else {
     // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
-    if (taosParseTime(pToken->z, time, pToken->n, timePrec) != TSDB_CODE_SUCCESS) {
+    if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(error, "invalid timestamp format", pToken->z);
     }
 
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 8e5a439431..7605fae5e5 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -138,7 +138,7 @@ static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVa
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
   
   if (seg != NULL) {
-    if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision) != TSDB_CODE_SUCCESS) {
+    if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg);
     }
   } else {
@@ -3950,7 +3950,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t
 
     char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false);
     if (seg != NULL) {
-      if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO) == TSDB_CODE_SUCCESS) {
+      if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) {
         parsed = true;
       } else {
         return TSDB_CODE_TSC_INVALID_SQL;
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index 53d821b3d8..eaceea468b 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -170,6 +170,7 @@ extern char gitinfo[];
 extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
+extern int8_t tsDaylight;
 extern char tsTimezone[64];
 extern char tsLocale[64];
 extern char tsCharset[64];  // default encode string
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 86fc6deb1b..e80597ca95 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -198,6 +198,7 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+int8_t tsDaylight = 0;
 char tsTimezone[64] = {0};
 char tsLocale[TSDB_LOCALE_LEN] = {0};
 char tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
diff --git a/src/common/src/ttimezone.c b/src/common/src/ttimezone.c
index 0e8e1316b3..ae6ffea59a 100644
--- a/src/common/src/ttimezone.c
+++ b/src/common/src/ttimezone.c
@@ -58,6 +58,7 @@ void tsSetTimeZone() {
   * (BST, +0100)
   */
   sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
+  tsDaylight = daylight;
 
   uPrint("timezone format changed to %s", tsTimezone);
 }
diff --git a/src/util/inc/ttime.h b/src/util/inc/ttime.h
index 61df65f345..576c9a51f6 100644
--- a/src/util/inc/ttime.h
+++ b/src/util/inc/ttime.h
@@ -56,7 +56,7 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
 
 int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts);
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec);
+int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void deltaToUtcInitOnce();
 
 #ifdef __cplusplus
diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c
index 8355ed9dc1..4dd6360752 100644
--- a/src/util/src/ttime.c
+++ b/src/util/src/ttime.c
@@ -24,7 +24,6 @@
 #include "taosdef.h"
 #include "ttime.h"
 #include "tutil.h"
-
 /*
  * mktime64 - Converts date to seconds.
  * Converts Gregorian date to seconds since 1970-01-01 00:00:00.
@@ -119,15 +118,21 @@ static int month[12] = {
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);
 static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec);
 static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
+static int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec);
+
+static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = {
+  parseLocaltime,
+  parseLocaltimeWithDst
+}; 
 
 int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec) {
+int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t daylight) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {
     return parseTimeWithTz(timestr, time, timePrec);
   } else {
-    return parseLocaltime(timestr, time, timePrec);
+    return (*parseLocaltimeFp[daylight])(timestr, time, timePrec);
   }
 }
 
@@ -304,9 +309,6 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
     return -1;
   }
 
-  /* mktime will be affected by TZ, set by using taos_options */
-  //int64_t seconds = mktime(&tm);
-  //int64_t seconds = (int64_t)user_mktime(&tm);
   int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
   
   int64_t fraction = 0;
@@ -324,6 +326,32 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) {
+  *time = 0;
+  struct tm tm = {0};
+  tm.tm_isdst = -1;
+
+  char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  if (str == NULL) {
+    return -1;
+  }
+
+  /* mktime will be affected by TZ, set by using taos_options */
+  int64_t seconds = mktime(&tm);
+  
+  int64_t fraction = 0;
+
+  if (*str == '.') {
+    /* parse the second fraction part */
+    if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) {
+      return -1;
+    }
+  }
+
+  int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : 1000000;
+  *time = factor * seconds + fraction;
+  return 0;
+}
 static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) {
   *result = val;
 

From b99b469da731d3e09ca3cdce0aef776dda46e801 Mon Sep 17 00:00:00 2001
From: dengyihao <luomoxyz@126.com>
Date: Fri, 12 Jun 2020 19:47:44 +0800
Subject: [PATCH 02/25] bugfix: double free

---
 src/kit/shell/src/shellEngine.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index f04607f6fa..7705db4b27 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -469,7 +469,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
   } while( row != NULL);
 
   fclose(fp);
-  taos_free_result(result);
   return numOfRows;
 }
 

From 6237292f4c567f9ed55b59a73f3a8a72a52b830b Mon Sep 17 00:00:00 2001
From: dengyihao <luomoxyz@126.com>
Date: Sat, 13 Jun 2020 05:40:07 +0800
Subject: [PATCH 03/25] fixbug TD-562

---
 src/kit/taosdemo/taosdemo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index bfedff0d94..4129659281 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -817,7 +817,7 @@ void queryDB(TAOS *taos, char *command) {
     i--; 
   }
 
-  if (i == 0) {
+  if (code != 0) {
     fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
     taos_free_result(pSql);
 

From f909b78c6c4dee5d3e18366b66e70d513c8473d0 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sat, 13 Jun 2020 00:18:36 +0000
Subject: [PATCH 04/25] release local resources only after all messages are
 consumed

---
 src/vnode/src/vnodeMain.c | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 67ba891f93..0882ee983d 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -321,6 +321,22 @@ void vnodeRelease(void *pVnodeRaw) {
     return;
   }
 
+  if (pVnode->tsdb)
+    tsdbCloseRepo(pVnode->tsdb, 1);
+  pVnode->tsdb = NULL;
+
+  if (pVnode->wal) 
+    walClose(pVnode->wal);
+  pVnode->wal = NULL;
+
+  if (pVnode->wqueue) 
+    dnodeFreeVnodeWqueue(pVnode->wqueue);
+  pVnode->wqueue = NULL;
+
+  if (pVnode->rqueue) 
+    dnodeFreeVnodeRqueue(pVnode->rqueue);
+  pVnode->rqueue = NULL;
+ 
   tfree(pVnode->rootDir);
 
   if (pVnode->status == TAOS_VN_STATUS_DELETING) {
@@ -411,33 +427,21 @@ void vnodeBuildStatusMsg(void *param) {
 }
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
+  // remove from hash, so new messages wont be consumed
   taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
 
+  // stop replication module
   if (pVnode->sync) {
     syncStop(pVnode->sync);
     pVnode->sync = NULL;
   }
 
-  if (pVnode->tsdb)
-    tsdbCloseRepo(pVnode->tsdb, 1);
-  pVnode->tsdb = NULL;
-
-  if (pVnode->wal) 
-    walClose(pVnode->wal);
-  pVnode->wal = NULL;
-
+  // stop continuous query
   if (pVnode->cq) 
     cqClose(pVnode->cq);
   pVnode->cq = NULL;
 
-  if (pVnode->wqueue) 
-    dnodeFreeVnodeWqueue(pVnode->wqueue);
-  pVnode->wqueue = NULL;
-
-  if (pVnode->rqueue) 
-    dnodeFreeVnodeRqueue(pVnode->rqueue);
-  pVnode->rqueue = NULL;
- 
+  // release local resources only after cutting off outside connections
   vnodeRelease(pVnode);
 }
 

From f87322f6802ffa453ce9a4f12d14f1888929b4c3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 11:06:59 +0800
Subject: [PATCH 05/25] [td-225] fix bugs in group by normal columns

---
 src/client/src/tscFunctionImpl.c              |  13 +-
 src/client/src/tscSQLParser.c                 |  27 ++-
 src/client/src/tscServer.c                    |   4 +-
 src/query/src/qExecutor.c                     | 224 +++++++++++++-----
 src/query/src/qUtil.c                         |  12 +-
 tests/script/general/parser/groupby.sim       |   2 +-
 .../script/general/parser/import_commit3.sim  |   2 +-
 7 files changed, 213 insertions(+), 71 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index a0deaa519a..457e187971 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -699,7 +699,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end,
 }
 
 static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return BLK_DATA_NO_NEEDED;
   }
   
@@ -727,7 +727,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY
 }
 
 static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return BLK_DATA_NO_NEEDED;
   }
   
@@ -1593,7 +1593,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
   if (pCtx->order == TSDB_ORDER_DESC) {
     return;
   }
@@ -1652,7 +1652,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
  *    least one data in this block that is not null.(TODO opt for this case)
  */
 static void last_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   
@@ -1681,7 +1681,6 @@ static void last_function(SQLFunctionCtx *pCtx) {
 }
 
 static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
-  assert(pCtx->order != TSDB_ORDER_ASC);
   void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
@@ -1725,7 +1724,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
    * 1. for scan data in asc order, no need to check data
    * 2. for data blocks that are not loaded, no need to check data
    */
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   
@@ -1763,7 +1762,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
    * 1. for scan data in asc order, no need to check data
    * 2. for data blocks that are not loaded, no need to check data
    */
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 60415a8d74..9db81054a9 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1452,6 +1452,13 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
   
   SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
   tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
+
+  // set reverse order scan data blocks for last query
+  if (functionID == TSDB_FUNC_LAST) {
+    pExpr->numOfParams = 1;
+    pExpr->param[0].i64Key = TSDB_ORDER_DESC;
+    pExpr->param[0].nType = TSDB_DATA_TYPE_INT;
+  }
   
   // for all queries, the timestamp column needs to be loaded
   SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
@@ -1724,6 +1731,22 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
             if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
             }
+
+            if (optr == TK_LAST) {  // todo refactor
+              SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr;
+              if (pGroupBy->numOfGroupCols > 0) {
+                for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) {
+                  SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k);
+                  if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns
+                    SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i);
+                    pExpr->numOfParams = 1;
+                    pExpr->param->i64Key = TSDB_ORDER_ASC;
+
+                    break;
+                  }
+                }
+              }
+            }
           }
         }
         
@@ -2586,9 +2609,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
 
       tscColumnListInsert(pQueryInfo->colList, &index);
       
-      SColIndex colIndex = {
-          .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId,
-      };
+      SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId };
       taosArrayPush(pGroupExpr->columnInfo, &colIndex);
       pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
 
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index cb6a736121..ce174c2473 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -430,7 +430,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
   /*
    * 1. if the subqueries are not launched or partially launched, we need to waiting the launched
    * query return to successfully free allocated resources.
-   * 2. if no any subqueries are launched yet, which means the metric query only in parse sql stage,
+   * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage,
    * set the res.code, and return.
    */
   const int64_t MAX_WAITING_TIME = 10000;  // 10 Sec.
@@ -2200,7 +2200,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
    * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache
    * instead.
    */
-  tscTrace("%p force release metermeta after drop table:%s", pSql, pTableMetaInfo->name);
+  tscTrace("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
   taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
 
   if (pTableMetaInfo->pTableMeta) {
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index aa602ed661..13b4511177 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -354,7 +354,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
                                              int16_t bytes) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  int32_t *p1 = (int32_t *)taosHashGet(pWindowResInfo->hashList, pData, bytes);
+  int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
   if (p1 != NULL) {
     pWindowResInfo->curIndex = *p1;
   } else {  // more than the capacity, reallocate the resources
@@ -919,12 +919,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
 
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
+  int64_t v = -1;
+  // not assign result buffer yet, add new result buffer
+  switch(type) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT:  v = GET_INT8_VAL(pData);  break;
+    case TSDB_DATA_TYPE_SMALLINT: v = GET_INT16_VAL(pData); break;
+    case TSDB_DATA_TYPE_INT:      v = GET_INT32_VAL(pData); break;
+    case TSDB_DATA_TYPE_BIGINT:   v = GET_INT64_VAL(pData); break;
+  }
+
+//  assert(pRuntimeEnv->windowResInfo.hashList->size <= 2);
   SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes);
   if (pWindowRes == NULL) {
     return -1;
   }
 
-  // not assign result buffer yet, add new result buffer
+  pWindowRes->window.skey = v;
+  pWindowRes->window.ekey = v;
+
   if (pWindowRes->pos.pageId == -1) {
     int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
     if (ret != 0) {
@@ -1022,12 +1035,16 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
     return false;
   }
 
-  if (functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST) {
-    return !QUERY_IS_ASC_QUERY(pQuery);
-  } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
+  if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
     return QUERY_IS_ASC_QUERY(pQuery);
   }
-  
+
+  // todo add comments
+  if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) {
+    return pCtx->param[0].i64Key == pQuery->order.order;
+//    return !QUERY_IS_ASC_QUERY(pQuery);
+  }
+
   // in the supplementary scan, only the following functions need to be executed
   if (IS_REVERSE_SCAN(pRuntimeEnv)) {
     return false;
@@ -1079,7 +1096,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
   int32_t j = 0;
   int32_t offset = -1;
-  
+
   for (j = 0; j < pDataBlockInfo->rows; ++j) {
     offset = GET_COL_DATA_POS(pQuery, j, step);
 
@@ -1478,6 +1495,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 }
 
 static bool isQueryKilled(SQInfo *pQInfo) {
+  return false;
   return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
 #if 0
   /*
@@ -1574,10 +1592,14 @@ static bool needReverseScan(SQuery *pQuery) {
       continue;
     }
 
-    if (((functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) && QUERY_IS_ASC_QUERY(pQuery)) ||
-        ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery))) {
+    if ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery)) {
       return true;
     }
+
+    if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) {
+      int32_t order = pQuery->pSelectExpr[i].base.arg->argValue.i64;
+      return order != pQuery->order.order;
+    }
   }
 
   return false;
@@ -2030,6 +2052,34 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
   return midPos;
 }
 
+static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capacity) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+
+  if (capacity < pQuery->rec.capacity) {
+    return;
+  }
+
+  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+    int32_t bytes = pQuery->pSelectExpr[i].bytes;
+    assert(bytes > 0 && capacity > 0);
+
+    char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage));
+    if (tmp == NULL) {  // todo handle the oom
+      assert(0);
+    } else {
+      pQuery->sdata[i] = (tFilePage *)tmp;
+    }
+
+    // set the pCtx output buffer position
+    pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data;
+  }
+
+  qTrace("QInfo:%p realloc output buffer to inc output buffer from: %d rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
+         pQuery->rec.capacity, capacity);
+
+  pQuery->rec.capacity = capacity;
+}
+
 static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) {
   // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
   SQuery* pQuery = pRuntimeEnv->pQuery;
@@ -2916,8 +2966,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
         pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
       }
     }
-  
-    
+
     updateNumOfResult(pRuntimeEnv, pQuery->rec.rows);
   }
 }
@@ -3054,7 +3103,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
   pQuery->window = pTableQueryInfo->win;
 }
 
-void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
+void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
   SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv);
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo *pTableQueryInfo = pQuery->current;
@@ -3496,18 +3545,32 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) {
   assert(pQuery->rec.rows <= pQuery->rec.capacity);
 }
 
-static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
+static UNUSED_FUNC void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // update the number of result for each, only update the number of rows for the corresponding window result.
   if (pQuery->intervalTime == 0) {
-    int32_t g = pTableQueryInfo->groupIndex;
-    assert(pRuntimeEnv->windowResInfo.size > 0);
 
-    SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
-    if (pWindowRes->numOfRows == 0) {
-      pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
+    for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
+      SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i];
+
+      for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+        int32_t functionId = pRuntimeEnv->pCtx[j].functionId;
+        if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
+          continue;
+        }
+
+        pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
+      }
     }
+
+//    int32_t g = pTableQueryInfo->groupIndex;
+//    assert(pRuntimeEnv->windowResInfo.size > 0);
+//
+//    SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
+//    if (pWindowRes->numOfRows == 0) {
+//      pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
+//    }
   }
 }
 
@@ -4081,21 +4144,22 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     SDataStatis *pStatis = NULL;
     SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
 
-    if (!isIntervalQuery(pQuery)) {
-      int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
-      setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
-    } else {  // interval query
-      TSKEY nextKey = blockInfo.window.skey;
-      setIntervalQueryRange(pQInfo, nextKey);
-      /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
+    if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
+      if (!isIntervalQuery(pQuery)) {
+        int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
+        setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
+      } else {  // interval query
+        TSKEY nextKey = blockInfo.window.skey;
+        setIntervalQueryRange(pQInfo, nextKey);
+        /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
+      }
     }
 
     summary->totalRows += blockInfo.rows;
     stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey);
   
     qTrace("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64,
-           GET_QINFO_ADDR(pRuntimeEnv), blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey,
-           blockInfo.rows, pQuery->current->lastKey);
+           pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey);
   }
 
   int64_t et = taosGetTimestampMs();
@@ -4220,7 +4284,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       
       // here we simply set the first table as current table
       pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
-      scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+      scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
       
       int64_t numOfRes = getNumOfResult(pRuntimeEnv);
       if (numOfRes > 0) {
@@ -4233,10 +4297,84 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       // enable execution for next table, when handling the projection query
       enableExecutionForNextTable(pRuntimeEnv);
+
+      if (pQuery->rec.rows >= pQuery->rec.capacity) {
+        setQueryStatus(pQuery, QUERY_RESBUF_FULL);
+        break;
+      }
+    }
+  } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group-by on normal columns query
+    while (pQInfo->groupIndex < numOfGroups) {
+      SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
+
+      qTrace("QInfo:%p group by normal columns group:%d, total group:%d", pQInfo, pQInfo->groupIndex, numOfGroups);
+
+      STsdbQueryCond cond = {
+          .twindow = pQuery->window,
+          .colList = pQuery->colList,
+          .order   = pQuery->order.order,
+          .numOfCols = pQuery->numOfCols,
+      };
+
+      SArray *g1 = taosArrayInit(1, POINTER_BYTES);
+      SArray *tx = taosArrayClone(group);
+      taosArrayPush(g1, &tx);
+
+      STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1};
+
+      // include only current table
+      if (pRuntimeEnv->pQueryHandle != NULL) {
+        tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
+        pRuntimeEnv->pQueryHandle = NULL;
+      }
+
+      pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+
+      SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
+      assert(taosArrayGetSize(s) >= 1);
+
+      setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
+
+      // here we simply set the first table as current table
+      scanMultiTableDataBlocks(pQInfo);
+      pQInfo->groupIndex += 1;
+
+      SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+
+        // no results generated for current group, continue to try the next group
+      if (pWindowResInfo->size <= 0) {
+        continue;
+      }
+
+      for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
+        SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
+        pStatus->closed = true;  // enable return all results for group by normal columns
+
+        SWindowResult *pResult = &pWindowResInfo->pResult[i];
+        for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+          pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
+        }
+      }
+
+      qTrace("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size,
+          pQInfo->groupIndex);
+      int32_t currentGroupIndex = pQInfo->groupIndex;
+
+      pQuery->rec.rows = 0;
+      pQInfo->groupIndex = 0;
+
+      ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size);
+      copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
+
+      pQInfo->groupIndex = currentGroupIndex;  //restore the group index
+      assert(pQuery->rec.rows == pWindowResInfo->size);
+
+      clearClosedTimeWindow(pRuntimeEnv);
+      break;
     }
   } else {
     /*
-     * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query
+     * 1. super table projection query, 2. ts-comp query
      * if the subgroup index is larger than 0, results generated by group by tbname,k is existed.
      * we need to return it to client in the first place.
      */
@@ -4283,7 +4421,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         }
       }
 
-      scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+      scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
       skipResults(pRuntimeEnv);
 
       // the limitation of output result is reached, set the query completed
@@ -4349,25 +4487,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
   }
 
-  // todo refactor
-  if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
-
-    for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-      SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
-      pStatus->closed = true;  // enable return all results for group by normal columns
-
-      SWindowResult *pResult = &pWindowResInfo->pResult[i];
-      for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
-        pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
-      }
-    }
-
-    pQInfo->groupIndex = 0;
-    pQuery->rec.rows = 0;
-    copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
-  }
-
   qTrace(
       "QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
       pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
@@ -4449,7 +4568,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
      */
     if (isIntervalQuery(pQuery)) {
       copyResToQueryResultBuf(pQInfo, pQuery);
-
 #ifdef _DEBUG_VIEW
       displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num);
 #endif
@@ -4527,7 +4645,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   
   pQuery->current = pTableInfo;  // set current query table info
   
-  scanAllDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
+  scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
   finalizeQueryResult(pRuntimeEnv);
 
   if (isQueryKilled(pQInfo)) {
@@ -4560,7 +4678,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   }
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+    scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
     finalizeQueryResult(pRuntimeEnv);
 
     if (isQueryKilled(pQInfo)) {
@@ -4607,7 +4725,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start)
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv, start);
+    scanOneTableDataBlocks(pRuntimeEnv, start);
 
     if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
       return;
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 143d86d5db..aa5550efcb 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -113,7 +113,9 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     SWindowResult *pResult = &pWindowResInfo->pResult[i];
     if (pResult->status.closed) {  // remove the window slot from hash table
-      taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
+      taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type);
+      printf("remove ============>%ld, remain size:%ld\n", pResult->window.skey, pWindowResInfo->hashList->size);
+
     } else {
       break;
     }
@@ -133,14 +135,16 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   }
   
   pWindowResInfo->size = remain;
-  
+  printf("---------------size:%ld\n", taosHashGetSize(pWindowResInfo->hashList));
   for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
     SWindowResult *pResult = &pWindowResInfo->pResult[k];
-    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
+    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey,
+        tDataTypeDesc[pWindowResInfo->type].nSize);
     
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
-    taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, TSDB_KEYSIZE, (char *)&v, sizeof(int32_t));
+    taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize,
+        (char *)&v, sizeof(int32_t));
   }
   
   pWindowResInfo->curIndex = -1;
diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim
index 513b3cbbbe..5d785a2fc3 100644
--- a/tests/script/general/parser/groupby.sim
+++ b/tests/script/general/parser/groupby.sim
@@ -355,7 +355,7 @@ if $data00 != 0 then
   return -1
 endi
 
-if $data01 != 800 then
+if $data11 != 800 then
   return -1
 endi
 
diff --git a/tests/script/general/parser/import_commit3.sim b/tests/script/general/parser/import_commit3.sim
index 916bf6d05e..99ece98278 100644
--- a/tests/script/general/parser/import_commit3.sim
+++ b/tests/script/general/parser/import_commit3.sim
@@ -25,7 +25,7 @@ step1:
 sql create database $db cache 16
 print ====== create tables
 sql use $db
-
+sql reset query cache
 $i = 0
 $ts = $ts0
 $tb = $tbPrefix . $i

From 315fa7404247750306afc9450699cac36f3cf761 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 11:58:58 +0800
Subject: [PATCH 06/25] [td-225] do not execute query if the link is already
 broken.

---
 src/inc/query.h           |  3 ++-
 src/vnode/src/vnodeRead.c | 16 +++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/inc/query.h b/src/inc/query.h
index 10ee0249b6..49ee5248f5 100644
--- a/src/inc/query.h
+++ b/src/inc/query.h
@@ -70,7 +70,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
 int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
 
 /**
- * Decide if more results will be produced or not
+ * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
+ * so it can be only called once for each retrieve
  *
  * @param qinfo
  * @return
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index d6227f4270..1e770d8d27 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -61,7 +61,7 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
 
 // notify connection(handle) that current qhandle is created, if current connection from
 // client is broken, the query needs to be killed immediately.
-static void vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
+static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
   SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
   killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
   killQueryMsg->free = htons(1);
@@ -69,7 +69,7 @@ static void vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId)
   killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
   vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle);
-  rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
+  return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
 }
 
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
@@ -106,7 +106,17 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     pRet->len = sizeof(SQueryTableRsp);
     pRet->rsp = pRsp;
 
-    vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId);
+    // current connect is broken
+    if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+      vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle);
+      pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+
+      //NOTE: there two refcount, needs to kill twice, todo refactor
+      qKillQuery(pQInfo);
+      qKillQuery(pQInfo);
+
+      return pRsp->code;
+    }
 
     vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo);
   } else {

From e65be1be0d3e627149873033aa69609488ea089a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 12:56:48 +0800
Subject: [PATCH 07/25] [td-225] fix bugs in restore from core dump

---
 src/mnode/src/mnodeDb.c    |  9 ++++++---
 src/mnode/src/mnodeTable.c | 10 ++++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 3666d61adc..69783defc7 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -84,9 +84,12 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
   mnodeDropAllChildTables(pDb);
   mnodeDropAllSuperTables(pDb);
   mnodeDropAllDbVgroups(pDb);
-  mnodeDropDbFromAcct(pAcct, pDb);
-  mnodeDecAcctRef(pAcct);
-  
+
+  if (pAcct) {
+    mnodeDropDbFromAcct(pAcct, pDb);
+    mnodeDecAcctRef(pAcct);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 478e7790fb..0ff198bf8f 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -157,10 +157,12 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
   if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
-    if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
-    mnodeRemoveTableFromStable(pTable->superTable, pTable);
-    mnodeDecTableRef(pTable->superTable);
+    if (pTable->superTable) {
+      grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
+      if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
+      mnodeRemoveTableFromStable(pTable->superTable, pTable);
+      mnodeDecTableRef(pTable->superTable);
+    }
   } else {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
     if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);

From 1c07ab90639fb6a1bb1fb25a296229bcbeeea440 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 13:10:21 +0800
Subject: [PATCH 08/25] [td-225]

---
 src/query/src/qExecutor.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 13b4511177..d4cd536adb 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1495,7 +1495,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 }
 
 static bool isQueryKilled(SQInfo *pQInfo) {
-  return false;
   return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
 #if 0
   /*

From f6a5c0eabeec2d657da851bf5ec7ac16528f8101 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 13:13:31 +0800
Subject: [PATCH 09/25] [td-225] fix bugs in group by normal columns

---
 src/query/src/qExecutor.c | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d4cd536adb..6ea73827ec 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1496,19 +1496,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
 static bool isQueryKilled(SQInfo *pQInfo) {
   return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
-#if 0
-  /*
-   * check if the queried meter is going to be deleted.
-   * if it will be deleted soon, stop current query ASAP.
-   */
-  SMeterObj *pMeterObj = pQInfo->pObj;
-  if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DROPPING)) {
-    pQInfo->killed = 1;
-    return true;
-  }
-
-  return (pQInfo->killed == 1);
-#endif
 }
 
 static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; }

From 9ed2c049f5972e30da457a688abef11c10c7d843 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 13:31:17 +0800
Subject: [PATCH 10/25] [td-225] fix bugs in group by normal columns

---
 src/query/src/qExecutor.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 6ea73827ec..a1af147b62 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -3568,7 +3568,7 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *
   SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
   pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1;
 
-  if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) {
+  if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
     rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
   } else {
     blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);

From 25a395ef8099b2297b878b8d9299279538ff40fc Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Sat, 13 Jun 2020 13:37:28 +0800
Subject: [PATCH 11/25] [TD-605]

---
 src/client/src/tscSQLParser.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 60415a8d74..d0b3d91d3b 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1990,6 +1990,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken
 
     if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) {
       columnIndex = i;
+      break;
     }
   }
 
@@ -2886,7 +2887,8 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
   SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
 
   const char* msg1 = "non binary column not support like operator";
-  const char* msg2 = "binary column not support this operator";
+  const char* msg2 = "binary column not support this operator";  
+  const char* msg3 = "bool column not support this operator";
 
   SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex);
   SColumnFilterInfo* pColFilter = NULL;
@@ -2920,6 +2922,12 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
     if (pExpr->nSQLOptr == TK_LIKE) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
     }
+    
+    if (pSchema->type == TSDB_DATA_TYPE_BOOL) {
+      if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg3);
+      }
+    }
   }
 
   pColumn->colIndex = *pIndex;
@@ -6052,6 +6060,12 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
         }
       }
     }
+
+    if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
+      if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
+        return TSDB_CODE_TSC_INVALID_SQL;
+      }
+    }
   }
   
   return TSDB_CODE_SUCCESS;

From fe2bd893c3b4921334202fac5030c6f152438e9c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 13:40:11 +0800
Subject: [PATCH 12/25] [td-225] update the tests scripts

---
 tests/script/general/parser/testSuite.sim | 64 +++++++++++------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index 4a3da2f2a0..67ef952c41 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -40,42 +40,42 @@
 #run general/parser/nchar.sim
 #sleep 2000
 ##run general/parser/null_char.sim
-#sleep 2000
-#run general/parser/single_row_in_tb.sim
-#sleep 2000
-#run general/parser/select_from_cache_disk.sim
-#sleep 2000
-#run general/parser/selectResNum.sim
-#sleep 2000
-#run general/parser/mixed_blocks.sim
-#sleep 2000
-#run general/parser/limit1.sim
-#sleep 2000
-#run general/parser/limit.sim
-#sleep 2000
-#run general/parser/limit1_tblocks100.sim
-#sleep 2000
-#run general/parser/select_across_vnodes.sim
-#sleep 2000
-#run general/parser/slimit1.sim
-#sleep 2000
-#run general/parser/tbnameIn.sim
-#sleep 2000
+sleep 2000
+run general/parser/single_row_in_tb.sim
+sleep 2000
+run general/parser/select_from_cache_disk.sim
+sleep 2000
+run general/parser/selectResNum.sim
+sleep 2000
+run general/parser/mixed_blocks.sim
+sleep 2000
+run general/parser/limit1.sim
+sleep 2000
+run general/parser/limit.sim
+sleep 2000
+run general/parser/limit1_tblocks100.sim
+sleep 2000
+run general/parser/select_across_vnodes.sim
+sleep 2000
+run general/parser/slimit1.sim
+sleep 2000
+run general/parser/tbnameIn.sim
+sleep 2000
 run general/parser/projection_limit_offset.sim
 sleep 2000
 run general/parser/limit2.sim
 sleep 2000
-#run general/parser/fill.sim
-#sleep 2000
-#run general/parser/fill_stb.sim
-#sleep 2000
-#run general/parser/where.sim
-#sleep 2000
-#run general/parser/slimit.sim
-#sleep 2000
-#run general/parser/select_with_tags.sim
-#sleep 2000
-#run general/parser/interp.sim
+run general/parser/fill.sim
+sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000
+run general/parser/where.sim
+sleep 2000
+run general/parser/slimit.sim
+sleep 2000
+run general/parser/select_with_tags.sim
+sleep 2000
+run general/parser/interp.sim
 sleep 2000
 run general/parser/tags_dynamically_specifiy.sim
 sleep 2000

From 65d580b8f77ae6065e8d808f4730a6eb3faa6772 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Sat, 13 Jun 2020 13:45:11 +0800
Subject: [PATCH 13/25] move more cases from full test to regression test.

---
 tests/script/regressionSuite.sim | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim
index 40aacd4e98..6ef4001495 100644
--- a/tests/script/regressionSuite.sim
+++ b/tests/script/regressionSuite.sim
@@ -70,9 +70,9 @@ run general/http/restful_insert.sim
 run general/http/restful_limit.sim
 run general/http/restful_full.sim
 run general/http/prepare.sim
-# run general/http/telegraf.sim
-# run general/http/grafana_bug.sim
-# run general/http/grafana.sim
+run general/http/telegraf.sim
+run general/http/grafana_bug.sim
+run general/http/grafana.sim
 run general/import/basic.sim
 run general/import/commit.sim
 run general/import/large.sim
@@ -102,32 +102,32 @@ run general/parser/import_commit1.sim
 run general/parser/import_commit2.sim
 run general/parser/import_commit3.sim
 run general/parser/insert_tb.sim
-# run general/parser/first_last.sim
+run general/parser/first_last.sim
 #unsupport run general/parser/import_file.sim
-# run general/parser/lastrow.sim
+run general/parser/lastrow.sim
 run general/parser/nchar.sim
 #unsupport run general/parser/null_char.sim
-# run general/parser/single_row_in_tb.sim
+run general/parser/single_row_in_tb.sim
 run general/parser/select_from_cache_disk.sim
-# run general/parser/limit.sim
+run general/parser/limit.sim
 # run general/parser/limit1.sim
-# run general/parser/limit1_tblocks100.sim
-# run general/parser/mixed_blocks.sim
+run general/parser/limit1_tblocks100.sim
+run general/parser/mixed_blocks.sim
 # run general/parser/selectResNum.sim
 run general/parser/select_across_vnodes.sim
 run general/parser/slimit1.sim
 run general/parser/tbnameIn.sim
 run general/parser/binary_escapeCharacter.sim
-# run general/parser/projection_limit_offset.sim
+run general/parser/projection_limit_offset.sim
 run general/parser/limit2.sim
-# run general/parser/slimit.sim
+run general/parser/slimit.sim
 run general/parser/fill.sim
-# run general/parser/fill_stb.sim
+run general/parser/fill_stb.sim
 # run general/parser/interp.sim
 # run general/parser/where.sim
 #unsupport run general/parser/join.sim
 #unsupport run general/parser/join_multivnode.sim
-# run general/parser/select_with_tags.sim
+run general/parser/select_with_tags.sim
 #unsupport run general/parser/groupby.sim
 #unsupport run general/parser/bug.sim
 #unsupport run general/parser/tags_dynamically_specifiy.sim
@@ -142,7 +142,7 @@ run general/stable/dnode3.sim
 run general/stable/metrics.sim
 run general/stable/values.sim
 run general/stable/vnode3.sim
-# run general/table/autocreate.sim
+run general/table/autocreate.sim
 run general/table/basic1.sim
 run general/table/basic2.sim
 run general/table/basic3.sim
@@ -166,7 +166,7 @@ run general/table/int.sim
 run general/table/limit.sim
 run general/table/smallint.sim
 run general/table/table_len.sim
-# run general/table/table.sim
+run general/table/table.sim
 run general/table/tinyint.sim
 run general/table/vgroup.sim
 run general/tag/3.sim
@@ -214,6 +214,7 @@ run general/vector/table_time.sim
 run general/stream/stream_1.sim
 run general/stream/stream_2.sim
 run general/stream/stream_3.sim
+run general/stream/stream_restart.sim
 run general/stream/table_1.sim
 run general/stream/metrics_1.sim
 run general/stream/table_n.sim

From 1d6edcfed8fcaaa207fe7a1a5ffa3507aa5efed2 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Sat, 13 Jun 2020 13:48:53 +0800
Subject: [PATCH 14/25] [TD-350]: sort test case for both columns and tags all
 - supporting types

---
 tests/pytest/query/querySort.py | 80 +++++++++++++++++++++++++++++++++
 tests/pytest/util/sql.py        | 35 +++++++++++++++
 2 files changed, 115 insertions(+)
 create mode 100644 tests/pytest/query/querySort.py

diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py
new file mode 100644
index 0000000000..bb806c27bc
--- /dev/null
+++ b/tests/pytest/query/querySort.py
@@ -0,0 +1,80 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())    
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.prepare()
+
+        print("======= step 1: create table and insert data =========")
+        tdLog.debug(
+                ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double,
+                tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, 
+                tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''')
+        tdSql.execute(
+                ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double,
+                tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, 
+                tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''')
+        
+        for i in range(self.rowNum):
+            tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d')" % (i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 
+            1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1))
+            for j in range(self.rowNum):
+                tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" % (i + 1, self.ts + 10 * (i + 1) + j + 1, 
+                        j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1))
+        
+
+        print("======= step 2: verify order for each column =========")        
+        # sort for timestamp in asc order 
+        tdSql.query("select * from st order by ts asc")
+        tdSql.checkColumnSorted(0, "asc")
+
+        # sort for timestamp in desc order 
+        tdSql.query("select * from st order by ts desc")
+        tdSql.checkColumnSorted(0, "desc")
+
+
+        for i in range(1, 10):            
+            tdSql.error("select * from st order by tbcol%d" % i)            
+            tdSql.error("select * from st order by tbcol%d asc" % i)                    
+            tdSql.error("select * from st order by tbcol%d desc" % i)   
+            
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d" % (i, i))        
+            tdSql.checkColumnSorted(1, "")
+
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d asc" % (i, i))        
+            tdSql.checkColumnSorted(1, "asc")
+            
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d desc" % (i, i))        
+            tdSql.checkColumnSorted(1, "desc")       
+                                           
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index ec7ac117c0..367217cd49 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -17,6 +17,7 @@ import time
 import datetime
 import inspect
 from util.log import *
+import numpy as np
 
 
 class TDSql:
@@ -196,5 +197,39 @@ class TDSql:
         tdLog.info("sql:%s, affectedRows:%d == expect:%d" %
                    (self.sql, self.affectedRows, expectAffectedRows))
 
+    def checkColumnSorted(self, col, order):
+        frame = inspect.stack()[1]
+        callerModule = inspect.getmodule(frame[0])
+        callerFilename = callerModule.__file__
+
+        if col < 0:
+            tdLog.exit(
+                "%s failed: sql:%s, col:%d is smaller than zero" %
+                (callerFilename, self.sql, col))        
+        if col > self.queryCols:
+            tdLog.exit(
+                "%s failed: sql:%s, col:%d is larger than queryCols:%d" %
+                (callerFilename, self.sql, col, self.queryCols))
+        
+        matrix = np.array(self.queryResult)
+        list = matrix[:, 0]        
+
+        if order == "" or order.upper() == "ASC": 
+            if all(sorted(list) == list):                
+                tdLog.info("sql:%s, column :%d is sorted in accending order as expected" %
+                    (self.sql, col))
+            else:
+                tdLog.exit("%s failed: sql:%s, col:%d is not sorted in accesnind order" %
+                    (callerFilename, self.sql, col))
+        elif order.upper() == "DESC":            
+            if all(sorted(list, reverse=True) == list):                
+                tdLog.info("sql:%s, column :%d is sorted in decending order as expected" %
+                    (self.sql, col))
+            else:
+                tdLog.exit("%s failed: sql:%s, col:%d is not sorted in decending order" %
+                    (callerFilename, self.sql, col)) 
+        else:
+            tdLog.exit("%s failed: sql:%s, the order provided for col:%d is not correct" %
+                    (callerFilename, self.sql, col)) 
 
 tdSql = TDSql()

From 1d79ec864063db6d6e63a361e311a425b32d061a Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Sat, 13 Jun 2020 14:00:00 +0800
Subject: [PATCH 15/25] move more pytest cases to regression test.

[TD-629]
---
 tests/pytest/regressiontest.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh
index 743a837f5e..5eefcfc926 100755
--- a/tests/pytest/regressiontest.sh
+++ b/tests/pytest/regressiontest.sh
@@ -80,8 +80,8 @@ python3 ./test.py -f import_merge/importCacheFileTO.py
 python3 ./test.py -f import_merge/importCacheFileTPO.py
 python3 ./test.py -f import_merge/importCacheFileT.py
 python3 ./test.py -f import_merge/importDataH2.py
-# python3 ./test.py -f import_merge/importDataHO2.py
-# python3 ./test.py -f import_merge/importDataHO.py
+python3 ./test.py -f import_merge/importDataHO2.py
+python3 ./test.py -f import_merge/importDataHO.py
 python3 ./test.py -f import_merge/importDataHPO.py
 python3 ./test.py -f import_merge/importDataLastHO.py
 python3 ./test.py -f import_merge/importDataLastHPO.py
@@ -92,7 +92,7 @@ python3 ./test.py -f import_merge/importDataLastTO.py
 python3 ./test.py -f import_merge/importDataLastTPO.py
 python3 ./test.py -f import_merge/importDataLastT.py
 python3 ./test.py -f import_merge/importDataS.py
-# python3 ./test.py -f import_merge/importDataSub.py
+python3 ./test.py -f import_merge/importDataSub.py
 python3 ./test.py -f import_merge/importDataTO.py
 python3 ./test.py -f import_merge/importDataTPO.py
 python3 ./test.py -f import_merge/importDataT.py

From 96d7ed762bc4f23116ed13f5a874ea6f92136cfa Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Sat, 13 Jun 2020 15:06:33 +0800
Subject: [PATCH 16/25] [TD-605]

---
 src/client/src/tscSQLParser.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index d0b3d91d3b..8cca7ed7f8 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -6061,9 +6061,13 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
       }
     }
 
-    if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
-      if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
-        return TSDB_CODE_TSC_INVALID_SQL;
+    if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
+      if (pRight->nodeType == TSQL_NODE_VALUE) {
+        if (  pRight->pVal->nType == TSDB_DATA_TYPE_BOOL 
+           || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY 
+           || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
+          return TSDB_CODE_TSC_INVALID_SQL;
+        }
       }
     }
   }

From 10a64cccc8964c618b8d29e05e7e1efcecf8e659 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Sat, 13 Jun 2020 15:12:28 +0800
Subject: [PATCH 17/25] add numpy install to travis

[TD-630]
---
 .travis.yml       | 4 ++++
 tests/test-all.sh | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 54a7856f7b..4ae38f5032 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,7 +45,9 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
+          pip install numpy
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install numpy
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
@@ -164,7 +166,9 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
+          pip install numpy
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install numpy
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
diff --git a/tests/test-all.sh b/tests/test-all.sh
index 485c852dc5..0c91229120 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -41,7 +41,7 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run TSIM full test ###"
     runSimCaseOneByOne fullGeneralSuite.sim
-  else
+  elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run TSIM smoke test ###"
     runSimCaseOneByOne basicSuite.sim
   fi
@@ -77,7 +77,7 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
-  else
+  elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run Python smoke test ###"
     runPyCaseOneByOne smoketest.sh
   fi

From 30b83affceaf0b1bd32dd815124ffc28cf1db507 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Sat, 13 Jun 2020 16:06:34 +0800
Subject: [PATCH 18/25] add stream cases to jenkins

---
 tests/script/jenkins/basic.txt | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 191e3212b6..084109623f 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -159,6 +159,21 @@ cd ../../../debug; make
 ./test.sh -f general/stable/values.sim
 ./test.sh -f general/stable/vnode3.sim
 
+#./test.sh -f general/stream/metrics_1.sim
+#./test.sh -f general/stream/metrics_del.sim
+#./test.sh -f general/stream/metrics_n.sim
+#./test.sh -f general/stream/metrics_replica1_vnoden.sim
+#./test.sh -f general/stream/new_stream.sim
+#./test.sh -f general/stream/restart_stream.sim
+#./test.sh -f general/stream/stream_1.sim
+#./test.sh -f general/stream/stream_2.sim
+#./test.sh -f general/stream/stream_3.sim
+#./test.sh -f general/stream/stream_restart.sim
+#./test.sh -f general/stream/table_1.sim
+#./test.sh -f general/stream/table_del.sim
+#./test.sh -f general/stream/table_n.sim
+#./test.sh -f general/stream/table_replica1_vnoden.sim
+
 ./test.sh -f general/table/autocreate.sim
 ./test.sh -f general/table/basic1.sim
 ./test.sh -f general/table/basic2.sim

From cc3fb92f4b5a5ee190b7a7072a3f7bfc34f26ec0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 16:18:14 +0800
Subject: [PATCH 19/25] [td-225] fix error in scripts

---
 tests/script/general/parser/limit1_stb.sim |  2 +-
 tests/script/general/parser/nchar.sim      | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim
index a1f9aabd9d..fbcd8d0965 100644
--- a/tests/script/general/parser/limit1_stb.sim
+++ b/tests/script/general/parser/limit1_stb.sim
@@ -352,7 +352,7 @@ if $data07 != nchar0 then
   return -1 
 endi
 
-sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 > 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0;
+sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 != 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0;
 if $rows != 1 then
   return -1
 endi
diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim
index 2c86748f21..c0de6f5058 100644
--- a/tests/script/general/parser/nchar.sim
+++ b/tests/script/general/parser/nchar.sim
@@ -198,15 +198,15 @@ if $rows != 100  then
   return -1
 endi
 
-sql select * from $mt where tgcol > '0'
-#print rows = $rows
-if $rows != 100 then
-  return -1
-endi
-#print $data03
-if $data03 != 1 then
-  return -1
-endi
+#sql select * from $mt where tgcol > '0'
+##print rows = $rows
+#if $rows != 100 then
+#  return -1
+#endi
+##print $data03
+#if $data03 != 1 then
+#  return -1
+#endi
 
 # cumulative query with nchar tag filtering
 sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tgcol = '1'

From 995eb63d98c2efc4e04d79a23e903cf2d330764f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 13 Jun 2020 17:02:23 +0800
Subject: [PATCH 20/25] [td-541] add string length overflow check.

---
 src/kit/taosdemo/taosdemo.c | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index bfedff0d94..bd9d41a4de 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -297,7 +297,7 @@ void *deleteTable();
 
 void *asyncWrite(void *sarg);
 
-void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
+int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
 
 void rand_string(char *str, int size);
 
@@ -846,14 +846,19 @@ void *syncWrite(void *sarg) {
       int k;
       for (k = 0; k < winfo->nrecords_per_request;) {
         int rand_num = rand() % 100;
-        if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
-        {
+        int len = -1;
+        if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
           long d = tmp_time - rand() % 1000000 + rand_num;
-          generateData(data, data_type, ncols_per_record, d, len_of_binary);
-        } else 
-        {
-          generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
+          len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
+        } else {
+          len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
         }
+
+        //assert(len + pstr - buffer < BUFFER_SIZE);
+        if (len + pstr - buffer >= BUFFER_SIZE) { // too long
+          break;
+        }
+
         pstr += sprintf(pstr, " %s", data);
         inserted++;
         k++;
@@ -968,7 +973,7 @@ double getCurrentTime() {
   return tv.tv_sec + tv.tv_usec / 1E6;
 }
 
-void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
+int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
   memset(res, 0, MAX_DATA_SIZE);
   char *pstr = res;
   pstr += sprintf(pstr, "(%" PRId64, timestamp);
@@ -1002,9 +1007,16 @@ void generateData(char *res, char **data_type, int num_of_cols, int64_t timestam
       rand_string(s, len_of_binary);
       pstr += sprintf(pstr, ", \"%s\"", s);
     }
+
+    if (pstr - res > MAX_DATA_SIZE) {
+      perror("column length too long, abort");
+      exit(-1);
+    }
   }
 
   pstr += sprintf(pstr, ")");
+
+  return pstr - res;
 }
 
 static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";

From 3ddef8f7e2ec86cdc70b8902958596a26b3ae609 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sun, 14 Jun 2020 00:21:32 +0000
Subject: [PATCH 21/25] add rpcCancelRequest API

---
 src/inc/trpc.h        |  3 ++-
 src/rpc/src/rpcMain.c | 20 ++++++++++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/inc/trpc.h b/src/inc/trpc.h
index 748fad0e62..6c5d7fa889 100644
--- a/src/inc/trpc.h
+++ b/src/inc/trpc.h
@@ -78,12 +78,13 @@ void  rpcClose(void *);
 void *rpcMallocCont(int contLen);
 void  rpcFreeCont(void *pCont);
 void *rpcReallocCont(void *ptr, int contLen);
-void  rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
+void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
 void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
 int   rpcReportProgress(void *pConn, char *pCont, int contLen);
+void  rpcCanelRequest(void *pContext);
 
 #ifdef __cplusplus
 }
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index cf15cc690a..86d47f6c65 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -73,6 +73,7 @@ typedef struct {
   SRpcInfo *pRpc;       // associated SRpcInfo
   SRpcIpSet ipSet;      // ip list provided by app
   void     *ahandle;    // handle provided by app
+  struct SRpcConn *pConn; // pConn allocated
   char      msgType;    // message type
   uint8_t  *pCont;      // content provided by app
   int32_t   contLen;    // content length
@@ -339,7 +340,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
 }
 
-void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
+void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
   SRpcInfo       *pRpc = (SRpcInfo *)shandle;
   SRpcReqContext *pContext;
 
@@ -367,7 +368,7 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   
   rpcSendReqToServer(pRpc, pContext);
 
-  return;
+  return pContext;
 }
 
 void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -501,6 +502,19 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
   return -1;
 }
 
+/* todo: cancel process may have race condition, pContext may have been released 
+   just before app calls the rpcCancelRequest */
+void rpcCancelRequest(void *handle) {
+  SRpcReqContext *pContext = handle;
+
+  if (pContext->pConn) {
+    tTrace("%s, app trys to cancel request", pConn->info);
+    rpcCloseConn(pContext->pConn);
+    pContext->pConn = NULL;
+    rpcFreeCont(pContext->pCont);
+  }
+}
+
 static void rpcFreeMsg(void *msg) {
   if ( msg ) {
     char *temp = (char *)msg - sizeof(SRpcReqContext);
@@ -942,6 +956,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
 static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
   SRpcInfo       *pRpc = pContext->pRpc;
 
+  pContext->pConn = NULL;
   if (pContext->pRsp) { 
     // for synchronous API
     memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet));
@@ -1110,6 +1125,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
     return;
   }
 
+  pContext->pConn = pConn;
   pConn->ahandle = pContext->ahandle;
   rpcLockConn(pConn);
 

From b6a1529322b80914589801e3ea744843b327661b Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sun, 14 Jun 2020 01:17:47 +0000
Subject: [PATCH 22/25] add debug info

---
 src/rpc/src/rpcMain.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 86d47f6c65..e4ba388394 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -508,7 +508,7 @@ void rpcCancelRequest(void *handle) {
   SRpcReqContext *pContext = handle;
 
   if (pContext->pConn) {
-    tTrace("%s, app trys to cancel request", pConn->info);
+    tTrace("%s, app trys to cancel request", pContext->pConn->info);
     rpcCloseConn(pContext->pConn);
     pContext->pConn = NULL;
     rpcFreeCont(pContext->pCont);

From 6380afb63da4ee330b7721e18475195b90a1f70d Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sun, 14 Jun 2020 01:28:26 +0000
Subject: [PATCH 23/25] set handle when report broken link to server

---
 src/rpc/src/rpcMain.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index e4ba388394..dcbcae452d 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -888,6 +888,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
   SRpcMsg rpcMsg;
   rpcMsg.pCont = pConn->pReqMsg;     // pReqMsg is re-used to store the APP context from server
   rpcMsg.contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length
+  rpcMsg.ahandle = pConn->ahandle;
   rpcMsg.handle = pConn;
   rpcMsg.msgType = pConn->inType;
   rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; 

From e258f1ace0baa5648adcaabacbbd87f71b52852e Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Mon, 15 Jun 2020 11:49:29 +0800
Subject: [PATCH 24/25] fix td-320

---
 src/client/inc/tsclient.h     | 2 +-
 src/client/src/tscSQLParser.c | 2 +-
 src/inc/taosmsg.h             | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 24e0f48ec9..ed3b481d0e 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -286,7 +286,7 @@ typedef struct STscObj {
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
   char               acctId[TSDB_ACCT_LEN];
-  char               db[TSDB_DB_NAME_LEN];
+  char               db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 8e5a439431..3f0af8f466 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1042,7 +1042,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
 
   /* db name is not specified, the tableName dose not include db name */
   if (pDB != NULL) {
-    if (pDB->n >= TSDB_DB_NAME_LEN) {
+    if (pDB->n >= TSDB_ACCT_LEN + TSDB_DB_NAME_LEN) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index 1198097895..8e732f0cac 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -250,7 +250,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN];
-  char    db[TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   int8_t  igExists;
   int8_t  getMeta;
   int16_t numOfTags;
@@ -268,7 +268,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN];
-  char    db[TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   int16_t type; /* operation type   */
   int16_t numOfCols; /* number of schema */
   int32_t tagValLen;
@@ -670,7 +670,7 @@ typedef struct {
  */
 typedef struct {
   int8_t   type;
-  char     db[TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   uint16_t payloadLen;
   char     payload[];
 } SCMShowMsg;

From 1688b5a396037c79b4bed3b6cd877ba9aa3ce9a5 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Mon, 15 Jun 2020 13:40:18 +0800
Subject: [PATCH 25/25] [TD-641]: Update documentation for running python test
 cases

---
 tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md
index 3477fe9152..b6859827bf 100644
--- a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md
+++ b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md
@@ -11,6 +11,8 @@
 4.  pip install ../src/connector/python/linux/python2 ; pip3 install
     ../src/connector/python/linux/python3
 
+5.  pip install numpy; pip3 install numpy    
+
 >   Note: Both Python2 and Python3 are currently supported by the Python test
 >   framework. Since Python2 is no longer officially supported by Python Software
 >   Foundation since January 1, 2020, it is recommended that subsequent test case