From 0755ec1f12c9d02540bd7a548ea9bfe9b8b221ef Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 7 Sep 2020 23:58:53 +0800
Subject: [PATCH 01/56] [td-225] fix bug while the content in binary column
 equals to the maximum allowed length

---
 src/client/src/TSDBJNIConnector.c | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c
index b25f620508..bd980b75a3 100644
--- a/src/client/src/TSDBJNIConnector.c
+++ b/src/client/src/TSDBJNIConnector.c
@@ -437,13 +437,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData
  * @return
  */
 jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) {
-  int len = (int)strlen(nchar);
-  if (len > maxBytes) {  // no terminated symbol exists '\0'
-    len = maxBytes;
-  }
-
-  jbyteArray bytes = (*env)->NewByteArray(env, len);
-  (*env)->SetByteArrayRegion(env, bytes, 0, len, (jbyte *)nchar);
+  jbyteArray bytes = (*env)->NewByteArray(env, maxBytes);
+  (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar);
   return bytes;
 }
 
@@ -481,6 +476,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
     }
   }
 
+  int32_t* length = taos_fetch_lengths(result);
+
   char tmp[TSDB_MAX_BYTES_PER_ROW] = {0};
 
   for (int i = 0; i < num_fields; i++) {
@@ -515,15 +512,15 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv);
       } break;
       case TSDB_DATA_TYPE_BINARY: {
-        strncpy(tmp, row[i], (size_t)fields[i].bytes);  // handle the case that terminated does not exist
+        memcpy(tmp, row[i], length[i]);  // handle the case that terminated does not exist
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp));
 
-        memset(tmp, 0, (size_t)fields[i].bytes);
+        memset(tmp, 0, length[i]);
         break;
       }
       case TSDB_DATA_TYPE_NCHAR: {
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i,
-                               jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes));
+                               jniFromNCharToByteArray(env, (char *)row[i], length[i]));
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP:

From ed7e81f1545c3f1a44c2282d6b3f0b579494503d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 13:17:24 +0800
Subject: [PATCH 02/56] [td-1393] opt last_row query with tags filter
 conditions.

---
 src/client/src/tscSQLParser.c | 97 +++++++++++++++++------------------
 1 file changed, 47 insertions(+), 50 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 48980fefb6..75d21fd9eb 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -92,7 +92,6 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS
 static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
 static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo);
 static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
-static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString);
 static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
 static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type);
 static int32_t validateEp(char* ep);
@@ -103,6 +102,7 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy
 
 static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
 static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
+static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo);
 
 static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql);
 static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql);
@@ -1330,7 +1330,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
 
   /*
    * transfer sql functions that need secondary merge into another format
-   * in dealing with metric queries such as: count/first/last
+   * in dealing with super table queries such as: count/first/last
    */
   if (isSTable) {
     tscTansformSQLFuncForSTableQuery(pQueryInfo);
@@ -1515,16 +1515,16 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
 
 static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName,
                                        int32_t resColIdx, SColumnIndex* pColIndex) {
+  const char* msg1 = "not support column types";
+
   int16_t type = 0;
   int16_t bytes = 0;
-  char        columnName[TSDB_COL_NAME_LEN] = {0};
-  const char* msg1 = "not support column types";
+  char    columnName[TSDB_COL_NAME_LEN] = {0};
   int32_t functionID = cvtFunc.execFuncId;
 
   if (functionID == TSDB_FUNC_SPREAD) {
-    if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY ||
-        pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_NCHAR ||
-        pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BOOL) {
+    int32_t t1 = pSchema[pColIndex->columnIndex].type;
+    if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
       return -1;
     } else {
@@ -1541,8 +1541,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
   } else {
     getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
   }
-  
-  
+
   SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
   tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
 
@@ -1783,10 +1782,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
       }
+
+      // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query
       SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID};
-      if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) {
+      if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) {
         cvtFunc.execFuncId = TSDB_FUNC_LAST;
       }
+
       if (!requireAllFields) {
         if (pItem->pNode->pParam->nExpr < 1) {
           return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
@@ -3204,19 +3206,6 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
   return TSDB_CODE_SUCCESS;
 }
 
-static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) {
-  char* start = *str;
-
-  int32_t code = doArithmeticExprToString(pExpr, str);
-  if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis
-    int32_t len = (int32_t)strlen(start);
-    memmove(start, start + 1, len - 2);
-    start[len - 2] = 0;
-  }
-
-  return code;
-}
-
 static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) {
   if (pExpr->nSQLOptr == TK_ID) {
     if (*type == NON_ARITHMEIC_EXPR) {
@@ -3615,7 +3604,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr
 }
 
 static void doCompactQueryExpr(tSQLExpr** pExpr) {
-  if (*pExpr == NULL || isExprDirectParentOfLeaftNode(*pExpr)) {
+  if (*pExpr == NULL || isExprDirectParentOfLeafNode(*pExpr)) {
     return;
   }
 
@@ -3975,7 +3964,6 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
   const char* msg2 = "invalid filter expression";
 
   int32_t ret = TSDB_CODE_SUCCESS;
-  pQueryInfo->window = TSWINDOW_INITIALIZER;
 
   // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
   SStringBuilder sb; memset(&sb, 0, sizeof(sb));
@@ -6109,11 +6097,27 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
+  // set where info
+  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
+
+  if (pQuerySql->pWhere != NULL) {
+    if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_SQL;
+    }
+
+    pQuerySql->pWhere = NULL;
+    if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
+      pQueryInfo->window.skey = pQueryInfo->window.skey / 1000;
+      pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
+    }
+  } else {  // set the time rang
+    if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
+    }
+  }
+
   int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
 
-  if (pQuerySql->pWhere) {
-    pQueryInfo->window = TSWINDOW_INITIALIZER;
-  }
   if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
@@ -6133,26 +6137,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
-  // set where info
-  STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
-  
-  if (pQuerySql->pWhere != NULL) {
-    if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_TSC_INVALID_SQL;
-    }
-
-    pQuerySql->pWhere = NULL;
-    if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
-      pQueryInfo->window.skey = pQueryInfo->window.skey / 1000;
-      pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
-    }
-  } else {  // set the time rang
-    pQueryInfo->window = TSWINDOW_INITIALIZER;
-    if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
-      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
-    }
-  }
-
   // user does not specified the query time window, twa is not allowed in such case.
   if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
        (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {
@@ -6303,7 +6287,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
     (*pExpr)->_node.optr = getBinaryExprOptr(&t);
     
     assert((*pExpr)->_node.optr != 0);
-    
+
+    // check for dividing by 0
     if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) {
       if (pRight->nodeType == TSQL_NODE_VALUE) {
         if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) {
@@ -6316,7 +6301,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
 
     // NOTE: binary|nchar data allows the >|< type filter
     if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
-      if (pRight->nodeType == TSQL_NODE_VALUE) {
+      if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) {
         if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
@@ -6326,3 +6311,15 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
   
   return TSDB_CODE_SUCCESS;
 }
+
+bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) {
+  size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
+    if (pCol->numOfFilters > 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
\ No newline at end of file

From fb25b058aecf496801d4fddd181bb6b9446ce618 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 13:22:45 +0800
Subject: [PATCH 03/56] [td-1394] add the returning value check after invoking
 the load data block from file function.

---
 src/query/src/qExecutor.c | 65 +++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d48d7d5ea1..37fe302d43 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -2185,43 +2185,43 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
   return false;
 }
 
-int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) {
+int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  uint32_t status = 0;
+  *status = 0;
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) {
-    status = BLK_DATA_ALL_NEEDED;
+    *status = BLK_DATA_ALL_NEEDED;
   } else { // check if this data block is required to load
 
     // Calculate all time windows that are overlapping or contain current data block.
     // If current data block is contained by all possible time window, do not load current data block.
     if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) {
-      status = BLK_DATA_ALL_NEEDED;
+      *status = BLK_DATA_ALL_NEEDED;
     }
 
-    if (status != BLK_DATA_ALL_NEEDED) {
+    if ((*status) != BLK_DATA_ALL_NEEDED) {
       for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
         SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base;
 
         int32_t functionId = pSqlFunc->functionId;
         int32_t colId = pSqlFunc->colInfo.colId;
 
-        status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
-        if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
+        (*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
+        if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
           break;
         }
       }
     }
   }
 
-  if (status == BLK_DATA_NO_NEEDED) {
+  if ((*status) == BLK_DATA_NO_NEEDED) {
     qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
            pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
     pRuntimeEnv->summary.discardBlocks += 1;
-  } else if (status == BLK_DATA_STATIS_NEEDED) {
-    if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) {
-      //        return DISK_DATA_LOAD_FAILED;
-    }
+  } else if ((*status) == BLK_DATA_STATIS_NEEDED) {
+
+    // this function never returns error?
+    tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis);
 
     pRuntimeEnv->summary.loadBlockStatis += 1;
 
@@ -2230,24 +2230,26 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
       pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows;
     }
   } else {
-    assert(status == BLK_DATA_ALL_NEEDED);
+    assert((*status) == BLK_DATA_ALL_NEEDED);
 
     // load the data block statistics to perform further filter
     pRuntimeEnv->summary.loadBlockStatis += 1;
-    if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) {
-    }
+    tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis);
 
     if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) {
       // current block has been discard due to filter applied
       pRuntimeEnv->summary.discardBlocks += 1;
       qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-      return BLK_DATA_DISCARD;
+      (*status) = BLK_DATA_DISCARD;
     }
 
     pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows;
     pRuntimeEnv->summary.loadBlocks += 1;
     *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL);
+    if (*pDataBlock == NULL) {
+      return terrno;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2431,15 +2433,18 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     ensureOutputBuffer(pRuntimeEnv, &blockInfo);
 
     SDataStatis *pStatis = NULL;
-    SArray *pDataBlock   = NULL;
-    if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) {
-      pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step;
-      continue;
+    SArray *     pDataBlock = NULL;
+    uint32_t     status = 0;
+
+    int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status);
+    if (ret != TSDB_CODE_SUCCESS) {
+      break;
     }
 
-    if (terrno != TSDB_CODE_SUCCESS) { // load data block failed, abort query
-      longjmp(pRuntimeEnv->env, terrno);
-      break;
+    if (status == BLK_DATA_DISCARD) {
+      pQuery->current->lastKey =
+              QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step;
+      continue;
     }
 
     // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition
@@ -4651,9 +4656,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     }
 
     SDataStatis *pStatis = NULL;
-    SArray *pDataBlock = NULL;
-    if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) {
-      pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step;
+    SArray *     pDataBlock = NULL;
+    uint32_t     status = 0;
+
+    int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status);
+    if (ret != TSDB_CODE_SUCCESS) {
+      break;
+    }
+
+    if (status == BLK_DATA_DISCARD) {
+      pQuery->current->lastKey =
+          QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step;
       continue;
     }
 

From 2a395670bde9ac81cf96e00f8d07cd486a3eec4d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 13:24:07 +0800
Subject: [PATCH 04/56] [td-1369]

---
 src/client/src/tscSQLParser.c |  37 ++-
 src/client/src/tscUtil.c      |   5 +-
 src/inc/taoserror.h           |   2 +-
 src/query/inc/qAst.h          |   1 +
 src/query/inc/sql.y           |  72 ++---
 src/query/src/qAst.c          |  36 ++-
 src/query/src/qExecutor.c     |   5 +-
 src/query/src/qFilterfunc.c   |  81 +++++
 src/query/src/qParserImpl.c   |   5 +
 src/query/src/sql.c           | 548 +++++++++++++++++-----------------
 src/tsdb/src/tsdbRead.c       |  14 +-
 11 files changed, 471 insertions(+), 335 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 75d21fd9eb..adb6bfccbc 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -2784,6 +2784,12 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
     case TK_LIKE:
       pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE;
       break;
+    case TK_ISNULL:
+      pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL;
+      break;
+    case TK_NOTNULL:
+      pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL;
+      break;
     default:
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
@@ -2829,19 +2835,19 @@ static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) {
   return TSDB_CODE_SUCCESS;
 }
 
+// pExpr->nSQLOptr == 0 while handling "is null" query
 static bool isExprLeafNode(tSQLExpr* pExpr) {
   return (pExpr->pRight == NULL && pExpr->pLeft == NULL) &&
-         (pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) ||
-          pExpr->nSQLOptr == TK_SET);
+         (pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET);
 }
 
-static bool isExprDirectParentOfLeaftNode(tSQLExpr* pExpr) {
+static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) {
   return (pExpr->pLeft != NULL && pExpr->pRight != NULL) &&
          (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight));
 }
 
 static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) {
-  if (!isExprDirectParentOfLeaftNode(pExpr)) {
+  if (!isExprDirectParentOfLeafNode(pExpr)) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
@@ -3052,7 +3058,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (!isExprDirectParentOfLeaftNode(pExpr)) {
+  if (!isExprDirectParentOfLeafNode(pExpr)) {
     *(*str) = '(';
     *str += 1;
 
@@ -3108,7 +3114,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ
     return TSDB_CODE_SUCCESS;
   }
 
-  if (!isExprDirectParentOfLeaftNode(pExpr)) {  // internal node
+  if (!isExprDirectParentOfLeafNode(pExpr)) {  // internal node
     int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
@@ -3134,7 +3140,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
     return TSDB_CODE_SUCCESS;
   }
 
-  if (!isExprDirectParentOfLeaftNode(pExpr)) {
+  if (!isExprDirectParentOfLeafNode(pExpr)) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
 
@@ -3453,7 +3459,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
   }
 
-  assert(isExprDirectParentOfLeaftNode(*pExpr));
+  assert(isExprDirectParentOfLeafNode(*pExpr));
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
@@ -3499,7 +3505,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
       }
     }
 
-    // in case of in operator, keep it in a seperate attribute
+    // in case of in operator, keep it in a seprate attribute
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
       if (!validTableNameOptr(*pExpr)) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
@@ -3520,7 +3526,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
       *type = TSQL_EXPR_TBNAME;
       *pExpr = NULL;
     } else {
-      if (pRight->nSQLOptr == TK_ID) {  // join on tag columns for stable query
+      if (pRight != NULL && pRight->nSQLOptr == TK_ID) {  // join on tag columns for stable query
         if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
@@ -3573,7 +3579,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr
   int32_t leftType = -1;
   int32_t rightType = -1;
 
-  if (!isExprDirectParentOfLeaftNode(*pExpr)) {
+  if (!isExprDirectParentOfLeafNode(*pExpr)) {
     int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
@@ -3635,7 +3641,7 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) {
 }
 
 static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, tSQLExpr** pOut, int32_t tableIndex) {
-  if (isExprDirectParentOfLeaftNode(*pExpr)) {
+  if (isExprDirectParentOfLeafNode(*pExpr)) {
     tSQLExpr* pLeft = (*pExpr)->pLeft;
 
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
@@ -3794,7 +3800,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE
     return TSDB_CODE_SUCCESS;
   }
 
-  if (!isExprDirectParentOfLeaftNode(pExpr)) {
+  if (!isExprDirectParentOfLeafNode(pExpr)) {
     if (pExpr->nSQLOptr == TK_OR) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
@@ -6215,6 +6221,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
       return ret;
     }
   }
+
+  if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) {
+    *pExpr = calloc(1, sizeof(tExprNode));
+    return TSDB_CODE_SUCCESS;
+  }
   
   if (pSqlExpr->pLeft == NULL) {
     if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) {
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 80e3828c9d..ad87825e34 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1531,9 +1531,10 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
   pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo));
   
   assert(pQueryInfo->exprList == NULL);
-  pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
-  pQueryInfo->colList  = taosArrayInit(4, POINTER_BYTES);
+  pQueryInfo->exprList   = taosArrayInit(4, POINTER_BYTES);
+  pQueryInfo->colList    = taosArrayInit(4, POINTER_BYTES);
   pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX;
+  pQueryInfo->window     = TSWINDOW_INITIALIZER;
 }
 
 int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index d8e5c8f1d7..3198d6d97e 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -98,7 +98,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS,       0, 0x0212, "Action in
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED,             0, 0x0213, "Disconnected from service")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH,            0, 0x0214, "No write permission")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED,            0, 0x0215, "Connection killed")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR,            0, 0x0216, "Syntax errr in SQL")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR,            0, 0x0216, "Syntax error in SQL")
 
 // mnode
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED,        0, 0x0300, "Message not processed")
diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h
index 547616dee6..d3e60c21dc 100644
--- a/src/query/inc/qAst.h
+++ b/src/query/inc/qAst.h
@@ -32,6 +32,7 @@ struct tExprNode;
 struct SSchema;
 
 enum {
+  TSQL_NODE_DUMMY = 0x0,
   TSQL_NODE_EXPR  = 0x1,
   TSQL_NODE_COL   = 0x2,
   TSQL_NODE_VALUE = 0x4,
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index 79aec2f349..e0dd9d98f0 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -567,53 +567,53 @@ where_opt(A) ::= WHERE expr(X).       {A = X;}
 %type expr {tSQLExpr*}
 %destructor expr {tSQLExprDestroy($$);}
 
-expr(A) ::= LP expr(X) RP.   {A = X; }
+expr(A) ::= LP expr(X) RP.       {A = X; }
 
-expr(A) ::= ID(X).           {A = tSQLExprIdValueCreate(&X, TK_ID);}
-expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);}
-expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);}
+expr(A) ::= ID(X).               {A = tSQLExprIdValueCreate(&X, TK_ID);}
+expr(A) ::= ID(X) DOT ID(Y).     {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);}
+expr(A) ::= ID(X) DOT STAR(Y).   {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);}
 
-expr(A) ::= INTEGER(X).      {A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
+expr(A) ::= INTEGER(X).          {A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
 expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
 expr(A) ::= PLUS(X)  INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
-expr(A) ::= FLOAT(X).        {A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
-expr(A) ::= MINUS(X) FLOAT(Y).       {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
-expr(A) ::= PLUS(X) FLOAT(Y).        {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
-expr(A) ::= STRING(X).       {A = tSQLExprIdValueCreate(&X, TK_STRING);}
-expr(A) ::= NOW(X).          {A = tSQLExprIdValueCreate(&X, TK_NOW); }
-expr(A) ::= VARIABLE(X).     {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);}
-expr(A) ::= BOOL(X).         {A = tSQLExprIdValueCreate(&X, TK_BOOL);}
-// normal functions: min(x)
-expr(A) ::= ID(X) LP exprlist(Y) RP(E). {
-  A = tSQLExprCreateFunction(Y, &X, &E, X.type);
-}
+expr(A) ::= FLOAT(X).            {A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
+expr(A) ::= MINUS(X) FLOAT(Y).   {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
+expr(A) ::= PLUS(X) FLOAT(Y).    {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
+expr(A) ::= STRING(X).           {A = tSQLExprIdValueCreate(&X, TK_STRING);}
+expr(A) ::= NOW(X).              {A = tSQLExprIdValueCreate(&X, TK_NOW); }
+expr(A) ::= VARIABLE(X).         {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);}
+expr(A) ::= BOOL(X).             {A = tSQLExprIdValueCreate(&X, TK_BOOL);}
 
-// this is for: count(*)/first(*)/last(*) operation
-expr(A) ::= ID(X) LP STAR RP(Y). {
-  A = tSQLExprCreateFunction(NULL, &X, &Y, X.type);
-}
+// ordinary functions: min(x), max(x), top(k, 20)
+expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); }
 
-//binary expression: a+2, b+3
-expr(A) ::= expr(X) AND expr(Y).   {A = tSQLExprCreate(X, Y, TK_AND);}
-expr(A) ::= expr(X) OR  expr(Y).   {A = tSQLExprCreate(X, Y, TK_OR); }
+// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
+expr(A) ::= ID(X) LP STAR RP(Y).     { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); }
 
-//binary relational expression
-expr(A) ::= expr(X) LT expr(Y).    {A = tSQLExprCreate(X, Y, TK_LT);}
-expr(A) ::= expr(X) GT expr(Y).    {A = tSQLExprCreate(X, Y, TK_GT);}
-expr(A) ::= expr(X) LE expr(Y).    {A = tSQLExprCreate(X, Y, TK_LE);}
-expr(A) ::= expr(X) GE expr(Y).    {A = tSQLExprCreate(X, Y, TK_GE);}
-expr(A) ::= expr(X) NE expr(Y).    {A = tSQLExprCreate(X, Y, TK_NE);}
-expr(A) ::= expr(X) EQ expr(Y).    {A = tSQLExprCreate(X, Y, TK_EQ);}
+// is (not) null expression
+expr(A) ::= expr(X) IS NULL.           {A = tSQLExprCreate(X, NULL, TK_ISNULL);}
+expr(A) ::= expr(X) IS NOT NULL.       {A = tSQLExprCreate(X, NULL, TK_NOTNULL);}
 
-//binary arithmetic expression
+// relational expression
+expr(A) ::= expr(X) LT expr(Y).      {A = tSQLExprCreate(X, Y, TK_LT);}
+expr(A) ::= expr(X) GT expr(Y).      {A = tSQLExprCreate(X, Y, TK_GT);}
+expr(A) ::= expr(X) LE expr(Y).      {A = tSQLExprCreate(X, Y, TK_LE);}
+expr(A) ::= expr(X) GE expr(Y).      {A = tSQLExprCreate(X, Y, TK_GE);}
+expr(A) ::= expr(X) NE expr(Y).      {A = tSQLExprCreate(X, Y, TK_NE);}
+expr(A) ::= expr(X) EQ expr(Y).      {A = tSQLExprCreate(X, Y, TK_EQ);}
+
+expr(A) ::= expr(X) AND expr(Y).     {A = tSQLExprCreate(X, Y, TK_AND);}
+expr(A) ::= expr(X) OR  expr(Y).     {A = tSQLExprCreate(X, Y, TK_OR); }
+
+// binary arithmetic expression
 expr(A) ::= expr(X) PLUS  expr(Y).   {A = tSQLExprCreate(X, Y, TK_PLUS);  }
 expr(A) ::= expr(X) MINUS expr(Y).   {A = tSQLExprCreate(X, Y, TK_MINUS); }
 expr(A) ::= expr(X) STAR  expr(Y).   {A = tSQLExprCreate(X, Y, TK_STAR);  }
 expr(A) ::= expr(X) SLASH expr(Y).   {A = tSQLExprCreate(X, Y, TK_DIVIDE);}
 expr(A) ::= expr(X) REM   expr(Y).   {A = tSQLExprCreate(X, Y, TK_REM);   }
 
-//like expression
-expr(A) ::= expr(X) LIKE  expr(Y).   {A = tSQLExprCreate(X, Y, TK_LIKE);  }
+// like expression
+expr(A) ::= expr(X) LIKE expr(Y).    {A = tSQLExprCreate(X, Y, TK_LIKE);  }
 
 //in expression
 expr(A) ::= expr(X) IN LP exprlist(Y) RP.   {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); }
@@ -625,9 +625,9 @@ expr(A) ::= expr(X) IN LP exprlist(Y) RP.   {A = tSQLExprCreate(X, (tSQLExpr*)Y,
 %destructor expritem {tSQLExprDestroy($$);}
 
 exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);}
-exprlist(A) ::= expritem(X).            {A = tSQLExprListAppend(0,X,0);}
-expritem(A) ::= expr(X).                {A = X;}
-expritem(A) ::= .                       {A = 0;}
+exprlist(A) ::= expritem(X).                   {A = tSQLExprListAppend(0,X,0);}
+expritem(A) ::= expr(X).                       {A = X;}
+expritem(A) ::= .                              {A = 0;}
 
 ///////////////////////////////////reset query cache//////////////////////////////////////
 cmd ::= RESET QUERY CACHE.  { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c
index 634f014d97..63411aaf3f 100644
--- a/src/query/src/qAst.c
+++ b/src/query/src/qAst.c
@@ -188,6 +188,10 @@ uint8_t getBinaryExprOptr(SStrToken *pToken) {
       return TSDB_BINARY_OP_REMAINDER;
     case TK_LIKE:
       return TSDB_RELATION_LIKE;
+    case TK_ISNULL:
+      return TSDB_RELATION_ISNULL;
+    case TK_NOTNULL:
+      return TSDB_RELATION_NOTNULL;
     default: { return 0; }
   }
 }
@@ -486,29 +490,42 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   } else {
     int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID;
     if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) {
-      bool comp = true;
+      bool    comp = true;
       int32_t ret = 0;
-      
-      while(tSkipListIterNext(iter)) {
-        SSkipListNode* pNode = tSkipListIterGet(iter);
-      
+
+      while (tSkipListIterNext(iter)) {
+        SSkipListNode *pNode = tSkipListIterGet(iter);
+
         if (comp) {
           ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v);
           assert(ret <= 0);
         }
-        
+
         if (ret == 0 && optr == TSDB_RELATION_LESS) {
           continue;
         } else {
-          STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+          STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
           taosArrayPush(result, &info);
           comp = false;  // no need to compare anymore
         }
       }
+    } else {
+      assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL);
+
+      while (tSkipListIterNext(iter)) {
+        SSkipListNode *pNode = tSkipListIterGet(iter);
+
+        bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
+        if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
+            (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
+          STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+          taosArrayPush(result, &info);
+        }
+      }
     }
   }
 
-  free(cond.start); 
+  free(cond.start);
   free(cond.end);
   tSkipListDestroyIter(iter);
 }
@@ -683,6 +700,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
     char *         pData = SL_GET_NODE_DATA(pNode);
 
     tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
+
     // todo speed up by using hash
     if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
@@ -714,7 +732,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
 
   // column project
   if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) {
-    assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE);
+    assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY));
 
     param->setupInfoFn(pExpr, param->pExtInfo);
     if (pSkipList == NULL) {
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 37fe302d43..2ca2545818 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -200,9 +200,6 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) {
     SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k];
 
     char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos;
-    if (isNull(pElem, pFilterInfo->info.type)) {
-      return false;
-    }
 
     bool qualified = false;
     for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) {
@@ -1002,7 +999,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   }
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
-  if (QUERY_IS_INTERVAL_QUERY(pQuery)/* && tsCols != NULL*/) {
+  if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
     TSKEY ts = TSKEY_INITIAL_VAL;
 
     if (tsCols == NULL) {
diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c
index 7e9f5c7da5..6b88171e71 100644
--- a/src/query/src/qFilterfunc.c
+++ b/src/query/src/qFilterfunc.c
@@ -284,6 +284,71 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
 
   return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0;
 }
+////////////////////////////////////////////////////////////////
+bool isNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_TINYINT);
+}
+
+bool isNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_SMALLINT);
+}
+
+bool isNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_INT);
+}
+
+bool isNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_BIGINT);
+}
+
+bool isNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_FLOAT);
+}
+
+bool isNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_DOUBLE);
+}
+
+bool isNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_BINARY);
+}
+
+bool isNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_NCHAR);
+}
+
+////////////////////////////////////////////////////////////////
+bool notNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_TINYINT);
+}
+
+bool notNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_SMALLINT);
+}
+
+bool notNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_INT);
+}
+
+bool notNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_BIGINT);
+}
+
+bool notNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_FLOAT);
+}
+
+bool notNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return isNull(minval, TSDB_DATA_TYPE_DOUBLE);
+}
+
+bool notNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_BINARY);
+}
+
+bool notNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
+  return !isNull(minval, TSDB_DATA_TYPE_NCHAR);
+}
 
 ////////////////////////////////////////////////////////////////
 
@@ -398,6 +463,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i8,
   nequal_i8,
   NULL,
+  isNull_i8,
+  notNull_i8,
 };
 
 bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -409,6 +476,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i16,
   nequal_i16,
   NULL,
+  isNull_i16,
+  notNull_i16,
 };
 
 bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -420,6 +489,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i32,
   nequal_i32,
   NULL,
+  isNull_i32,
+  notNull_i32,
 };
 
 bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -431,6 +502,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i64,
   nequal_i64,
   NULL,
+  isNull_i64,
+  notNull_i64,
 };
 
 bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -442,6 +515,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_ds,
   nequal_ds,
   NULL,
+  isNull_ds,
+  notNull_ds,
 };
 
 bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -453,6 +528,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_dd,
   nequal_dd,
   NULL,
+  isNull_dd,
+  notNull_dd,
 };
 
 bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = {
@@ -464,6 +541,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval)
   NULL,
   nequal_str,
   like_str,
+  isNull_binary,
+  notNull_binary,
 };
 
 bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = {
@@ -475,6 +554,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva
   NULL,
   nequal_nchar,
   like_nchar,
+  isNull_nchar,
+  notNull_nchar,
 };
 
 bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index 9629f24cc2..1e8ceffae6 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -275,6 +275,11 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
   } else {
     pExpr->nSQLOptr = optrType;
     pExpr->pLeft = pLeft;
+
+    if (pRight == NULL) {
+      pRight = calloc(1, sizeof(tSQLExpr));
+    }
+
     pExpr->pRight = pRight;
   }
 
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index 373e57963c..35e36032d4 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -126,17 +126,17 @@ typedef union {
 #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo
 #define ParseARG_STORE yypParser->pInfo = pInfo
 #define YYFALLBACK 1
-#define YYNSTATE             244
-#define YYNRULE              225
+#define YYNSTATE             246
+#define YYNRULE              227
 #define YYNTOKEN             206
-#define YY_MAX_SHIFT         243
-#define YY_MIN_SHIFTREDUCE   403
-#define YY_MAX_SHIFTREDUCE   627
-#define YY_ERROR_ACTION      628
-#define YY_ACCEPT_ACTION     629
-#define YY_NO_ACTION         630
-#define YY_MIN_REDUCE        631
-#define YY_MAX_REDUCE        855
+#define YY_MAX_SHIFT         245
+#define YY_MIN_SHIFTREDUCE   407
+#define YY_MAX_SHIFTREDUCE   633
+#define YY_ERROR_ACTION      634
+#define YY_ACCEPT_ACTION     635
+#define YY_NO_ACTION         636
+#define YY_MIN_REDUCE        637
+#define YY_MAX_REDUCE        863
 /************* End control #defines *******************************************/
 
 /* Define the yytestcase() macro to be a no-op if is not already defined
@@ -202,121 +202,122 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (549)
+#define YY_ACTTAB_COUNT (556)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */   731,  444,  221,  729,  730,  629,  243,  510,  732,  445,
- /*    10 */   734,  735,  733,   41,   43,  526,   35,   36,  523,   11,
- /*    20 */   524,   29,  525,  444,  199,   39,   37,   40,   38,  155,
- /*    30 */   241,  445,  748,   34,   33,  219,  218,   32,   31,   30,
- /*    40 */    41,   43,  761,   35,   36,  136,  172,  173,   29,  137,
- /*    50 */    21,  199,   39,   37,   40,   38,  184,  141,  160,  843,
- /*    60 */    34,   33,  839,  772,   32,   31,   30,  404,  405,  406,
- /*    70 */   407,  408,  409,  410,  411,  412,  413,  414,  415,  242,
- /*    80 */    41,   43,  230,   35,   36,  746,   62,  137,   29,  137,
- /*    90 */    21,  199,   39,   37,   40,   38,  159,  843,   27,  842,
- /*   100 */    34,   33,   56,  838,   32,   31,   30,  105,   43,    8,
- /*   110 */    35,   36,   63,  115,  769,   29,  761,  527,  199,   39,
- /*   120 */    37,   40,   38,  168,  539,  747,  583,   34,   33,   18,
- /*   130 */   156,   32,   31,   30,   16,  210,  236,  235,  209,  208,
- /*   140 */   207,  234,  206,  233,  232,  231,  205,  727,  105,  715,
- /*   150 */   716,  717,  718,  719,  720,  721,  722,  723,  724,  725,
- /*   160 */   726,   35,   36,  798,  837,  194,   29,  177,  157,  199,
- /*   170 */    39,   37,   40,   38,  181,  180,   21,   21,   34,   33,
- /*   180 */   444,   12,   32,   31,   30,  164,  596,  750,  445,  587,
- /*   190 */   153,  590,  154,  593,  105,  164,  596,   21,   17,  587,
- /*   200 */   150,  590,  196,  593,   60,   26,   90,   89,  144,  169,
- /*   210 */   217,  747,  747,   16,  149,  236,  235,  161,  162,  167,
- /*   220 */   234,  198,  233,  232,  231,  142,  670,  161,  162,  128,
- /*   230 */   222,  542,  747,  164,  596,   17,  143,  587,  750,  590,
- /*   240 */   105,  593,   26,   39,   37,   40,   38,  100,  170,  145,
- /*   250 */   797,   34,   33,  101,   26,   32,   31,   30,   32,   31,
- /*   260 */    30,   78,  183,  564,  565,  161,  162,  230,  589,  152,
- /*   270 */   592,   76,   80,   85,   88,   79,  240,  239,   97,   34,
- /*   280 */    33,   82,   42,   32,   31,   30,  118,  119,   70,   66,
- /*   290 */    69,  237,   42,  595,  679,  163,   61,  128,  132,  130,
- /*   300 */    93,   92,   91,  595,  671,  187,  585,  128,  594,  588,
- /*   310 */   750,  591,  171,  534,   47,  216,  215,  146,  594,  555,
- /*   320 */   186,  147,  556,   46,  613,  148,   14,  597,   13,  139,
- /*   330 */    42,   13,   50,   48,    3,  135,   75,   74,  140,  516,
- /*   340 */   515,  595,  586,   46,   22,  138,  203,   10,    9,   51,
- /*   350 */    22,  852,  530,  528,  531,  529,  594,   87,   86,  749,
- /*   360 */   808,  807,  165,  804,  803,  166,  771,  741,  220,  776,
- /*   370 */   763,  778,  102,  790,  789,  116,  117,  114,  681,  204,
- /*   380 */   133,   24,  213,  678,  214,  851,   72,  850,  848,   26,
- /*   390 */   120,  699,   25,   23,  185,   95,  134,  668,   81,  551,
- /*   400 */   666,   83,   84,  664,  188,  663,  174,  129,  661,  660,
- /*   410 */   659,  658,  657,  649,  131,  655,  653,  192,   52,  651,
- /*   420 */   760,   57,   49,   58,  791,   44,  197,  195,  193,  191,
- /*   430 */   189,   28,  212,   77,  223,  224,  225,  226,  227,  228,
- /*   440 */   229,  238,  627,  176,  175,  626,  201,  178,  179,   53,
- /*   450 */   625,  618,  182,  536,   64,  151,  186,   67,  552,   55,
- /*   460 */   103,  158,  662,   59,  200,   94,   96,  123,  700,  121,
- /*   470 */   126,  106,  107,  122,  124,  125,  127,  112,  108,  109,
- /*   480 */   113,  745,  110,  656,  111,    1,    2,  190,    5,  557,
- /*   490 */   104,   19,    6,  598,   20,    4,   15,    7,   65,  485,
- /*   500 */   202,  481,  479,  478,  477,  474,  448,  211,   68,   45,
- /*   510 */    71,   73,   22,  512,  511,  509,   54,  469,  467,  459,
- /*   520 */   465,  461,  463,  457,  455,  484,  483,  482,  480,  476,
- /*   530 */   475,   46,  446,  419,  417,  631,  630,  630,  630,  630,
- /*   540 */   630,  630,  630,  630,  630,  630,  630,   98,   99,
+ /*     0 */   737,  448,   11,  735,  736,  635,  245,  448,  738,  449,
+ /*    10 */   740,  741,  739,   35,   36,  449,   37,   38,  155,  243,
+ /*    20 */   165,   29,  137,  136,  200,   41,   39,   43,   40,  105,
+ /*    30 */   514,  160,  851,   34,   33,  778,  137,   32,   31,   30,
+ /*    40 */    35,   36,  767,   37,   38,  159,  851,  165,   29,  767,
+ /*    50 */   105,  200,   41,   39,   43,   40,  185,  157,  221,  220,
+ /*    60 */    34,   33,  137,  156,   32,   31,   30,   35,   36,  448,
+ /*    70 */    37,   38,  850,  141,  165,   29,  756,  449,  200,   41,
+ /*    80 */    39,   43,   40,  197,   78,   60,  775,   34,   33,  232,
+ /*    90 */   232,   32,   31,   30,   21,   41,   39,   43,   40,   32,
+ /*   100 */    31,   30,   56,   34,   33,  847,  803,   32,   31,   30,
+ /*   110 */    21,   21,  105,  408,  409,  410,  411,  412,  413,  414,
+ /*   120 */   415,  416,  417,  418,  419,  244,  587,  169,   36,  753,
+ /*   130 */    37,   38,  223,   50,  165,   29,   21,   62,  200,   41,
+ /*   140 */    39,   43,   40,  170,  219,  753,  753,   34,   33,   27,
+ /*   150 */    51,   32,   31,   30,    8,   37,   38,   63,  115,  165,
+ /*   160 */    29,  101,  754,  200,   41,   39,   43,   40,  804,  224,
+ /*   170 */   195,  753,   34,   33,  168,  846,   32,   31,   30,   16,
+ /*   180 */   212,  238,  237,  211,  210,  209,  236,  208,  235,  234,
+ /*   190 */   233,  207,  733,  756,  721,  722,  723,  724,  725,  726,
+ /*   200 */   727,  728,  729,  730,  731,  732,  164,  600,   12,  239,
+ /*   210 */   591,   17,  594,  188,  597,  105,  164,  600,   26,  559,
+ /*   220 */   591,  845,  594,   46,  597,   34,   33,  150,  756,   32,
+ /*   230 */    31,   30,   21,   90,   89,  144,  568,  569,  161,  162,
+ /*   240 */   171,  149,  199,   76,   80,   85,   88,   79,  161,  162,
+ /*   250 */   164,  600,  546,   82,  591,  589,  594,  100,  597,  242,
+ /*   260 */   241,   97,   17,   16,   26,  238,  237,  752,  201,   26,
+ /*   270 */   236,   61,  235,  234,  233,  118,  119,   70,   66,   69,
+ /*   280 */   538,  676,  161,  162,  128,  530,  178,  187,  527,  184,
+ /*   290 */   528,  590,  529,  182,  181,  593,  152,  596,  132,  130,
+ /*   300 */    93,   92,   91,   42,  172,  543,  685,  218,  217,  128,
+ /*   310 */    18,  163,  677,   42,  599,  128,  173,  174,  560,  619,
+ /*   320 */   153,   47,   14,   13,  599,  592,  601,  595,  520,  598,
+ /*   330 */    13,  519,   46,  154,  205,   22,   75,   74,   22,  598,
+ /*   340 */    48,   10,    9,  534,  532,  535,  533,   42,   87,   86,
+ /*   350 */     3,  139,  860,  140,  755,  142,  143,  603,  599,  147,
+ /*   360 */   148,  146,  135,  145,  138,  814,  813,  166,  810,  809,
+ /*   370 */   167,  777,  747,  598,  222,  769,  782,  784,  102,  796,
+ /*   380 */   114,  116,  795,  117,  687,  206,  133,  531,  186,   26,
+ /*   390 */    24,   95,  215,  684,  216,  859,   72,  858,  856,  120,
+ /*   400 */   705,   25,  555,   23,  134,  674,   81,  672,   83,  189,
+ /*   410 */    84,  670,  193,  669,  175,   52,  129,  667,   49,  666,
+ /*   420 */   665,  106,  664,   44,  663,  107,  655,  131,  198,  766,
+ /*   430 */   196,  661,  659,  657,  194,   57,   58,  797,  192,  190,
+ /*   440 */    28,  214,   77,  225,  226,  227,  228,  229,  230,  203,
+ /*   450 */    53,  231,  240,  633,  151,  177,   64,   67,  176,  668,
+ /*   460 */   632,  179,  180,  631,  624,  187,  123,  183,  122,  706,
+ /*   470 */   121,  125,  124,   94,  127,  662,  126,   96,    1,    2,
+ /*   480 */   540,  112,  108,  109,  751,  110,  113,  111,   59,   55,
+ /*   490 */   556,  103,  158,   19,  191,   20,  561,  104,    5,  602,
+ /*   500 */     6,    4,  604,   15,  202,    7,  204,   65,  489,  485,
+ /*   510 */   483,  482,  481,  478,  452,  213,   68,   45,   71,   22,
+ /*   520 */   516,  515,  513,   54,  473,  471,  463,  469,  465,  467,
+ /*   530 */    73,  461,  459,  488,  487,  486,  484,  480,  479,   46,
+ /*   540 */   450,  423,  421,  637,  636,  636,   98,  636,  636,  636,
+ /*   550 */   636,  636,  636,  636,  636,   99,
 };
 static const YYCODETYPE yy_lookahead[] = {
- /*     0 */   226,    1,  210,  229,  230,  207,  208,    5,  234,    9,
- /*    10 */   236,  237,  238,   13,   14,    2,   16,   17,    5,  260,
- /*    20 */     7,   21,    9,    1,   24,   25,   26,   27,   28,  209,
- /*    30 */   210,    9,  240,   33,   34,   33,   34,   37,   38,   39,
- /*    40 */    13,   14,  244,   16,   17,  260,   33,   34,   21,  260,
- /*    50 */   210,   24,   25,   26,   27,   28,  258,  260,  269,  270,
- /*    60 */    33,   34,  260,  210,   37,   38,   39,   45,   46,   47,
- /*    70 */    48,   49,   50,   51,   52,   53,   54,   55,   56,   57,
- /*    80 */    13,   14,   78,   16,   17,  245,  247,  260,   21,  260,
- /*    90 */   210,   24,   25,   26,   27,   28,  269,  270,  259,  270,
- /*   100 */    33,   34,  102,  260,   37,   38,   39,  210,   14,   98,
- /*   110 */    16,   17,  101,  102,  261,   21,  244,  104,   24,   25,
- /*   120 */    26,   27,   28,  243,  103,  245,   99,   33,   34,  108,
- /*   130 */   258,   37,   38,   39,   85,   86,   87,   88,   89,   90,
- /*   140 */    91,   92,   93,   94,   95,   96,   97,  226,  210,  228,
- /*   150 */   229,  230,  231,  232,  233,  234,  235,  236,  237,  238,
- /*   160 */   239,   16,   17,  266,  260,  268,   21,  126,  227,   24,
- /*   170 */    25,   26,   27,   28,  133,  134,  210,  210,   33,   34,
- /*   180 */     1,   44,   37,   38,   39,    1,    2,  246,    9,    5,
- /*   190 */   260,    7,  260,    9,  210,    1,    2,  210,   98,    5,
- /*   200 */    63,    7,  264,    9,  266,  105,   69,   70,   71,  243,
- /*   210 */   243,  245,  245,   85,   77,   87,   88,   33,   34,  227,
- /*   220 */    92,   37,   94,   95,   96,  260,  214,   33,   34,  217,
- /*   230 */   243,   37,  245,    1,    2,   98,  260,    5,  246,    7,
- /*   240 */   210,    9,  105,   25,   26,   27,   28,   98,   63,  260,
- /*   250 */   266,   33,   34,  210,  105,   37,   38,   39,   37,   38,
- /*   260 */    39,   72,  125,  115,  116,   33,   34,   78,    5,  132,
- /*   270 */     7,   64,   65,   66,   67,   68,   60,   61,   62,   33,
- /*   280 */    34,   74,   98,   37,   38,   39,   64,   65,   66,   67,
- /*   290 */    68,  227,   98,  109,  214,   59,  266,  217,   64,   65,
- /*   300 */    66,   67,   68,  109,  214,  262,    1,  217,  124,    5,
- /*   310 */   246,    7,  127,   99,  103,  130,  131,  260,  124,   99,
- /*   320 */   106,  260,   99,  103,   99,  260,  103,   99,  103,  260,
- /*   330 */    98,  103,  103,  122,   98,  260,  128,  129,  260,   99,
- /*   340 */    99,  109,   37,  103,  103,  260,   99,  128,  129,  120,
- /*   350 */   103,  246,    5,    5,    7,    7,  124,   72,   73,  246,
- /*   360 */   241,  241,  241,  241,  241,  241,  210,  242,  241,  210,
- /*   370 */   244,  210,  210,  267,  267,  210,  210,  248,  210,  210,
- /*   380 */   210,  210,  210,  210,  210,  210,  210,  210,  210,  105,
- /*   390 */   210,  210,  210,  210,  244,   59,  210,  210,  210,  109,
- /*   400 */   210,  210,  210,  210,  263,  210,  210,  210,  210,  210,
- /*   410 */   210,  210,  210,  210,  210,  210,  210,  263,  119,  210,
- /*   420 */   257,  211,  121,  211,  211,  118,  113,  117,  112,  111,
- /*   430 */   110,  123,   75,   84,   83,   49,   80,   82,   53,   81,
- /*   440 */    79,   75,    5,    5,  135,    5,  211,  135,    5,  211,
- /*   450 */     5,   86,  126,   99,  215,  211,  106,  215,   99,  107,
- /*   460 */    98,    1,  211,  103,  100,  212,  212,  219,  225,  224,
- /*   470 */   221,  256,  255,  223,  222,  220,  218,  250,  254,  253,
- /*   480 */   249,  244,  252,  211,  251,  216,  213,   98,  114,   99,
- /*   490 */    98,  103,  114,   99,  103,   98,   98,   98,   72,    9,
- /*   500 */   100,    5,    5,    5,    5,    5,   76,   15,   72,   16,
- /*   510 */   129,  129,  103,    5,    5,   99,   98,    5,    5,    5,
- /*   520 */     5,    5,    5,    5,    5,    5,    5,    5,    5,    5,
- /*   530 */     5,  103,   76,   59,   58,    0,  271,  271,  271,  271,
- /*   540 */   271,  271,  271,  271,  271,  271,  271,   21,   21,  271,
- /*   550 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
+ /*     0 */   226,    1,  260,  229,  230,  207,  208,    1,  234,    9,
+ /*    10 */   236,  237,  238,   13,   14,    9,   16,   17,  209,  210,
+ /*    20 */    20,   21,  260,  260,   24,   25,   26,   27,   28,  210,
+ /*    30 */     5,  269,  270,   33,   34,  210,  260,   37,   38,   39,
+ /*    40 */    13,   14,  244,   16,   17,  269,  270,   20,   21,  244,
+ /*    50 */   210,   24,   25,   26,   27,   28,  258,  227,   33,   34,
+ /*    60 */    33,   34,  260,  258,   37,   38,   39,   13,   14,    1,
+ /*    70 */    16,   17,  270,  260,   20,   21,  246,    9,   24,   25,
+ /*    80 */    26,   27,   28,  264,   72,  266,  261,   33,   34,   78,
+ /*    90 */    78,   37,   38,   39,  210,   25,   26,   27,   28,   37,
+ /*   100 */    38,   39,  102,   33,   34,  260,  266,   37,   38,   39,
+ /*   110 */   210,  210,  210,   45,   46,   47,   48,   49,   50,   51,
+ /*   120 */    52,   53,   54,   55,   56,   57,   99,  243,   14,  245,
+ /*   130 */    16,   17,  210,  103,   20,   21,  210,  247,   24,   25,
+ /*   140 */    26,   27,   28,  243,  243,  245,  245,   33,   34,  259,
+ /*   150 */   120,   37,   38,   39,   98,   16,   17,  101,  102,   20,
+ /*   160 */    21,  210,  240,   24,   25,   26,   27,   28,  266,  243,
+ /*   170 */   268,  245,   33,   34,  227,  260,   37,   38,   39,   85,
+ /*   180 */    86,   87,   88,   89,   90,   91,   92,   93,   94,   95,
+ /*   190 */    96,   97,  226,  246,  228,  229,  230,  231,  232,  233,
+ /*   200 */   234,  235,  236,  237,  238,  239,    1,    2,   44,  227,
+ /*   210 */     5,   98,    7,  262,    9,  210,    1,    2,  105,   99,
+ /*   220 */     5,  260,    7,  103,    9,   33,   34,   63,  246,   37,
+ /*   230 */    38,   39,  210,   69,   70,   71,  115,  116,   33,   34,
+ /*   240 */    63,   77,   37,   64,   65,   66,   67,   68,   33,   34,
+ /*   250 */     1,    2,   37,   74,    5,    1,    7,   98,    9,   60,
+ /*   260 */    61,   62,   98,   85,  105,   87,   88,  245,   15,  105,
+ /*   270 */    92,  266,   94,   95,   96,   64,   65,   66,   67,   68,
+ /*   280 */    99,  214,   33,   34,  217,    2,  126,  106,    5,  125,
+ /*   290 */     7,   37,    9,  133,  134,    5,  132,    7,   64,   65,
+ /*   300 */    66,   67,   68,   98,  127,  103,  214,  130,  131,  217,
+ /*   310 */   108,   59,  214,   98,  109,  217,   33,   34,   99,   99,
+ /*   320 */   260,  103,  103,  103,  109,    5,   99,    7,   99,  124,
+ /*   330 */   103,   99,  103,  260,   99,  103,  128,  129,  103,  124,
+ /*   340 */   122,  128,  129,    5,    5,    7,    7,   98,   72,   73,
+ /*   350 */    98,  260,  246,  260,  246,  260,  260,  104,  109,  260,
+ /*   360 */   260,  260,  260,  260,  260,  241,  241,  241,  241,  241,
+ /*   370 */   241,  210,  242,  124,  241,  244,  210,  210,  210,  267,
+ /*   380 */   248,  210,  267,  210,  210,  210,  210,  104,  244,  105,
+ /*   390 */   210,   59,  210,  210,  210,  210,  210,  210,  210,  210,
+ /*   400 */   210,  210,  109,  210,  210,  210,  210,  210,  210,  263,
+ /*   410 */   210,  210,  263,  210,  210,  119,  210,  210,  121,  210,
+ /*   420 */   210,  256,  210,  118,  210,  255,  210,  210,  113,  257,
+ /*   430 */   117,  210,  210,  210,  112,  211,  211,  211,  111,  110,
+ /*   440 */   123,   75,   84,   83,   49,   80,   82,   53,   81,  211,
+ /*   450 */   211,   79,   75,    5,  211,    5,  215,  215,  135,  211,
+ /*   460 */     5,  135,    5,    5,   86,  106,  219,  126,  223,  225,
+ /*   470 */   224,  220,  222,  212,  218,  211,  221,  212,  216,  213,
+ /*   480 */    99,  250,  254,  253,  244,  252,  249,  251,  103,  107,
+ /*   490 */    99,   98,    1,  103,   98,  103,   99,   98,  114,   99,
+ /*   500 */   114,   98,  104,   98,  100,   98,  100,   72,    9,    5,
+ /*   510 */     5,    5,    5,    5,   76,   15,   72,   16,  129,  103,
+ /*   520 */     5,    5,   99,   98,    5,    5,    5,    5,    5,    5,
+ /*   530 */   129,    5,    5,    5,    5,    5,    5,    5,    5,  103,
+ /*   540 */    76,   59,   58,    0,  271,  271,   21,  271,  271,  271,
+ /*   550 */   271,  271,  271,  271,  271,   21,  271,  271,  271,  271,
  /*   560 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
  /*   570 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
  /*   580 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
@@ -336,83 +337,84 @@ static const YYCODETYPE yy_lookahead[] = {
  /*   720 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
  /*   730 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
  /*   740 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
- /*   750 */   271,  271,  271,  271,  271,
+ /*   750 */   271,  271,  271,  271,  271,  271,  271,  271,  271,  271,
+ /*   760 */   271,  271,
 };
-#define YY_SHIFT_COUNT    (243)
+#define YY_SHIFT_COUNT    (245)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (535)
+#define YY_SHIFT_MAX      (543)
 static const unsigned short int yy_shift_ofst[] = {
- /*     0 */   137,   49,  128,  184,  232,  179,  179,  179,  179,  179,
- /*    10 */   179,    0,   22,  232,   13,   13,   13,  100,  179,  179,
- /*    20 */   179,  179,  179,  189,    4,    4,  549,  194,  232,  232,
- /*    30 */   232,  232,  232,  232,  232,  232,  232,  232,  232,  232,
- /*    40 */   232,  232,  232,  232,  232,   13,   13,    2,    2,    2,
- /*    50 */     2,    2,    2,   11,    2,  149,  179,  179,  179,  179,
- /*    60 */   148,  148,   21,  179,  179,  179,  179,  179,  179,  179,
- /*    70 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    80 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*    90 */   179,  179,  179,  179,  179,  179,  179,  179,  179,  179,
- /*   100 */   284,  336,  336,  290,  290,  336,  299,  301,  307,  313,
- /*   110 */   310,  316,  318,  320,  308,  284,  336,  336,  357,  357,
- /*   120 */   336,  349,  351,  386,  356,  355,  385,  358,  361,  336,
- /*   130 */   366,  336,  366,  549,  549,   27,   67,   67,   67,   94,
- /*   140 */   145,  218,  218,  218,  207,  246,  246,  246,  246,  222,
- /*   150 */   234,  185,   41,  221,  221,  216,  214,  220,  223,  225,
- /*   160 */   228,  263,  304,  305,  236,  211,  229,  240,  241,  247,
- /*   170 */   208,  219,  347,  348,  285,  437,  309,  438,  440,  312,
- /*   180 */   443,  445,  365,  326,  350,  354,  352,  360,  359,  362,
- /*   190 */   460,  389,  390,  392,  388,  374,  391,  378,  394,  397,
- /*   200 */   398,  364,  399,  400,  426,  490,  496,  497,  498,  499,
- /*   210 */   500,  430,  492,  436,  493,  381,  382,  409,  508,  509,
- /*   220 */   416,  418,  409,  512,  513,  514,  515,  516,  517,  518,
- /*   230 */   519,  520,  521,  522,  523,  524,  525,  428,  456,  526,
- /*   240 */   527,  474,  476,  535,
+ /*     0 */   164,   94,  178,  205,  249,    6,    6,    6,    6,    6,
+ /*    10 */     6,    0,   68,  249,  283,  283,  283,  113,    6,    6,
+ /*    20 */     6,    6,    6,   12,   11,   11,  556,  215,  249,  249,
+ /*    30 */   249,  249,  249,  249,  249,  249,  249,  249,  249,  249,
+ /*    40 */   249,  249,  249,  249,  249,  283,  283,   25,   25,   25,
+ /*    50 */    25,   25,   25,   56,   25,  159,    6,    6,    6,    6,
+ /*    60 */   121,  121,  202,    6,    6,    6,    6,    6,    6,    6,
+ /*    70 */     6,    6,    6,    6,    6,    6,    6,    6,    6,    6,
+ /*    80 */     6,    6,    6,    6,    6,    6,    6,    6,    6,    6,
+ /*    90 */     6,    6,    6,    6,    6,    6,    6,    6,    6,    6,
+ /*   100 */   284,  332,  332,  293,  293,  332,  296,  297,  305,  315,
+ /*   110 */   313,  322,  327,  329,  317,  284,  332,  332,  366,  366,
+ /*   120 */   332,  358,  360,  395,  365,  364,  394,  367,  372,  332,
+ /*   130 */   377,  332,  377,  556,  556,   27,   54,   54,   54,  114,
+ /*   140 */   139,   70,   70,   70,  179,  192,  192,  192,  192,  211,
+ /*   150 */   234,  177,  160,   62,   62,  199,  181,  120,  219,  220,
+ /*   160 */   227,  290,  320,  254,  252,  253,  218,   30,  229,  232,
+ /*   170 */   235,  208,  213,  338,  339,  276,  448,  323,  450,  455,
+ /*   180 */   326,  457,  458,  378,  341,  359,  381,  382,  385,  391,
+ /*   190 */   393,  491,  396,  397,  399,  390,  384,  392,  386,  400,
+ /*   200 */   403,  398,  405,  404,  407,  406,  435,  499,  504,  505,
+ /*   210 */   506,  507,  508,  438,  500,  444,  501,  389,  401,  416,
+ /*   220 */   515,  516,  423,  425,  416,  519,  520,  521,  522,  523,
+ /*   230 */   524,  526,  527,  528,  529,  530,  531,  532,  533,  436,
+ /*   240 */   464,  525,  534,  482,  484,  543,
 };
 #define YY_REDUCE_COUNT (134)
-#define YY_REDUCE_MIN   (-241)
-#define YY_REDUCE_MAX   (273)
+#define YY_REDUCE_MIN   (-258)
+#define YY_REDUCE_MAX   (266)
 static const short yy_reduce_ofst[] = {
- /*     0 */  -202,  -79, -226, -211, -173, -103,  -62, -120,  -34,  -33,
- /*    10 */   -13, -147, -180, -171,  -59,   -8,   64, -128,   43,  -16,
- /*    20 */    30, -208, -160,   12,   80,   90, -161, -241, -215, -203,
- /*    30 */  -198, -157,  -96,  -70,  -68,  -35,  -24,  -11,   57,   61,
- /*    40 */    65,   69,   75,   78,   85,  105,  113,  119,  120,  121,
- /*    50 */   122,  123,  124,  125,  127,  126,  156,  159,  161,  162,
- /*    60 */   106,  107,  129,  165,  166,  168,  169,  170,  171,  172,
- /*    70 */   173,  174,  175,  176,  177,  178,  180,  181,  182,  183,
- /*    80 */   186,  187,  188,  190,  191,  192,  193,  195,  196,  197,
- /*    90 */   198,  199,  200,  201,  202,  203,  204,  205,  206,  209,
- /*   100 */   150,  210,  212,  141,  154,  213,  163,  215,  217,  224,
- /*   110 */   226,  230,  233,  227,  231,  237,  235,  238,  239,  242,
- /*   120 */   244,  243,  245,  250,  248,  252,  255,  249,  258,  251,
- /*   130 */   253,  272,  254,  269,  273,
+ /*     0 */  -202,  -34, -226, -238, -224,  -98, -181, -116, -100,  -99,
+ /*    10 */   -74, -175, -191, -198, -170,  -53,  -18, -195,  -49, -160,
+ /*    20 */     5,  -78,   22,   67,   92,   98, -110, -258, -237, -187,
+ /*    30 */  -155,  -85,  -39,   60,   73,   91,   93,   95,   96,   99,
+ /*    40 */   100,  101,  102,  103,  104,  106,  108,  124,  125,  126,
+ /*    50 */   127,  128,  129,  130,  133,  131,  161,  166,  167,  168,
+ /*    60 */   112,  115,  132,  171,  173,  174,  175,  176,  180,  182,
+ /*    70 */   183,  184,  185,  186,  187,  188,  189,  190,  191,  193,
+ /*    80 */   194,  195,  196,  197,  198,  200,  201,  203,  204,  206,
+ /*    90 */   207,  209,  210,  212,  214,  216,  217,  221,  222,  223,
+ /*   100 */   144,  224,  225,  146,  149,  226,  172,  165,  170,  228,
+ /*   110 */   230,  233,  236,  231,  237,  240,  238,  239,  241,  242,
+ /*   120 */   243,  244,  246,  245,  247,  250,  251,  255,  256,  248,
+ /*   130 */   261,  264,  265,  262,  266,
 };
 static const YYACTIONTYPE yy_default[] = {
- /*     0 */   628,  680,  669,  845,  845,  628,  628,  628,  628,  628,
- /*    10 */   628,  773,  646,  845,  628,  628,  628,  628,  628,  628,
- /*    20 */   628,  628,  628,  682,  682,  682,  768,  628,  628,  628,
- /*    30 */   628,  628,  628,  628,  628,  628,  628,  628,  628,  628,
- /*    40 */   628,  628,  628,  628,  628,  628,  628,  628,  628,  628,
- /*    50 */   628,  628,  628,  628,  628,  628,  628,  775,  777,  628,
- /*    60 */   794,  794,  766,  628,  628,  628,  628,  628,  628,  628,
- /*    70 */   628,  628,  628,  628,  628,  628,  628,  628,  628,  628,
- /*    80 */   628,  667,  628,  665,  628,  628,  628,  628,  628,  628,
- /*    90 */   628,  628,  628,  628,  628,  628,  628,  654,  628,  628,
- /*   100 */   628,  648,  648,  628,  628,  648,  801,  805,  799,  787,
- /*   110 */   795,  786,  782,  781,  809,  628,  648,  648,  677,  677,
- /*   120 */   648,  698,  696,  694,  686,  692,  688,  690,  684,  648,
- /*   130 */   675,  648,  675,  714,  728,  628,  810,  844,  800,  828,
- /*   140 */   827,  840,  834,  833,  628,  832,  831,  830,  829,  628,
- /*   150 */   628,  628,  628,  836,  835,  628,  628,  628,  628,  628,
- /*   160 */   628,  628,  628,  628,  812,  806,  802,  628,  628,  628,
- /*   170 */   628,  628,  628,  628,  628,  628,  628,  628,  628,  628,
- /*   180 */   628,  628,  628,  628,  765,  628,  628,  774,  628,  628,
- /*   190 */   628,  628,  628,  628,  796,  628,  788,  628,  628,  628,
- /*   200 */   628,  628,  628,  742,  628,  628,  628,  628,  628,  628,
- /*   210 */   628,  628,  628,  628,  628,  628,  628,  849,  628,  628,
- /*   220 */   628,  736,  847,  628,  628,  628,  628,  628,  628,  628,
- /*   230 */   628,  628,  628,  628,  628,  628,  628,  701,  628,  652,
- /*   240 */   650,  628,  644,  628,
+ /*     0 */   634,  686,  675,  853,  853,  634,  634,  634,  634,  634,
+ /*    10 */   634,  779,  652,  853,  634,  634,  634,  634,  634,  634,
+ /*    20 */   634,  634,  634,  688,  688,  688,  774,  634,  634,  634,
+ /*    30 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  634,
+ /*    40 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  634,
+ /*    50 */   634,  634,  634,  634,  634,  634,  634,  781,  783,  634,
+ /*    60 */   800,  800,  772,  634,  634,  634,  634,  634,  634,  634,
+ /*    70 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  634,
+ /*    80 */   634,  673,  634,  671,  634,  634,  634,  634,  634,  634,
+ /*    90 */   634,  634,  634,  634,  634,  634,  634,  660,  634,  634,
+ /*   100 */   634,  654,  654,  634,  634,  654,  807,  811,  805,  793,
+ /*   110 */   801,  792,  788,  787,  815,  634,  654,  654,  683,  683,
+ /*   120 */   654,  704,  702,  700,  692,  698,  694,  696,  690,  654,
+ /*   130 */   681,  654,  681,  720,  734,  634,  816,  852,  806,  842,
+ /*   140 */   841,  848,  840,  839,  634,  835,  836,  838,  837,  634,
+ /*   150 */   634,  634,  634,  844,  843,  634,  634,  634,  634,  634,
+ /*   160 */   634,  634,  634,  634,  818,  634,  812,  808,  634,  634,
+ /*   170 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  634,
+ /*   180 */   634,  634,  634,  634,  634,  771,  634,  634,  780,  634,
+ /*   190 */   634,  634,  634,  634,  634,  802,  634,  794,  634,  634,
+ /*   200 */   634,  634,  634,  634,  634,  748,  634,  634,  634,  634,
+ /*   210 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  857,
+ /*   220 */   634,  634,  634,  742,  855,  634,  634,  634,  634,  634,
+ /*   230 */   634,  634,  634,  634,  634,  634,  634,  634,  634,  707,
+ /*   240 */   634,  658,  656,  634,  650,  634,
 };
 /********** End of lemon-generated parsing tables *****************************/
 
@@ -1198,35 +1200,37 @@ static const char *const yyRuleName[] = {
  /* 193 */ "expr ::= BOOL",
  /* 194 */ "expr ::= ID LP exprlist RP",
  /* 195 */ "expr ::= ID LP STAR RP",
- /* 196 */ "expr ::= expr AND expr",
- /* 197 */ "expr ::= expr OR expr",
+ /* 196 */ "expr ::= expr IS NULL",
+ /* 197 */ "expr ::= expr IS NOT NULL",
  /* 198 */ "expr ::= expr LT expr",
  /* 199 */ "expr ::= expr GT expr",
  /* 200 */ "expr ::= expr LE expr",
  /* 201 */ "expr ::= expr GE expr",
  /* 202 */ "expr ::= expr NE expr",
  /* 203 */ "expr ::= expr EQ expr",
- /* 204 */ "expr ::= expr PLUS expr",
- /* 205 */ "expr ::= expr MINUS expr",
- /* 206 */ "expr ::= expr STAR expr",
- /* 207 */ "expr ::= expr SLASH expr",
- /* 208 */ "expr ::= expr REM expr",
- /* 209 */ "expr ::= expr LIKE expr",
- /* 210 */ "expr ::= expr IN LP exprlist RP",
- /* 211 */ "exprlist ::= exprlist COMMA expritem",
- /* 212 */ "exprlist ::= expritem",
- /* 213 */ "expritem ::= expr",
- /* 214 */ "expritem ::=",
- /* 215 */ "cmd ::= RESET QUERY CACHE",
- /* 216 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
- /* 217 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
- /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
- /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
- /* 220 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
- /* 221 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
- /* 222 */ "cmd ::= KILL CONNECTION INTEGER",
- /* 223 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
- /* 224 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
+ /* 204 */ "expr ::= expr AND expr",
+ /* 205 */ "expr ::= expr OR expr",
+ /* 206 */ "expr ::= expr PLUS expr",
+ /* 207 */ "expr ::= expr MINUS expr",
+ /* 208 */ "expr ::= expr STAR expr",
+ /* 209 */ "expr ::= expr SLASH expr",
+ /* 210 */ "expr ::= expr REM expr",
+ /* 211 */ "expr ::= expr LIKE expr",
+ /* 212 */ "expr ::= expr IN LP exprlist RP",
+ /* 213 */ "exprlist ::= exprlist COMMA expritem",
+ /* 214 */ "exprlist ::= expritem",
+ /* 215 */ "expritem ::= expr",
+ /* 216 */ "expritem ::=",
+ /* 217 */ "cmd ::= RESET QUERY CACHE",
+ /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
+ /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
+ /* 220 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
+ /* 221 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
+ /* 222 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
+ /* 223 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
+ /* 224 */ "cmd ::= KILL CONNECTION INTEGER",
+ /* 225 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
+ /* 226 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
 };
 #endif /* NDEBUG */
 
@@ -1880,35 +1884,37 @@ static const struct {
   {  260,   -1 }, /* (193) expr ::= BOOL */
   {  260,   -4 }, /* (194) expr ::= ID LP exprlist RP */
   {  260,   -4 }, /* (195) expr ::= ID LP STAR RP */
-  {  260,   -3 }, /* (196) expr ::= expr AND expr */
-  {  260,   -3 }, /* (197) expr ::= expr OR expr */
+  {  260,   -3 }, /* (196) expr ::= expr IS NULL */
+  {  260,   -4 }, /* (197) expr ::= expr IS NOT NULL */
   {  260,   -3 }, /* (198) expr ::= expr LT expr */
   {  260,   -3 }, /* (199) expr ::= expr GT expr */
   {  260,   -3 }, /* (200) expr ::= expr LE expr */
   {  260,   -3 }, /* (201) expr ::= expr GE expr */
   {  260,   -3 }, /* (202) expr ::= expr NE expr */
   {  260,   -3 }, /* (203) expr ::= expr EQ expr */
-  {  260,   -3 }, /* (204) expr ::= expr PLUS expr */
-  {  260,   -3 }, /* (205) expr ::= expr MINUS expr */
-  {  260,   -3 }, /* (206) expr ::= expr STAR expr */
-  {  260,   -3 }, /* (207) expr ::= expr SLASH expr */
-  {  260,   -3 }, /* (208) expr ::= expr REM expr */
-  {  260,   -3 }, /* (209) expr ::= expr LIKE expr */
-  {  260,   -5 }, /* (210) expr ::= expr IN LP exprlist RP */
-  {  269,   -3 }, /* (211) exprlist ::= exprlist COMMA expritem */
-  {  269,   -1 }, /* (212) exprlist ::= expritem */
-  {  270,   -1 }, /* (213) expritem ::= expr */
-  {  270,    0 }, /* (214) expritem ::= */
-  {  208,   -3 }, /* (215) cmd ::= RESET QUERY CACHE */
-  {  208,   -7 }, /* (216) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
-  {  208,   -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
-  {  208,   -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
-  {  208,   -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
-  {  208,   -8 }, /* (220) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
-  {  208,   -9 }, /* (221) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
-  {  208,   -3 }, /* (222) cmd ::= KILL CONNECTION INTEGER */
-  {  208,   -5 }, /* (223) cmd ::= KILL STREAM INTEGER COLON INTEGER */
-  {  208,   -5 }, /* (224) cmd ::= KILL QUERY INTEGER COLON INTEGER */
+  {  260,   -3 }, /* (204) expr ::= expr AND expr */
+  {  260,   -3 }, /* (205) expr ::= expr OR expr */
+  {  260,   -3 }, /* (206) expr ::= expr PLUS expr */
+  {  260,   -3 }, /* (207) expr ::= expr MINUS expr */
+  {  260,   -3 }, /* (208) expr ::= expr STAR expr */
+  {  260,   -3 }, /* (209) expr ::= expr SLASH expr */
+  {  260,   -3 }, /* (210) expr ::= expr REM expr */
+  {  260,   -3 }, /* (211) expr ::= expr LIKE expr */
+  {  260,   -5 }, /* (212) expr ::= expr IN LP exprlist RP */
+  {  269,   -3 }, /* (213) exprlist ::= exprlist COMMA expritem */
+  {  269,   -1 }, /* (214) exprlist ::= expritem */
+  {  270,   -1 }, /* (215) expritem ::= expr */
+  {  270,    0 }, /* (216) expritem ::= */
+  {  208,   -3 }, /* (217) cmd ::= RESET QUERY CACHE */
+  {  208,   -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+  {  208,   -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+  {  208,   -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+  {  208,   -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+  {  208,   -8 }, /* (222) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+  {  208,   -9 }, /* (223) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+  {  208,   -3 }, /* (224) cmd ::= KILL CONNECTION INTEGER */
+  {  208,   -5 }, /* (225) cmd ::= KILL STREAM INTEGER COLON INTEGER */
+  {  208,   -5 }, /* (226) cmd ::= KILL QUERY INTEGER COLON INTEGER */
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
@@ -2570,7 +2576,7 @@ static void yy_reduce(
         break;
       case 168: /* having_opt ::= */
       case 178: /* where_opt ::= */ yytestcase(yyruleno==178);
-      case 214: /* expritem ::= */ yytestcase(yyruleno==214);
+      case 216: /* expritem ::= */ yytestcase(yyruleno==216);
 {yymsp[1].minor.yy66 = 0;}
         break;
       case 169: /* having_opt ::= HAVING expr */
@@ -2643,24 +2649,20 @@ static void yy_reduce(
   yymsp[0].minor.yy66 = yylhsminor.yy66;
         break;
       case 194: /* expr ::= ID LP exprlist RP */
-{
-  yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
-}
+{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
   yymsp[-3].minor.yy66 = yylhsminor.yy66;
         break;
       case 195: /* expr ::= ID LP STAR RP */
-{
-  yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type);
-}
+{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
   yymsp[-3].minor.yy66 = yylhsminor.yy66;
         break;
-      case 196: /* expr ::= expr AND expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
+      case 196: /* expr ::= expr IS NULL */
+{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);}
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 197: /* expr ::= expr OR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
-  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+      case 197: /* expr ::= expr IS NOT NULL */
+{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);}
+  yymsp[-3].minor.yy66 = yylhsminor.yy66;
         break;
       case 198: /* expr ::= expr LT expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);}
@@ -2686,57 +2688,65 @@ static void yy_reduce(
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);}
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 204: /* expr ::= expr PLUS expr */
+      case 204: /* expr ::= expr AND expr */
+{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
+  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+        break;
+      case 205: /* expr ::= expr OR expr */
+{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
+  yymsp[-2].minor.yy66 = yylhsminor.yy66;
+        break;
+      case 206: /* expr ::= expr PLUS expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS);  }
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 205: /* expr ::= expr MINUS expr */
+      case 207: /* expr ::= expr MINUS expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); }
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 206: /* expr ::= expr STAR expr */
+      case 208: /* expr ::= expr STAR expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR);  }
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 207: /* expr ::= expr SLASH expr */
+      case 209: /* expr ::= expr SLASH expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);}
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 208: /* expr ::= expr REM expr */
+      case 210: /* expr ::= expr REM expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM);   }
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 209: /* expr ::= expr LIKE expr */
+      case 211: /* expr ::= expr LIKE expr */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE);  }
   yymsp[-2].minor.yy66 = yylhsminor.yy66;
         break;
-      case 210: /* expr ::= expr IN LP exprlist RP */
+      case 212: /* expr ::= expr IN LP exprlist RP */
 {yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); }
   yymsp[-4].minor.yy66 = yylhsminor.yy66;
         break;
-      case 211: /* exprlist ::= exprlist COMMA expritem */
+      case 213: /* exprlist ::= exprlist COMMA expritem */
 {yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);}
   yymsp[-2].minor.yy224 = yylhsminor.yy224;
         break;
-      case 212: /* exprlist ::= expritem */
+      case 214: /* exprlist ::= expritem */
 {yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);}
   yymsp[0].minor.yy224 = yylhsminor.yy224;
         break;
-      case 213: /* expritem ::= expr */
+      case 215: /* expritem ::= expr */
 {yylhsminor.yy66 = yymsp[0].minor.yy66;}
   yymsp[0].minor.yy66 = yylhsminor.yy66;
         break;
-      case 215: /* cmd ::= RESET QUERY CACHE */
+      case 217: /* cmd ::= RESET QUERY CACHE */
 { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
         break;
-      case 216: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+      case 218: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 217: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+      case 219: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2747,14 +2757,14 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 218: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+      case 220: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 219: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+      case 221: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
@@ -2765,7 +2775,7 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 220: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+      case 222: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
 
@@ -2779,7 +2789,7 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 221: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+      case 223: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
 {
     yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
 
@@ -2791,13 +2801,13 @@ static void yy_reduce(
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
-      case 222: /* cmd ::= KILL CONNECTION INTEGER */
+      case 224: /* cmd ::= KILL CONNECTION INTEGER */
 {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);}
         break;
-      case 223: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
+      case 225: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);}
         break;
-      case 224: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
+      case 226: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
 {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);}
         break;
       default:
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index d829a85754..5fece58ef7 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -2071,13 +2071,17 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) {
     if (keyInfo.pTable != NULL) {
       totalNumOfTable++;
       taosArrayPush(pGroup, &keyInfo);
+    } else {
+      taosArrayRemove(groupList->pGroupList, j);
+      numOfGroups -= 1;
+      j -= 1;
     }
   }
 
   // window does not being updated, so set the original
   if (window.skey == INT64_MAX && window.ekey == INT64_MIN) {
     window = TSWINDOW_INITIALIZER;
-    assert(totalNumOfTable == 0);
+    assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0);
   }
 
   groupList->numOfTables = totalNumOfTable;
@@ -2398,6 +2402,14 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) {
     val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
   }
 
+  if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) {
+    if (pInfo->optr == TSDB_RELATION_ISNULL) {
+      return (val == NULL) || isNull(val, pInfo->sch.type);
+    } else if (pInfo->optr == TSDB_RELATION_NOTNULL) {
+      return (val != NULL) && (!isNull(val, pInfo->sch.type));
+    }
+  }
+
   int32_t ret = 0;
   if (val == NULL) { //the val is possible to be null, so check it out carefully
     ret = -1; // val is missing in table tags value pairs

From 25b2cec2f5aee6db59745c96c58812c2c96e6f7c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 13:28:08 +0800
Subject: [PATCH 05/56] [td-1369]

---
 src/inc/taosdef.h | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 1a40f3b56d..7437eceecb 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -209,21 +209,24 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_RELATION_GREATER_EQUAL 5
 #define TSDB_RELATION_NOT_EQUAL   6
 #define TSDB_RELATION_LIKE        7
-#define TSDB_RELATION_IN          8
+#define TSDB_RELATION_ISNULL      8
+#define TSDB_RELATION_NOTNULL     9
+#define TSDB_RELATION_IN          10
 
-#define TSDB_RELATION_AND         9
-#define TSDB_RELATION_OR          10
-#define TSDB_RELATION_NOT         11
+#define TSDB_RELATION_AND         11
+#define TSDB_RELATION_OR          12
+#define TSDB_RELATION_NOT         13
 
-#define TSDB_BINARY_OP_ADD        12
-#define TSDB_BINARY_OP_SUBTRACT   13
-#define TSDB_BINARY_OP_MULTIPLY   14
-#define TSDB_BINARY_OP_DIVIDE     15
-#define TSDB_BINARY_OP_REMAINDER  16
+#define TSDB_BINARY_OP_ADD        30
+#define TSDB_BINARY_OP_SUBTRACT   31
+#define TSDB_BINARY_OP_MULTIPLY   32
+#define TSDB_BINARY_OP_DIVIDE     33
+#define TSDB_BINARY_OP_REMAINDER  34
 #define TS_PATH_DELIMITER_LEN     1
 
 #define TSDB_UNI_LEN              24
 #define TSDB_USER_LEN             TSDB_UNI_LEN
+
 // ACCOUNT is a 32 bit positive integer
 // this is the length of its string representation
 // including the terminator zero

From 5b6ca20f0bffd9a6e34c7af177e8aad77d2af7dd Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 18:15:59 +0800
Subject: [PATCH 06/56] [td-225] refactor codes.

---
 src/client/inc/tscUtil.h                  |   7 +-
 src/client/src/tscSubquery.c              |  13 +-
 src/client/src/tscUtil.c                  |   3 +-
 src/query/src/qExecutor.c                 |   2 +-
 tests/script/general/parser/join.sim      |   2 +
 tests/script/general/parser/testSuite.sim | 168 +++++++++++-----------
 6 files changed, 91 insertions(+), 104 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index 9b31b8fc6a..748b19069b 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -69,17 +69,12 @@ typedef struct SJoinSupporter {
   SSubqueryState* pState;
   SSqlObj*        pObj;           // parent SqlObj
   int32_t         subqueryIndex;  // index of sub query
-  char            intervalTimeUnit;
-  char            slidingTimeUnit;
-  int64_t         intervalTime;   // interval time
-  int64_t         slidingTime;    // sliding time
   SLimitVal       limit;          // limit info
-  uint64_t        uid;            // query meter uid
+  uint64_t        uid;            // query table uid
   SArray*         colList;        // previous query information, no need to use this attribute, and the corresponding attribution
   SArray*         exprList;
   SFieldInfo      fieldsInfo;
   STagCond        tagCond;
-  SSqlGroupbyExpr groupbyExpr;
   struct STSBuf*  pTSBuf;          // the TSBuf struct that holds the compressed timestamp array
   FILE*           f;               // temporary file in order to create TSBuf
   char            path[PATH_MAX];  // temporary file path, todo dynamic allocate memory
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index e264fa9b33..c4b07f7813 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -178,10 +178,6 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
   pSupporter->subqueryIndex = index;
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
   
-  pSupporter->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
-  pSupporter->slidingTime = pQueryInfo->slidingTimeUnit;
-  pSupporter->intervalTime = pQueryInfo->intervalTime;
-  pSupporter->slidingTime = pQueryInfo->slidingTime;
   pSupporter->limit = pQueryInfo->limit;
 
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index);
@@ -311,18 +307,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
     // set the second stage sub query for join process
     TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE);
 
-    pQueryInfo->intervalTimeUnit = pSupporter->intervalTimeUnit;
-    pQueryInfo->slidingTimeUnit = pSupporter->slidingTimeUnit;
-    pQueryInfo->intervalTime = pSupporter->intervalTime;
-    pQueryInfo->slidingTime = pSupporter->slidingTime;
-    pQueryInfo->groupbyExpr = pSupporter->groupbyExpr;
-    
     tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
   
     pQueryInfo->colList = pSupporter->colList;
     pQueryInfo->exprList = pSupporter->exprList;
     pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
-    
+
     pSupporter->exprList = NULL;
     pSupporter->colList = NULL;
     memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
@@ -1221,7 +1211,6 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     pNewQueryInfo->limit.offset = 0;
 
     // backup the data and clear it in the sqlcmd object
-    pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr;
     memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
     
     tscInitQueryInfo(pNewQueryInfo);
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index ad87825e34..5d50d7791a 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1534,7 +1534,6 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
   pQueryInfo->exprList   = taosArrayInit(4, POINTER_BYTES);
   pQueryInfo->colList    = taosArrayInit(4, POINTER_BYTES);
   pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX;
-  pQueryInfo->window     = TSWINDOW_INITIALIZER;
 }
 
 int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
@@ -1555,6 +1554,8 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
   }
 
   tscInitQueryInfo(pQueryInfo);
+
+  pQueryInfo->window = TSWINDOW_INITIALIZER;
   pQueryInfo->msg = pCmd->payload;  // pointer to the parent error message buffer
 
   pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo;
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 2ca2545818..fc932b8999 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1177,7 +1177,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) {
 
 #if defined(_DEBUG_VIEW)
   printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n",
-         elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder,
+         elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder,
          pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex);
 #endif
 
diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim
index ef3245ccaf..254571bda1 100644
--- a/tests/script/general/parser/join.sim
+++ b/tests/script/general/parser/join.sim
@@ -205,10 +205,12 @@ if $rows != 9 then
 endi
 
 if $data00 != @70-01-01 08:01:40.100@ then
+  print $data00
   return -1
 endi
 
 if $data10 != @70-01-01 08:01:40.200@ then
+  print $data10
   return -1
 endi
 
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index 6790564cc7..721eb0bfa1 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -1,87 +1,87 @@
-sleep 2000
-run general/parser/alter.sim
-sleep 2000
-run general/parser/alter1.sim
-sleep 2000
-run general/parser/alter_stable.sim
-sleep 2000
-run general/parser/auto_create_tb.sim
-sleep 2000
-run general/parser/auto_create_tb_drop_tb.sim
-sleep 2000
-run general/parser/col_arithmetic_operation.sim
-sleep 2000
-run general/parser/columnValue.sim
-sleep 2000
-run general/parser/commit.sim
-sleep 2000
-run general/parser/create_db.sim
-sleep 2000
-run general/parser/create_mt.sim
-sleep 2000
-run general/parser/create_tb.sim
-sleep 2000
-run general/parser/dbtbnameValidate.sim
-sleep 2000
-run general/parser/fill.sim
-sleep 2000
-run general/parser/fill_stb.sim
-sleep 2000
-#run general/parser/fill_us.sim               #
-sleep 2000
-run general/parser/first_last.sim
-sleep 2000
-run general/parser/import_commit1.sim
-sleep 2000
-run general/parser/import_commit2.sim
-sleep 2000
-run general/parser/import_commit3.sim
-sleep 2000
-#run general/parser/import_file.sim
-sleep 2000
-run general/parser/insert_tb.sim
-sleep 2000
-run general/parser/tags_dynamically_specifiy.sim
-sleep 2000
-run general/parser/interp.sim
-sleep 2000
-run general/parser/lastrow.sim
-sleep 2000
-run general/parser/limit.sim
-sleep 2000
-run general/parser/limit1.sim
-sleep 2000
-run general/parser/limit1_tblocks100.sim
-sleep 2000
-run general/parser/limit2.sim
-sleep 2000
-run general/parser/mixed_blocks.sim
-sleep 2000
-run general/parser/nchar.sim
-sleep 2000
-run general/parser/null_char.sim
-sleep 2000
-run general/parser/selectResNum.sim
-sleep 2000
-run general/parser/select_across_vnodes.sim
-sleep 2000
-run general/parser/select_from_cache_disk.sim
-sleep 2000
-run general/parser/set_tag_vals.sim
-sleep 2000
-run general/parser/single_row_in_tb.sim
-sleep 2000
-run general/parser/slimit.sim
-sleep 2000
-run general/parser/slimit1.sim
-sleep 2000
-run general/parser/slimit_alter_tags.sim
-sleep 2000
-run general/parser/tbnameIn.sim
-sleep 2000
-run general/parser/slimit_alter_tags.sim  # persistent failed
-sleep 2000
-run general/parser/join.sim
+#sleep 2000
+#run general/parser/alter.sim
+#sleep 2000
+#run general/parser/alter1.sim
+#sleep 2000
+#run general/parser/alter_stable.sim
+#sleep 2000
+#run general/parser/auto_create_tb.sim
+#sleep 2000
+#run general/parser/auto_create_tb_drop_tb.sim
+#sleep 2000
+#run general/parser/col_arithmetic_operation.sim
+#sleep 2000
+#run general/parser/columnValue.sim
+#sleep 2000
+#run general/parser/commit.sim
+#sleep 2000
+#run general/parser/create_db.sim
+#sleep 2000
+#run general/parser/create_mt.sim
+#sleep 2000
+#run general/parser/create_tb.sim
+#sleep 2000
+#run general/parser/dbtbnameValidate.sim
+#sleep 2000
+#run general/parser/fill.sim
+#sleep 2000
+#run general/parser/fill_stb.sim
+#sleep 2000
+##run general/parser/fill_us.sim               #
+#sleep 2000
+#run general/parser/first_last.sim
+#sleep 2000
+#run general/parser/import_commit1.sim
+#sleep 2000
+#run general/parser/import_commit2.sim
+#sleep 2000
+#run general/parser/import_commit3.sim
+#sleep 2000
+##run general/parser/import_file.sim
+#sleep 2000
+#run general/parser/insert_tb.sim
+#sleep 2000
+#run general/parser/tags_dynamically_specifiy.sim
+#sleep 2000
+#run general/parser/interp.sim
+#sleep 2000
+#run general/parser/lastrow.sim
+#sleep 2000
+#run general/parser/limit.sim
+#sleep 2000
+#run general/parser/limit1.sim
+#sleep 2000
+#run general/parser/limit1_tblocks100.sim
+#sleep 2000
+#run general/parser/limit2.sim
+#sleep 2000
+#run general/parser/mixed_blocks.sim
+#sleep 2000
+#run general/parser/nchar.sim
+#sleep 2000
+#run general/parser/null_char.sim
+#sleep 2000
+#run general/parser/selectResNum.sim
+#sleep 2000
+#run general/parser/select_across_vnodes.sim
+#sleep 2000
+#run general/parser/select_from_cache_disk.sim
+#sleep 2000
+#run general/parser/set_tag_vals.sim
+#sleep 2000
+#run general/parser/single_row_in_tb.sim
+#sleep 2000
+#run general/parser/slimit.sim
+#sleep 2000
+#run general/parser/slimit1.sim
+#sleep 2000
+#run general/parser/slimit_alter_tags.sim
+#sleep 2000
+#run general/parser/tbnameIn.sim
+#sleep 2000
+#run general/parser/slimit_alter_tags.sim  # persistent failed
+#sleep 2000
+#run general/parser/join.sim
 sleep 2000
 run general/parser/join_multivnode.sim
 sleep 2000

From 9768c30be1f946830361873a091af1bf71993ff1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 9 Sep 2020 22:15:12 +0800
Subject: [PATCH 07/56] [td-1372]

---
 src/client/src/tscFunctionImpl.c | 138 ++++++++++++++++++++++++++++---
 src/query/inc/qPercentile.h      |   4 +-
 src/query/src/qPercentile.c      |  89 +++++++++++++++++---
 3 files changed, 206 insertions(+), 25 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 4b31a8001f..1356b98c67 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -117,6 +117,10 @@ typedef struct SFirstLastInfo {
 typedef struct SFirstLastInfo SLastrowInfo;
 typedef struct SPercentileInfo {
   tMemBucket *pMemBucket;
+  int32_t     stage;
+  double      minval;
+  double      maxval;
+  int64_t     numOfElems;
 } SPercentileInfo;
 
 typedef struct STopBotInfo {
@@ -302,7 +306,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   } else if (functionId == TSDB_FUNC_PERCT) {
     *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
     *bytes = (int16_t)sizeof(double);
-    *interBytes = (int16_t)sizeof(double);
+    *interBytes = (int16_t)sizeof(SPercentileInfo);
   } else if (functionId == TSDB_FUNC_LEASTSQR) {
     *type = TSDB_DATA_TYPE_BINARY;
     *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE;  // string
@@ -2428,12 +2432,14 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
   if (!function_setup(pCtx)) {
     return false;
   }
-  
-  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
 
-  ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket =
-      tMemBucketCreate(pCtx->inputBytes, pCtx->inputType);
-  
+  // in the first round, get the min-max value of all involved data
+  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+  SPercentileInfo *pInfo = pResInfo->interResultBuf;
+  pInfo->minval = DBL_MAX;
+  pInfo->maxval = -DBL_MAX;
+  pInfo->numOfElems = 0;
+
   return true;
 }
 
@@ -2442,7 +2448,65 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
   
   SResultInfo *    pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = pResInfo->interResultBuf;
-  
+
+  // the first stage, only acquire the min/max value
+  if (pInfo->stage == 0) {
+    if (pCtx->preAggVals.isSet) {
+      if (pInfo->minval > pCtx->preAggVals.statis.min) {
+        pInfo->minval = pCtx->preAggVals.statis.min;
+      }
+
+      if (pInfo->maxval < pCtx->preAggVals.statis.max) {
+        pInfo->maxval = pCtx->preAggVals.statis.max;
+      }
+
+      pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
+    } else {
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
+        if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+          continue;
+        }
+
+        // TODO extract functions
+        double v = 0;
+        switch (pCtx->inputType) {
+          case TSDB_DATA_TYPE_TINYINT:
+            v = GET_INT8_VAL(data);
+            break;
+          case TSDB_DATA_TYPE_SMALLINT:
+            v = GET_INT16_VAL(data);
+            break;
+          case TSDB_DATA_TYPE_BIGINT:
+            v = (double)(GET_INT64_VAL(data));
+            break;
+          case TSDB_DATA_TYPE_FLOAT:
+            v = GET_FLOAT_VAL(data);
+            break;
+          case TSDB_DATA_TYPE_DOUBLE:
+            v = GET_DOUBLE_VAL(data);
+            break;
+          default:
+            v = GET_INT32_VAL(data);
+            break;
+        }
+
+        if (v < pInfo->minval) {
+          pInfo->minval = v;
+        }
+
+        if (v > pInfo->maxval) {
+          pInfo->maxval = v;
+        }
+
+        pInfo->numOfElems += 1;
+      }
+    }
+
+    return;
+  }
+
+  // the second stage, calculate the true percentile value
   for (int32_t i = 0; i < pCtx->size; ++i) {
     char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
     if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
@@ -2462,10 +2526,47 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-  
+
   SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf;
+
+  if (pInfo->stage == 0) {
+    // TODO extract functions
+    double v = 0;
+    switch (pCtx->inputType) {
+      case TSDB_DATA_TYPE_TINYINT:
+        v = GET_INT8_VAL(pData);
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        v = GET_INT16_VAL(pData);
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        v = (double)(GET_INT64_VAL(pData));
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        v = GET_FLOAT_VAL(pData);
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        v = GET_DOUBLE_VAL(pData);
+        break;
+      default:
+        v = GET_INT32_VAL(pData);
+        break;
+    }
+
+    if (v < pInfo->minval) {
+      pInfo->minval = v;
+    }
+
+    if (v > pInfo->maxval) {
+      pInfo->maxval = v;
+    }
+
+    pInfo->numOfElems += 1;
+    return;
+  }
+  
   tMemBucketPut(pInfo->pMemBucket, pData, 1);
   
   SET_VAL(pCtx, 1, 1);
@@ -2488,6 +2589,23 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
   doFinalizer(pCtx);
 }
 
+static void percentile_next_step(SQLFunctionCtx *pCtx) {
+  SResultInfo *    pResInfo = GET_RES_INFO(pCtx);
+  SPercentileInfo *pInfo = pResInfo->interResultBuf;
+
+  if (pInfo->stage == 0) {
+    // all data are null, set it completed
+    if (pInfo->numOfElems == 0) {
+      pResInfo->complete = true;
+    }
+
+    pInfo->stage += 1;
+    pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+  } else {
+    pResInfo->complete = true;
+  }
+}
+
 //////////////////////////////////////////////////////////////////////////////////
 static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
   SResultInfo *pResInfo = GET_RES_INFO(pCtx);
@@ -4513,7 +4631,7 @@ SQLAggFuncElem aAggs[] = {{
                               percentile_function_setup,
                               percentile_function,
                               percentile_function_f,
-                              no_next_step,
+                              percentile_next_step,
                               percentile_finalizer,
                               noop1,
                               noop1,
diff --git a/src/query/inc/qPercentile.h b/src/query/inc/qPercentile.h
index 0a52d4f205..c34c24c5b2 100644
--- a/src/query/inc/qPercentile.h
+++ b/src/query/inc/qPercentile.h
@@ -64,11 +64,11 @@ typedef struct tMemBucket {
   __perc_hash_func_t hashFunc;
 } tMemBucket;
 
-tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType);
+tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
 
 void tMemBucketDestroy(tMemBucket *pBucket);
 
-void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
+int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
 
 double getPercentile(tMemBucket *pMemBucket, double percent);
 
diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index 1ce5861e52..ccc451108e 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -70,6 +70,33 @@ static void resetBoundingBox(MinMaxEntry* range, int32_t type) {
   }
 }
 
+static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) {
+  if (minval > maxval) {
+    return -1;
+  }
+
+  switch(type) {
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+      range->iMinVal = minval;
+      range->iMaxVal = maxval;
+      break;
+
+    case TSDB_DATA_TYPE_BIGINT:
+      range->i64MinVal = minval;
+      range->i64MaxVal = maxval;
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE:
+      range->dMinVal = minval;
+      range->dMaxVal = maxval;
+      break;
+  }
+
+  return 0;
+}
+
 static void resetPosInfo(SSlotInfo* pInfo) {
   pInfo->size   = 0;
   pInfo->pageId = -1;
@@ -135,6 +162,11 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) {
 
     return index;
   } else {
+    // out of range
+    if (v < pBucket->range.i64MinVal || v > pBucket->range.i64MaxVal) {
+      return -1;
+    }
+
     // todo hash for bigint and float and double
     int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal;
     if (span < pBucket->numOfSlots) {
@@ -179,6 +211,11 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) {
 
     return index;
   } else {
+    // out of range
+    if (v < pBucket->range.iMinVal || v > pBucket->range.iMaxVal) {
+      return -1;
+    }
+
     // divide a range of [iMinVal, iMaxVal] into 1024 buckets
     int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal;
     if (span < pBucket->numOfSlots) {
@@ -209,6 +246,12 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) {
     double posx = (v + DBL_MAX) / x;
     return ((int32_t)posx) % pBucket->numOfSlots;
   } else {
+
+    // out of range
+    if (v < pBucket->range.dMinVal || v > pBucket->range.dMaxVal) {
+      return -1;
+    }
+
     // divide a range of [dMinVal, dMaxVal] into 1024 buckets
     double span = pBucket->range.dMaxVal - pBucket->range.dMinVal;
     if (span < pBucket->numOfSlots) {
@@ -262,7 +305,7 @@ static void resetSlotInfo(tMemBucket* pBucket) {
   }
 }
 
-tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) {
+tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
   tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket));
   if (pBucket == NULL) {
     return NULL;
@@ -278,9 +321,14 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) {
 
   pBucket->maxCapacity = 200000;
 
+  if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) {
+    uError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
+    free(pBucket);
+    return NULL;
+  }
+
   pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes;
   pBucket->comparFn = getKeyComparFunc(pBucket->type);
-  resetBoundingBox(&pBucket->range, pBucket->type);
 
   pBucket->hashFunc = getHashFunc(pBucket->type);
   if (pBucket->hashFunc == NULL) {
@@ -395,23 +443,25 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) {
 /*
  * in memory bucket, we only accept data array list
  */
-void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
+int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
   assert(pBucket != NULL && data != NULL && size > 0);
+
   pBucket->total += (int32_t)size;
 
   int32_t bytes = pBucket->bytes;
-
   for (int32_t i = 0; i < size; ++i) {
     char *d = (char *) data + i * bytes;
 
-    int32_t slotIdx = (pBucket->hashFunc)(pBucket, d);
-    assert(slotIdx >= 0);
+    int32_t index = (pBucket->hashFunc)(pBucket, d);
+    if (index == -1) {  // the value is out of range, do not add it into bucket
+      return -1;
+    }
 
-    tMemBucketSlot *pSlot = &pBucket->pSlots[slotIdx];
+    tMemBucketSlot *pSlot = &pBucket->pSlots[index];
     tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type);
 
     // ensure available memory pages to allocate
-    int32_t groupId = getGroupId(pBucket->numOfSlots, slotIdx, pBucket->times);
+    int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times);
     int32_t pageId = -1;
 
     if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) {
@@ -432,10 +482,12 @@ void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
     pSlot->info.data->num += 1;
     pSlot->info.size += 1;
   }
+
+  return 0;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////
-static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) {
+static UNUSED_FUNC void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) {
   *minVal = DBL_MAX;
   *maxVal = -DBL_MAX;
 
@@ -681,16 +733,27 @@ double getPercentile(tMemBucket *pMemBucket, double percent) {
 
   // find the min/max value, no need to scan all data in bucket
   if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) {
-    double minx = 0, maxx = 0;
-    findMaxMinValue(pMemBucket, &maxx, &minx);
+    MinMaxEntry* pRange = &pMemBucket->range;
 
-    return fabs(percent - 100) < DBL_EPSILON ? maxx : minx;
+    switch(pMemBucket->type) {
+      case TSDB_DATA_TYPE_TINYINT:
+      case TSDB_DATA_TYPE_SMALLINT:
+      case TSDB_DATA_TYPE_INT:
+        return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal;
+      case TSDB_DATA_TYPE_BIGINT:
+        return fabs(percent - 100) < DBL_EPSILON? pRange->i64MaxVal:pRange->i64MinVal;
+      case TSDB_DATA_TYPE_FLOAT:
+      case TSDB_DATA_TYPE_DOUBLE:
+        return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;
+        default:
+        return -1;
+    }
   }
 
   double  percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0);
-  int32_t orderIdx = (int32_t)percentVal;
 
   // do put data by using buckets
+  int32_t orderIdx = (int32_t)percentVal;
   return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx);
 }
 

From 82714623852c0163f0f9c83d8b3aaa779a661a03 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 10 Sep 2020 13:05:59 +0800
Subject: [PATCH 08/56] [td-1369]

---
 src/query/inc/qUtil.h                     |   6 +-
 src/query/src/qExecutor.c                 |  17 +++
 src/query/src/qFilterfunc.c               |  97 +++---------
 tests/script/general/parser/testSuite.sim | 171 +++++++++++-----------
 4 files changed, 128 insertions(+), 163 deletions(-)

diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 6de3c7c0e5..314159484d 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -50,14 +50,16 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3
     tFilePage* page) {
   assert(pResult != NULL && pRuntimeEnv != NULL);
 
-  SQuery    *pQuery = pRuntimeEnv->pQuery;
-//  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+  SQuery *pQuery = pRuntimeEnv->pQuery;
 
   int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
   return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
       pQuery->pSelectExpr[columnIndex].bytes * realRowId;
 }
 
+bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
+bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
+
 __filter_func_t *getRangeFilterFuncArray(int32_t type);
 __filter_func_t *getValueFilterFuncArray(int32_t type);
 
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index fc932b8999..d00e616841 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -205,6 +205,23 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) {
     for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) {
       SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j];
 
+      bool isnull = isNull(pElem, pFilterInfo->info.type);
+      if (isnull) {
+        if (pFilterElem->fp == isNull_filter) {
+          qualified = true;
+          break;
+        } else {
+          continue;
+        }
+      } else {
+        if (pFilterElem->fp == notNull_filter) {
+          qualified = true;
+          break;
+        } else if (pFilterElem->fp == isNull_filter) {
+          continue;
+        }
+      }
+
       if (pFilterElem->fp(pFilterElem, pElem, pElem)) {
         qualified = true;
         break;
diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c
index 6b88171e71..b6050dddd8 100644
--- a/src/query/src/qFilterfunc.c
+++ b/src/query/src/qFilterfunc.c
@@ -285,69 +285,12 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
   return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0;
 }
 ////////////////////////////////////////////////////////////////
-bool isNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_TINYINT);
+bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) {
+  return true;
 }
 
-bool isNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_SMALLINT);
-}
-
-bool isNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_INT);
-}
-
-bool isNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_BIGINT);
-}
-
-bool isNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_FLOAT);
-}
-
-bool isNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_DOUBLE);
-}
-
-bool isNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_BINARY);
-}
-
-bool isNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_NCHAR);
-}
-
-////////////////////////////////////////////////////////////////
-bool notNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_TINYINT);
-}
-
-bool notNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_SMALLINT);
-}
-
-bool notNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_INT);
-}
-
-bool notNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_BIGINT);
-}
-
-bool notNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_FLOAT);
-}
-
-bool notNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return isNull(minval, TSDB_DATA_TYPE_DOUBLE);
-}
-
-bool notNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_BINARY);
-}
-
-bool notNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
-  return !isNull(minval, TSDB_DATA_TYPE_NCHAR);
+bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) {
+  return true;
 }
 
 ////////////////////////////////////////////////////////////////
@@ -463,8 +406,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i8,
   nequal_i8,
   NULL,
-  isNull_i8,
-  notNull_i8,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -476,8 +419,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i16,
   nequal_i16,
   NULL,
-  isNull_i16,
-  notNull_i16,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -489,8 +432,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i32,
   nequal_i32,
   NULL,
-  isNull_i32,
-  notNull_i32,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -502,8 +445,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_i64,
   nequal_i64,
   NULL,
-  isNull_i64,
-  notNull_i64,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -515,8 +458,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_ds,
   nequal_ds,
   NULL,
-  isNull_ds,
-  notNull_ds,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@@ -528,8 +471,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
   largeEqual_dd,
   nequal_dd,
   NULL,
-  isNull_dd,
-  notNull_dd,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = {
@@ -541,8 +484,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval)
   NULL,
   nequal_str,
   like_str,
-  isNull_binary,
-  notNull_binary,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = {
@@ -554,8 +497,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva
   NULL,
   nequal_nchar,
   like_nchar,
-  isNull_nchar,
-  notNull_nchar,
+  isNull_filter,
+  notNull_filter,
 };
 
 bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index 721eb0bfa1..f42254981c 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -1,87 +1,87 @@
-#sleep 2000
-#run general/parser/alter.sim
-#sleep 2000
-#run general/parser/alter1.sim
-#sleep 2000
-#run general/parser/alter_stable.sim
-#sleep 2000
-#run general/parser/auto_create_tb.sim
-#sleep 2000
-#run general/parser/auto_create_tb_drop_tb.sim
-#sleep 2000
-#run general/parser/col_arithmetic_operation.sim
-#sleep 2000
-#run general/parser/columnValue.sim
-#sleep 2000
-#run general/parser/commit.sim
-#sleep 2000
-#run general/parser/create_db.sim
-#sleep 2000
-#run general/parser/create_mt.sim
-#sleep 2000
-#run general/parser/create_tb.sim
-#sleep 2000
-#run general/parser/dbtbnameValidate.sim
-#sleep 2000
-#run general/parser/fill.sim
-#sleep 2000
-#run general/parser/fill_stb.sim
-#sleep 2000
-##run general/parser/fill_us.sim               #
-#sleep 2000
-#run general/parser/first_last.sim
-#sleep 2000
-#run general/parser/import_commit1.sim
-#sleep 2000
-#run general/parser/import_commit2.sim
-#sleep 2000
-#run general/parser/import_commit3.sim
-#sleep 2000
-##run general/parser/import_file.sim
-#sleep 2000
-#run general/parser/insert_tb.sim
-#sleep 2000
-#run general/parser/tags_dynamically_specifiy.sim
-#sleep 2000
-#run general/parser/interp.sim
-#sleep 2000
-#run general/parser/lastrow.sim
-#sleep 2000
-#run general/parser/limit.sim
-#sleep 2000
-#run general/parser/limit1.sim
-#sleep 2000
-#run general/parser/limit1_tblocks100.sim
-#sleep 2000
-#run general/parser/limit2.sim
-#sleep 2000
-#run general/parser/mixed_blocks.sim
-#sleep 2000
-#run general/parser/nchar.sim
-#sleep 2000
-#run general/parser/null_char.sim
-#sleep 2000
-#run general/parser/selectResNum.sim
-#sleep 2000
-#run general/parser/select_across_vnodes.sim
-#sleep 2000
-#run general/parser/select_from_cache_disk.sim
-#sleep 2000
-#run general/parser/set_tag_vals.sim
-#sleep 2000
-#run general/parser/single_row_in_tb.sim
-#sleep 2000
-#run general/parser/slimit.sim
-#sleep 2000
-#run general/parser/slimit1.sim
-#sleep 2000
-#run general/parser/slimit_alter_tags.sim
-#sleep 2000
-#run general/parser/tbnameIn.sim
-#sleep 2000
-#run general/parser/slimit_alter_tags.sim  # persistent failed
-#sleep 2000
-#run general/parser/join.sim
+sleep 2000
+run general/parser/alter.sim
+sleep 2000
+run general/parser/alter1.sim
+sleep 2000
+run general/parser/alter_stable.sim
+sleep 2000
+run general/parser/auto_create_tb.sim
+sleep 2000
+run general/parser/auto_create_tb_drop_tb.sim
+sleep 2000
+run general/parser/col_arithmetic_operation.sim
+sleep 2000
+run general/parser/columnValue.sim
+sleep 2000
+run general/parser/commit.sim
+sleep 2000
+run general/parser/create_db.sim
+sleep 2000
+run general/parser/create_mt.sim
+sleep 2000
+run general/parser/create_tb.sim
+sleep 2000
+run general/parser/dbtbnameValidate.sim
+sleep 2000
+run general/parser/fill.sim
+sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000
+#run general/parser/fill_us.sim               #
+sleep 2000
+run general/parser/first_last.sim
+sleep 2000
+run general/parser/import_commit1.sim
+sleep 2000
+run general/parser/import_commit2.sim
+sleep 2000
+run general/parser/import_commit3.sim
+sleep 2000
+#run general/parser/import_file.sim
+sleep 2000
+run general/parser/insert_tb.sim
+sleep 2000
+run general/parser/tags_dynamically_specifiy.sim
+sleep 2000
+run general/parser/interp.sim
+sleep 2000
+run general/parser/lastrow.sim
+sleep 2000
+run general/parser/limit.sim
+sleep 2000
+run general/parser/limit1.sim
+sleep 2000
+run general/parser/limit1_tblocks100.sim
+sleep 2000
+run general/parser/limit2.sim
+sleep 2000
+run general/parser/mixed_blocks.sim
+sleep 2000
+run general/parser/nchar.sim
+sleep 2000
+run general/parser/null_char.sim
+sleep 2000
+run general/parser/selectResNum.sim
+sleep 2000
+run general/parser/select_across_vnodes.sim
+sleep 2000
+run general/parser/select_from_cache_disk.sim
+sleep 2000
+run general/parser/set_tag_vals.sim
+sleep 2000
+run general/parser/single_row_in_tb.sim
+sleep 2000
+run general/parser/slimit.sim
+sleep 2000
+run general/parser/slimit1.sim
+sleep 2000
+run general/parser/slimit_alter_tags.sim
+sleep 2000
+run general/parser/tbnameIn.sim
+sleep 2000
+run general/parser/slimit_alter_tags.sim  # persistent failed
+sleep 2000
+run general/parser/join.sim
 sleep 2000
 run general/parser/join_multivnode.sim
 sleep 2000
@@ -99,10 +99,13 @@ run general/parser/union.sim
 sleep 2000
 run general/parser/constCol.sim
 sleep 2000
+run general/parser/where.sim
+sleep 2000
 run general/parser/timestamp.sim
 sleep 2000
 run general/parser/sliding.sim
 
+
 #sleep 2000
 #run general/parser/repeatStream.sim
 #sleep 2000

From b60e66bb78dfb97f05db3b7b529175ee0cc2a261 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 10 Sep 2020 14:53:32 +0800
Subject: [PATCH 09/56] [td-1409] remove the dependency of JDBC driver on
 common-util package. #3459

---
 src/connector/jdbc/pom.xml                           |  5 -----
 .../src/main/java/com/taosdata/jdbc/TSDBDriver.java  | 12 +++++-------
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml
index da0c5b12d4..0fed63fc80 100755
--- a/src/connector/jdbc/pom.xml
+++ b/src/connector/jdbc/pom.xml
@@ -48,11 +48,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>${commons-lang3.version}</version>
-    </dependency>    
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
index 02d642d643..e25bd64c73 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
@@ -14,8 +14,6 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
-import org.apache.commons.lang3.StringUtils;
-
 import java.sql.*;
 import java.util.Properties;
 import java.util.logging.Logger;
@@ -42,9 +40,8 @@ import java.util.logging.Logger;
 public class TSDBDriver implements java.sql.Driver {
 
 	@Deprecated
-	private static final String URL_PREFIX1 = "jdbc:TSDB://";
-
-	private static final String URL_PREFIX = "jdbc:TAOS://";
+	private static final String URL_PREFIX1 = "jdbc:tsdb://";
+	private static final String URL_PREFIX  = "jdbc:taos://";
 
 	/**
 	 * Key used to retrieve the database value from the properties instance passed
@@ -188,7 +185,7 @@ public class TSDBDriver implements java.sql.Driver {
 	}
 
 	public boolean acceptsURL(String url) throws SQLException {
-		return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX);
+		return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX);
 	}
 
 	public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@@ -238,7 +235,8 @@ public class TSDBDriver implements java.sql.Driver {
 			return null;
 		}
 
-		if (!StringUtils.startsWithIgnoreCase(url, URL_PREFIX) && !StringUtils.startsWithIgnoreCase(url, URL_PREFIX1)) {
+        String lowerUrl = url.toLowerCase();
+        if (!lowerUrl.startsWith(URL_PREFIX) && !lowerUrl.startsWith(URL_PREFIX1)) {
 			return null;
 		}
 

From a30dd56278fc02055215894ff157f3dcde48a702 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 10 Sep 2020 15:03:59 +0800
Subject: [PATCH 10/56] [td-225]

---
 src/client/src/tscLocal.c             | 1 -
 tests/script/general/parser/where.sim | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index b240d357a8..4b6174e13d 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -16,7 +16,6 @@
 #include "os.h"
 #include "taosmsg.h"
 
-#include "qExtbuffer.h"
 #include "taosdef.h"
 #include "tcache.h"
 #include "tname.h"
diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim
index dd3b11c2dc..fb15fb6dbe 100644
--- a/tests/script/general/parser/where.sim
+++ b/tests/script/general/parser/where.sim
@@ -2,6 +2,7 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000

From d74478425bda4c2266c60374d75702aed95bb471 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 10 Sep 2020 17:16:28 +0800
Subject: [PATCH 11/56] [td-1411]

---
 src/common/src/ttypes.c    |  9 ++--
 src/inc/taosdef.h          |  2 +-
 src/query/src/qExtbuffer.c | 93 ++++++++++++++++++++++----------------
 3 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c
index 3910d80ecf..ff417b6cde 100644
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@@ -540,9 +540,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
   }
 }
 
-void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) {
-  char tmpBuf[4096] = {0};
-  
+void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
   switch (type) {
     case TSDB_DATA_TYPE_INT: {
       SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
@@ -575,10 +573,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) {
     }
     
     default: {
-      assert(size <= 4096);
-      memcpy(tmpBuf, pLeft, size);
+      memcpy(buf, pLeft, size);
       memcpy(pLeft, pRight, size);
-      memcpy(pRight, tmpBuf, size);
+      memcpy(pRight, buf, size);
       break;
     }
   }
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 7437eceecb..17cbbca834 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -198,7 +198,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems);
 void* getNullValue(int32_t type);
 
 void assignVal(char *val, const char *src, int32_t len, int32_t type);
-void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
+void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf);
 
 // TODO: check if below is necessary
 #define TSDB_RELATION_INVALID     0
diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c
index 192a31ebf5..1d3120ead4 100644
--- a/src/query/src/qExtbuffer.c
+++ b/src/query/src/qExtbuffer.c
@@ -502,22 +502,22 @@ FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows
   return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data);
 }
 
-static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2) {
+static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) {
   for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) {
     void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i);
     void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i);
 
     SSchema* pSchema = &pColumnModel->pFields[i].field;
-    tsDataSwap(first, second, pSchema->type, pSchema->bytes);
+    tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf);
   }
 }
 
 static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
-                               __col_compar_fn_t compareFn) {
+                               __col_compar_fn_t compareFn, void* buf) {
   for (int32_t i = start + 1; i <= end; ++i) {
     for (int32_t j = i; j > start; --j) {
       if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) {
-        swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j);
+        swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf);
       } else {
         break;
       }
@@ -553,7 +553,7 @@ static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx,
 }
 
 static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
-                   __col_compar_fn_t compareFn) {
+                   __col_compar_fn_t compareFn, void* buf) {
   int32_t midIdx = ((end - start) >> 1) + start;
 
 #if defined(_DEBUG_VIEW)
@@ -567,15 +567,16 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
   tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx);
 #endif
 
+  SColumnModel* pModel = pDescriptor->pColumnModel;
   if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) {
-    swap(pDescriptor->pColumnModel, numOfRows, start, data, midIdx);
+    swap(pModel, numOfRows, start, data, midIdx, buf);
   }
 
   if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) {
-    swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, start);
-    swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, end);
+    swap(pModel, numOfRows, midIdx, data, start, buf);
+    swap(pModel, numOfRows, midIdx, data, end, buf);
   } else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) {
-    swap(pDescriptor->pColumnModel, numOfRows, start, data, end);
+    swap(pModel, numOfRows, start, data, end, buf);
   }
 
   assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 &&
@@ -626,32 +627,20 @@ static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t
   printf("\n");
 }
 
-static int32_t qsort_call = 0;
-
-void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
-                   int32_t orderType) {
-  // short array sort, incur another sort procedure instead of quick sort process
-  __col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd;
-
-  if (end - start + 1 <= 8) {
-    tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn);
-    return;
-  }
-
+static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
+                                int32_t orderType, __col_compar_fn_t compareFn, void* buf) {
 #ifdef _DEBUG_VIEW
-//  printf("before sort:\n");
-//  tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
+  printf("before sort:\n");
+  tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
 #endif
 
   int32_t s = start, e = end;
-  median(pDescriptor, numOfRows, start, end, data, compareFn);
+  median(pDescriptor, numOfRows, start, end, data, compareFn, buf);
 
 #ifdef _DEBUG_VIEW
-//  printf("%s called: %d\n", __FUNCTION__, qsort_call++);
+  //  printf("%s called: %d\n", __FUNCTION__, qsort_call++);
 #endif
 
-  UNUSED(qsort_call);
-
   int32_t end_same = end;
   int32_t start_same = start;
 
@@ -663,17 +652,17 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
       }
 
       if (ret == 0 && e != end_same) {
-        swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--);
+        swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf);
       }
       e--;
     }
 
     if (e != s) {
-      swap(pDescriptor->pColumnModel, numOfRows, s, data, e);
+      swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf);
     }
 
 #ifdef _DEBUG_VIEW
-//    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
+    //    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
 #endif
 
     while (s < e) {
@@ -683,16 +672,16 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
       }
 
       if (ret == 0 && s != start_same) {
-        swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++);
+        swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf);
       }
       s++;
     }
 
     if (s != e) {
-      swap(pDescriptor->pColumnModel, numOfRows, s, data, e);
+      swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf);
     }
 #ifdef _DEBUG_VIEW
-//    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
+    //    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
 #endif
   }
 
@@ -702,14 +691,14 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
     int32_t right = end;
 
     while (right > end_same && left <= end_same) {
-      swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--);
+      swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf);
     }
 
     // (pivotal+1) + steps of number that are identical pivotal
     rightx += (end - end_same);
 
 #ifdef _DEBUG_VIEW
-//    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
+    //    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
 #endif
   }
 
@@ -719,26 +708,52 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
     int32_t right = e - 1;
 
     while (left < start_same && right >= start_same) {
-      swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--);
+      swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf);
     }
 
     // (pivotal-1) - steps of number that are identical pivotal
     leftx -= (start_same - start);
 
 #ifdef _DEBUG_VIEW
-//    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
+    //    tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
 #endif
   }
 
   if (leftx > start) {
-    tColDataQSort(pDescriptor, numOfRows, start, leftx, data, orderType);
+    columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf);
   }
 
   if (rightx < end) {
-    tColDataQSort(pDescriptor, numOfRows, rightx, end, data, orderType);
+    columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf);
   }
 }
 
+void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) {
+  // short array sort, incur another sort procedure instead of quick sort process
+  __col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd;
+
+  SColumnModel* pModel = pDescriptor->pColumnModel;
+
+  size_t width = 0;
+  for(int32_t i = 0; i < pModel->numOfCols; ++i) {
+    SSchema* pSchema = &pModel->pFields[i].field;
+    if (width < pSchema->bytes) {
+      width = pSchema->bytes;
+    }
+  }
+
+  char* buf = malloc(width);
+  assert(width > 0 && buf != NULL);
+
+  if (end - start + 1 <= 8) {
+    tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf);
+  } else {
+    columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf);
+  }
+
+  free(buf);
+}
+
 /*
  * deep copy of sschema
  */

From dd368d91af23935ea7d16e44b264b59dbab2c8b0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 10 Sep 2020 22:12:17 +0800
Subject: [PATCH 12/56] [td-1412]

---
 src/query/src/qExecutor.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d00e616841..75614578fe 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -793,7 +793,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed
       pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
 
       if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-        pCtx[k].ptsList = &tsBuf[offset];
+        pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
       }
 
       // not a whole block involved in query processing, statistics data can not be used
@@ -1475,12 +1475,14 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
   pCtx->preAggVals.dataBlockLoaded = (inputData != NULL);
 
   // limit/offset query will affect this value
-  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
   pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
 
+  // minimum value no matter ascending/descending order query
+  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size - 1);
+
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
-    pCtx->ptsList = tsCol;
+    pCtx->ptsList = &tsCol[pCtx->startOffset];
   }
 
   if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {

From b9566424bae3182796bc676a69724245431283d4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 11 Sep 2020 10:22:42 +0800
Subject: [PATCH 13/56] [td-1412]

---
 src/query/inc/tsqlfunction.h |  2 +-
 src/query/src/qExecutor.c    | 28 +++++++++-------------------
 2 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index c314087179..a78bb7ec51 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -177,7 +177,7 @@ typedef struct SQLFunctionCtx {
   int16_t      outputType;
   int16_t      outputBytes;  // size of results, determined by function and input column data type
   bool         hasNull;      // null value exist in current block
-  bool         requireNull;    // require null in some function
+  bool         requireNull;  // require null in some function
   int16_t      functionId;   // function id
   void *       aInputElemBuf;
   char *       aOutputBuf;            // final result output buffer, point to sdata->data
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 75614578fe..511d6c36ef 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -743,9 +743,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
                                         int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) {
   assert(startPos >= 0 && startPos < pDataBlockInfo->rows);
 
-  int32_t num = -1;
+  int32_t num   = -1;
   int32_t order = pQuery->order.order;
-  int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
+  int32_t step  = GET_FORWARD_DIRECTION_FACTOR(order);
 
   STableQueryInfo* item = pQuery->current;
 
@@ -779,27 +779,24 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
   return num;
 }
 
-static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin,
-                                      int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
+static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset,
+                                      int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) {
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
   if (IS_MASTER_SCAN(pRuntimeEnv) || closed) {
     for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
-      int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
-
       pCtx[k].nStartQueryTimestamp = pWin->skey;
       pCtx[k].size = forwardStep;
       pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
 
+      int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
       if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-        pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
+        pCtx[k].ptsList = &tsCol[pCtx[k].startOffset];
       }
 
       // not a whole block involved in query processing, statistics data can not be used
-      if (forwardStep != numOfTotal) {
-        pCtx[k].preAggVals.isSet = false;
-      }
+      pCtx[k].preAggVals.isSet = (forwardStep == numOfTotal);
 
       if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
         aAggs[functionId].xFunction(&pCtx[k]);
@@ -924,19 +921,11 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
 
   char *dataBlock = NULL;
   SQuery *pQuery = pRuntimeEnv->pQuery;
-  SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
   int32_t functionId = pQuery->pSelectExpr[col].base.functionId;
   if (functionId == TSDB_FUNC_ARITHM) {
     sas->pArithExpr = &pQuery->pSelectExpr[col];
 
-    // set the start offset to be the lowest start position, no matter asc/desc query order
-    if (QUERY_IS_ASC_QUERY(pQuery)) {
-      pCtx->startOffset = pQuery->pos;
-    } else {
-      pCtx->startOffset = pQuery->pos - (size - 1);
-    }
-
     sas->offset  = 0;
     sas->colList = pQuery->colList;
     sas->numOfCols = pQuery->numOfCols;
@@ -1478,7 +1467,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
   pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
 
   // minimum value no matter ascending/descending order query
-  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size - 1);
+  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1);
+  assert(pCtx->startOffset >= 0);
 
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {

From bd0522b1a922f09019c93b0859d0e0e00e04b0aa Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 11 Sep 2020 10:35:03 +0800
Subject: [PATCH 14/56] [td-225] add test cases.

---
 tests/script/general/parser/timestamp_query.sim | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/script/general/parser/timestamp_query.sim b/tests/script/general/parser/timestamp_query.sim
index 6994b2d295..783c03602b 100644
--- a/tests/script/general/parser/timestamp_query.sim
+++ b/tests/script/general/parser/timestamp_query.sim
@@ -21,6 +21,10 @@ $tsu = $rowNum * $delta
 $tsu = $tsu - $delta
 $tsu = $tsu + $ts0
 
+print ==================>issue #3481, normal column not allowed,
+sql_error select ts,c1,min(c2) from ts_stb0
+
+
 ##### select from supertable
 $tb = $tbPrefix . 0
 sql select first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1)

From 9a87f3ccd36abcdd44878d9aaee9647b80c0a803 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 11 Sep 2020 11:05:10 +0800
Subject: [PATCH 15/56] [td-1416]

---
 src/client/src/tscSQLParser.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index adb6bfccbc..eaf2692e7e 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5226,7 +5226,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
   }
 }
 
-static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
+static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
   size_t size = taosArrayGetSize(pQueryInfo->exprList);
   
   for (int32_t i = 0; i < size; ++i) {
@@ -5244,9 +5244,14 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
         }
       }
 
-      assert(qualifiedCol);
+      // it is not a tag column/tbname column/user-defined column, return error
+      if (!qualifiedCol) {
+        return TSDB_CODE_TSC_INVALID_SQL;
+      }
     }
   }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) {
@@ -5329,7 +5334,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
     SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i);
     if (pExpr->functionId == TSDB_FUNC_TAGPRJ ||
         (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) {
-      tagColExists = true;
+      tagColExists = true;  // selectivity + ts/tag column
       break;
     }
   }
@@ -5362,7 +5367,11 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
      */
     if (numOfSelectivity == 1) {
       doUpdateSqlFunctionForTagPrj(pQueryInfo);
-      doUpdateSqlFunctionForColPrj(pQueryInfo);
+      int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
     } else if (numOfSelectivity > 1) {
       /*
        * If more than one selectivity functions exist, all the selectivity functions must be last_row.
@@ -5381,7 +5390,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
       }
 
       doUpdateSqlFunctionForTagPrj(pQueryInfo);
-      doUpdateSqlFunctionForColPrj(pQueryInfo);
+      int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
     }
   } else {
     if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) {
@@ -5392,7 +5404,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
       if (numOfAggregation > 0 || numOfSelectivity > 0) {
         // clear the projection type flag
         pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY);
-        doUpdateSqlFunctionForColPrj(pQueryInfo);
+        int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
       }
     }
   }

From 562f6ce245a524635bc6ce6fde251792bed921a2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 11 Sep 2020 11:22:45 +0800
Subject: [PATCH 16/56] [td-1416]

---
 src/client/src/tscSQLParser.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index eaf2692e7e..e4654efe1c 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5231,7 +5231,8 @@ static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
   
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-    if (pExpr->functionId == TSDB_FUNC_PRJ) {
+
+    if (pExpr->functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) {
       bool qualifiedCol = false;
       for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) {
         SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j);

From a28d714d98dce759db3ed4a86198d3b9da04fa19 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 11 Sep 2020 18:35:10 +0800
Subject: [PATCH 17/56] td-1423] fix bugs in group by nchar/binary columns.
 #3476

---
 src/query/inc/qExecutor.h |  2 +-
 src/query/src/qExecutor.c | 43 +++++++++++++++++++++++++----------
 src/query/src/qUtil.c     | 47 +++++++++++++++++++++++++++++----------
 3 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index 25fb04fb9a..169bf907c6 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -57,7 +57,7 @@ typedef struct SWindowResult {
   uint16_t      numOfRows;   // number of rows of current time window
   bool          closed;      // this result status: closed or opened
   SResultInfo*  resultInfo;  // For each result column, there is a resultInfo
-  TSKEY         skey;        // start key of current time window
+  union {STimeWindow win; char* key;};  // start key of current time window
 } SWindowResult;
 
 /**
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 511d6c36ef..f4927ecb78 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -518,7 +518,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
   } else {
     int32_t slot = curTimeWindowIndex(pWindowResInfo);
     SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot);
-    w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes);
+    w = pWindowRes->win;
   }
 
   if (w.skey > ts || w.ekey < ts) {
@@ -624,7 +624,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
   }
 
   // set time window for current result
-  pWindowRes->skey = win->skey;
+  pWindowRes->win = (*win);
 
   setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes);
   return TSDB_CODE_SUCCESS;
@@ -697,12 +697,12 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
         continue;
       }
 
-      TSKEY ekey = pResult->skey + pWindowResInfo->interval;
+      TSKEY ekey = pResult->win.ekey;
       if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) ||
-          (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) {
+          (pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) {
         closeTimeWindow(pWindowResInfo, i);
       } else {
-        skey = pResult->skey;
+        skey = pResult->win.skey;
         break;
       }
     }
@@ -715,7 +715,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
       pWindowResInfo->curIndex = i;
     }
 
-    pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey;
+    pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey;
 
     // the number of completed slots are larger than the threshold, return current generated results to client.
     if (numOfClosed > pWindowResInfo->threshold) {
@@ -1097,8 +1097,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
 
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
-  int64_t v = -1;
   // not assign result buffer yet, add new result buffer
+  char* d = pData;
+  int16_t len = bytes;
+  if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) {
+    d = varDataVal(pData);
+    len = varDataLen(pData);
+  } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
+    SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
+    qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo);
+
+    longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true);
+  if (pWindowRes == NULL) {
+    return -1;
+  }
+
+  int64_t v = -1;
   switch(type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:  v = GET_INT8_VAL(pData);  break;
@@ -1107,12 +1124,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
     case TSDB_DATA_TYPE_BIGINT:   v = GET_INT64_VAL(pData); break;
   }
 
-  SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true);
-  if (pWindowRes == NULL) {
-    return -1;
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+    pWindowRes->key = malloc(varDataTLen(pData));
+    varDataCopy(pWindowRes->key, pData);
+  } else {
+    pWindowRes->win.skey = v;
+    pWindowRes->win.ekey = v;
   }
 
-  pWindowRes->skey = v;
   assert(pRuntimeEnv->windowResInfo.interval == 0);
 
   if (pWindowRes->pos.pageId == -1) {
@@ -3004,7 +3023,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
     TSKEY ts = GET_INT64_VAL(b);
 
-    assert(ts == pWindowRes->skey);
+    assert(ts == pWindowRes->win.skey);
     int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes);
     if (num <= 0) {
       cs.position[pos] += 1;
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 509362863c..c195a0b76c 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -126,11 +126,26 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   
   int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
   assert(num >= 0 && num <= numOfClosed);
-  
+
+  int16_t type = pWindowResInfo->type;
+
+  char *key = NULL;
+  int16_t bytes = -1;
+
   for (int32_t i = 0; i < num; ++i) {
     SWindowResult *pResult = &pWindowResInfo->pResult[i];
     if (pResult->closed) {  // remove the window slot from hash table
-      taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type);
+
+      // todo refactor
+      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+        key = varDataVal(pResult->key);
+        bytes = varDataLen(pResult->key);
+      } else {
+        key = (char*) &pResult->win.skey;
+        bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
+      }
+
+      taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes);
     } else {
       break;
     }
@@ -150,15 +165,24 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   }
   
   pWindowResInfo->size = remain;
+
   for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
     SWindowResult *pResult = &pWindowResInfo->pResult[k];
-    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey,
-        tDataTypeDesc[pWindowResInfo->type].nSize);
+
+    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+      key = varDataVal(pResult->key);
+      bytes = varDataLen(pResult->key);
+    } else {
+      key = (char*) &pResult->win.skey;
+      bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
+    }
+
+    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes);
     assert(p != NULL); 
+
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
-    taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize,
-        (char *)&v, sizeof(int32_t));
+    taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t));
   }
   
   pWindowResInfo->curIndex = -1;
@@ -207,20 +231,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
   }
 
   // get the result order
-  int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1;
-
+  int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1;
   if (order != resultOrder) {
     return;
   }
 
   int32_t i = 0;
   if (order == QUERY_ASC_FORWARD_STEP) {
-    TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval;
+    TSKEY ekey = pWindowResInfo->pResult[i].win.ekey;
     while (i < pWindowResInfo->size && (ekey < lastKey)) {
       ++i;
     }
   } else if (order == QUERY_DESC_FORWARD_STEP) {
-    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) {
+    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) {
       ++i;
     }
   }
@@ -258,7 +281,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
   pWindowRes->numOfRows = 0;
   pWindowRes->pos = (SPosInfo){-1, -1};
   pWindowRes->closed = false;
-  pWindowRes->skey = TSKEY_INITIAL_VAL;
+  pWindowRes->win = TSWINDOW_INITIALIZER;
 }
 
 /**
@@ -268,7 +291,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
  */
 void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
   dst->numOfRows = src->numOfRows;
-  dst->skey   = src->skey;
+  dst->win   = src->win;
   dst->closed = src->closed;
   
   int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;

From 2b1519c167beaa5b2a098a3b01dac46060891d71 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 12 Sep 2020 15:50:04 +0800
Subject: [PATCH 18/56] [td-1319]

---
 src/client/inc/tsclient.h               |  20 ++--
 src/client/src/tscAsync.c               |  10 +-
 src/client/src/tscLocalMerge.c          |   2 -
 src/client/src/tscServer.c              |  45 +++++---
 src/client/src/tscSql.c                 | 133 +++++++++++++-----------
 src/client/src/tscSubquery.c            |  17 +--
 src/client/src/tscSystem.c              |   7 +-
 src/client/src/tscUtil.c                |  58 +++++++++--
 src/plugins/http/src/httpSql.c          |  13 ---
 src/tsdb/src/tsdbRead.c                 |   1 +
 src/util/src/tcache.c                   |   9 +-
 tests/script/general/parser/groupby.sim |   2 +
 12 files changed, 188 insertions(+), 129 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 5f4a46ddad..11b7815586 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -29,6 +29,7 @@ extern "C" {
 #include "tglobal.h"
 #include "tsqlfunction.h"
 #include "tutil.h"
+#include "tcache.h"
 
 #include "qExecutor.h"
 #include "qSqlparser.h"
@@ -359,6 +360,8 @@ typedef struct SSqlObj {
   uint16_t         numOfSubs;
   struct SSqlObj **pSubs;
   struct SSqlObj * prev, *next;
+
+  struct SSqlObj **self;
 } SSqlObj;
 
 typedef struct SSqlStream {
@@ -413,7 +416,6 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
 void    tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
 
 int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
-void    tscDestroyResPointerInfo(SSqlRes *pRes);
 
 void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
 
@@ -425,17 +427,19 @@ void tscFreeSqlResult(SSqlObj *pSql);
 
 /**
  * only free part of resources allocated during query.
+ * TODO remove it later
  * Note: this function is multi-thread safe.
  * @param pObj
  */
-void tscPartiallyFreeSqlObj(SSqlObj *pObj);
+void tscPartiallyFreeSqlObj(SSqlObj *pSql);
 
 /**
  * free sql object, release allocated resource
- * @param pObj  Free metric/meta information, dynamically allocated payload, and
- * response buffer, object itself
+ * @param pObj
  */
-void tscFreeSqlObj(SSqlObj *pObj);
+void tscFreeSqlObj(SSqlObj *pSql);
+
+void tscFreeSqlObjInCache(void *pSql);
 
 void tscCloseTscObj(STscObj *pObj);
 
@@ -451,9 +455,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
 bool tscIsUpdateQuery(SSqlObj* pSql);
 bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
 
-// todo remove this function.
-bool tscResultsetFetchCompleted(TAOS_RES *result);
-
 char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
 
 int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
@@ -502,7 +503,8 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
   }
 }
 
-extern void *    tscCacheHandle;
+extern SCacheObj*    tscCacheHandle;
+extern SCacheObj*    tscObjCache;
 extern void *    tscTmr;
 extern void *    tscQhandle;
 extern int       tscKeepConn[];
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index d07089539a..5e9aa1b1f8 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -18,6 +18,7 @@
 
 #include "tnote.h"
 #include "trpc.h"
+#include "tcache.h"
 #include "tscLog.h"
 #include "tscSubquery.h"
 #include "tscLocalMerge.h"
@@ -40,6 +41,8 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
 
 void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
+  SSqlCmd* pCmd = &pSql->cmd;
+
   pSql->signature = pSql;
   pSql->param     = param;
   pSql->pTscObj   = pObj;
@@ -59,7 +62,10 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen);
 
   tscDebugL("%p SQL: %s", pSql, pSql->sqlstr);
-  pSql->cmd.curSql = pSql->sqlstr;
+  pCmd->curSql = pSql->sqlstr;
+
+  uint64_t handle = (uint64_t) pSql;
+  pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
 
   int32_t code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
@@ -69,7 +75,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
     tscQueueAsyncRes(pSql);
     return;
   }
-  
+
   tscDoQuery(pSql);
 }
 
diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 39a757795e..af6a546ff4 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -472,10 +472,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
     return;
   }
 
-  tscDebug("%p start to free local reducer", pSql);
   SSqlRes *pRes = &(pSql->res);
   if (pRes->pLocalReducer == NULL) {
-    tscDebug("%p local reducer has been freed, abort", pSql);
     return;
   }
 
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index fbc02cc40e..0733690e3f 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -27,10 +27,7 @@
 #include "tutil.h"
 #include "tlockfree.h"
 
-#define TSC_MGMT_VNODE 999
-
 SRpcCorEpSet  tscMgmtEpSet;
-SRpcEpSet  tscDnodeEpSet;
 
 int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
 
@@ -236,12 +233,17 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 }
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
-  SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle;
-  if (pSql == NULL || pSql->signature != pSql) {
-    tscError("%p sql is already released", pSql);
+  uint64_t handle = (uint64_t) rpcMsg->ahandle;
+
+  void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(uint64_t));
+  if (p == NULL) {
+    rpcFreeCont(rpcMsg->pCont);
     return;
   }
 
+  SSqlObj* pSql = *p;
+  assert(pSql != NULL);
+
   STscObj *pObj = pSql->pTscObj;
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
@@ -249,7 +251,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   if (pObj->signature != pObj) {
     tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
 
-    tscFreeSqlObj(pSql);
+    taosCacheRelease(tscObjCache, (void**) &p, true);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
@@ -261,18 +263,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
         pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
 
-    tscFreeSqlObj(pSql);
+    taosCacheRelease(tscObjCache, (void**) &p, true);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
 
-  if (pEpSet) { 
+  if (pEpSet) {
     if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
-      if(pCmd->command < TSDB_SQL_MGMT)  { 
-        tscUpdateVgroupInfo(pSql, pEpSet); 
+      if (pCmd->command < TSDB_SQL_MGMT) {
+        tscUpdateVgroupInfo(pSql, pEpSet);
       } else {
         tscUpdateMgmtEpSet(pEpSet);
-    }
+      }
     }
   }
 
@@ -294,7 +296,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     if (pSql->retry > pSql->maxRetry) {
       tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
     } else {
-      // wait for a little bit moment and then retry
+      // wait for a little bit moment and then retry, todo do not sleep in rpc callback thread
       if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
         int32_t duration = getWaitingTimeInterval(pSql->retry);
         taosMsleep(duration);
@@ -304,6 +306,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
 
       // if there is an error occurring, proceed to the following error handling procedure.
       if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+        taosCacheRelease(tscObjCache, (void**) &p, false);
         rpcFreeCont(rpcMsg->pCont);
         return;
       }
@@ -365,18 +368,21 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
 
+  bool shouldFree = false;
   if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code;
     
-    bool shouldFree = tscShouldBeFreed(pSql);
+    shouldFree = tscShouldBeFreed(pSql);
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
 
     if (shouldFree) {
+      void** p1 = p;
+      taosCacheRelease(tscObjCache, (void **)&p1, true);
       tscDebug("%p sqlObj is automatically freed", pSql);
-      tscFreeSqlObj(pSql);
     }
   }
 
+  taosCacheRelease(tscObjCache, (void**) &p, false);
   rpcFreeCont(rpcMsg->pCont);
 }
 
@@ -2000,7 +2006,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
 
   createHBObj(pObj);
 
-  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
+//  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
 
   return 0;
 }
@@ -2164,6 +2170,10 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
 
+  // TODO add test case on x86 platform
+  uint64_t adr = (uint64_t) pNew;
+  pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000);
+
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated
@@ -2265,6 +2275,9 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
   }
 
   pNewQueryInfo->numOfTables = pQueryInfo->numOfTables;
+
+  uint64_t p = (uint64_t) pNew;
+  pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
   tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables);
 
   pNew->fp = tscTableMetaCallBack;
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 9fa4db999f..33996307ad 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -100,6 +100,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
   }
 
   pObj->signature = pObj;
+  pObj->pDnodeConn = pDnodeConn;
 
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@@ -132,20 +133,15 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     return NULL;
   }
 
-  pSql->pTscObj = pObj;
+  pSql->pTscObj   = pObj;
   pSql->signature = pSql;
-  pSql->maxRetry = TSDB_MAX_REPLICA;
-  tsem_init(&pSql->rspSem, 0, 0);
-  
-  pObj->pDnodeConn = pDnodeConn;
-  
-  pSql->fp = fp;
-  pSql->param = param;
-  if (taos != NULL) {
-    *taos = pObj;
-  }
-
+  pSql->maxRetry  = TSDB_MAX_REPLICA;
+  pSql->fp        = fp;
+  pSql->param     = param;
   pSql->cmd.command = TSDB_SQL_CONNECT;
+
+  tsem_init(&pSql->rspSem, 0, 0);
+
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
@@ -154,7 +150,14 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     return NULL;
   }
 
+  if (taos != NULL) {
+    *taos = pObj;
+  }
+
+  uint64_t key = (uint64_t) pSql;
+  pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
   tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
+
   return pSql;
 }
 
@@ -533,60 +536,72 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 // send free message to vnode to free qhandle and corresponding resources in vnode
-static bool tscKillQueryInVnode(SSqlObj* pSql) {
-  SSqlCmd* pCmd = &pSql->cmd;
-  SSqlRes* pRes = &pSql->res;
-
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
-  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) &&
-      (pCmd->command == TSDB_SQL_SELECT ||
-       pCmd->command == TSDB_SQL_SHOW ||
-       pCmd->command == TSDB_SQL_RETRIEVE ||
-       pCmd->command == TSDB_SQL_FETCH) &&
-      (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
-
-    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
-    tscProcessSql(pSql);
-    return true;
-  }
-
-  return false;
-}
+//static bool tscKillQueryInVnode(SSqlObj* pSql) {
+//  SSqlCmd* pCmd = &pSql->cmd;
+//  SSqlRes* pRes = &pSql->res;
+//
+//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+//  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+//
+//  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+//    return false;
+//  }
+//
+//  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL) &&
+//      (pCmd->command == TSDB_SQL_SELECT ||
+//       pCmd->command == TSDB_SQL_SHOW ||
+//       pCmd->command == TSDB_SQL_RETRIEVE ||
+//       pCmd->command == TSDB_SQL_FETCH)) {
+//
+//    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
+//    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
+//    tscProcessSql(pSql);
+//    return true;
+//  }
+//
+//  return false;
+//}
 
 void taos_free_result(TAOS_RES *res) {
-  SSqlObj *pSql = (SSqlObj *)res;
-
-  if (pSql == NULL || pSql->signature != pSql) {
-    tscDebug("%p sqlObj has been freed", pSql);
-    return;
-  }
-  
-  // The semaphore can not be changed while freeing async sub query objects.
-  SSqlRes *pRes = &pSql->res;
-  if (pRes == NULL || pRes->qhandle == 0) {
-    tscFreeSqlObj(pSql);
-    tscDebug("%p SqlObj is freed by app, qhandle is null", pSql);
+  if (res == NULL) {
     return;
   }
 
-  // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node
-  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-  if (pQueryInfo == NULL) {
-    tscFreeSqlObj(pSql);
-    tscDebug("%p SqlObj is freed by app", pSql);
-    return;
-  }
-
-  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-  if (!tscKillQueryInVnode(pSql)) {
-    tscFreeSqlObj(pSql);
-    tscDebug("%p sqlObj is freed by app", pSql);
-  }
+  SSqlObj* pSql = (SSqlObj*) res;
+  taosCacheRelease(tscObjCache, (void**) &pSql->self, true);
 }
 
+//static void doFreeResult(TAOS_RES *res) {
+//  SSqlObj *pSql = (SSqlObj *)res;
+//
+//  if (pSql == NULL || pSql->signature != pSql) {
+//    tscDebug("%p sqlObj has been freed", pSql);
+//    return;
+//  }
+//
+//  // The semaphore can not be changed while freeing async sub query objects.
+//  SSqlRes *pRes = &pSql->res;
+//  if (pRes == NULL || pRes->qhandle == 0) {
+//    tscFreeSqlObj(pSql);
+//    tscDebug("%p SqlObj is freed by app, qhandle is null", pSql);
+//    return;
+//  }
+//
+//  // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node
+//  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+//  if (pQueryInfo == NULL) {
+//    tscFreeSqlObj(pSql);
+//    tscDebug("%p SqlObj is freed by app", pSql);
+//    return;
+//  }
+//
+//  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
+//  if (!tscKillQueryInVnode(pSql)) {
+//    tscFreeSqlObj(pSql);
+//    tscDebug("%p sqlObj is freed by app", pSql);
+//  }
+//}
+
 int taos_errno(TAOS_RES *tres) {
   SSqlObj *pSql = (SSqlObj *) tres;
   if (pSql == NULL || pSql->signature != pSql) {
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index c4b07f7813..e9ec272ea4 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1512,9 +1512,9 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
   SSqlObj *pParentSql = trsupport->pParentSql;
 
   assert(pSql == pParentSql->pSubs[index]);
-  pParentSql->pSubs[index] = NULL;
-
-  taos_free_result(pSql);
+//  pParentSql->pSubs[index] = NULL;
+//
+//  taos_free_result(pSql);
   taosTFree(trsupport->localBuffer);
   taosTFree(trsupport);
 }
@@ -1728,10 +1728,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
 
   assert(tres != NULL);
   SSqlObj *pSql = (SSqlObj *)tres;
-//  if (pSql == NULL) {  // sql object has been released in error process, return immediately
-//    tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx);
-//    return;
-//  }
 
   SSubqueryState* pState = trsupport->pState;
   assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
@@ -1907,9 +1903,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
     pParentObj->res.code = pSql->res.code;
   }
 
-  taos_free_result(tres);
   taosTFree(pSupporter);
-
   if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
     return;
   }
@@ -2029,11 +2023,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 
   _error:
-  for(int32_t j = 0; j < numOfSub; ++j) {
-    taosTFree(pSql->pSubs[j]->param);
-    taos_free_result(pSql->pSubs[j]);
-  }
-
   taosTFree(pState);
   return TSDB_CODE_TSC_OUT_OF_MEMORY;
 }
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 72f23881d2..2c7fcf05c9 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -30,7 +30,8 @@
 #include "tlocale.h"
 
 // global, not configurable
-void *  tscCacheHandle;
+SCacheObj*  tscCacheHandle;
+SCacheObj*  tscObjCache;
 void *  tscTmr;
 void *  tscQhandle;
 void *  tscCheckDiskUsageTmr;
@@ -146,6 +147,7 @@ void taos_init_imp(void) {
 
   if (tscCacheHandle == NULL) {
     tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
+    tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime, false, tscFreeSqlObjInCache, "sqlObjHandle");
   }
 
   tscDebug("client is initialized successfully");
@@ -157,6 +159,9 @@ void taos_cleanup() {
   if (tscCacheHandle != NULL) {
     taosCacheCleanup(tscCacheHandle);
     tscCacheHandle = NULL;
+
+    taosCacheCleanup(tscObjCache);
+    tscObjCache = NULL;
   }
   
   if (tscQhandle != NULL) {
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 5d50d7791a..47abe60ddd 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -252,11 +252,11 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
   if (pRes->tsrow == NULL) {
     int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput;
     pRes->numOfCols = numOfOutput;
-  
+
     pRes->tsrow  = calloc(numOfOutput, POINTER_BYTES);
     pRes->length = calloc(numOfOutput, sizeof(int32_t));
     pRes->buffer = calloc(numOfOutput, POINTER_BYTES);
-  
+
     // not enough memory
     if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
       taosTFree(pRes->tsrow);
@@ -268,7 +268,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-void tscDestroyResPointerInfo(SSqlRes* pRes) {
+static void tscDestroyResPointerInfo(SSqlRes* pRes) {
   if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
     for (int i = 0; i < pRes->numOfCols; i++) {
       taosTFree(pRes->buffer[i]);
@@ -367,12 +367,36 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   tscResetSqlCmdObj(pCmd, false);
 }
 
+static void tscFreeSubobj(SSqlObj* pSql) {
+  if (pSql->numOfSubs == 0) {
+    return;
+  }
+
+  tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs);
+
+  for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
+    tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i);
+    taos_free_result(pSql->pSubs[i]);
+    pSql->pSubs[i] = NULL;
+  }
+
+  pSql->numOfSubs = 0;
+}
+
+void tscFreeSqlObjInCache(void *pSql) {
+  assert(pSql != NULL);
+  SSqlObj** p = (SSqlObj**) pSql;
+
+  tscFreeSqlObj(*p);
+}
+
 void tscFreeSqlObj(SSqlObj* pSql) {
   if (pSql == NULL || pSql->signature != pSql) {
     return;
   }
 
   tscDebug("%p start to free sql object", pSql);
+  tscFreeSubobj(pSql);
   tscPartiallyFreeSqlObj(pSql);
 
   pSql->signature = NULL;
@@ -724,13 +748,25 @@ void tscCloseTscObj(STscObj* pObj) {
   
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
-  pthread_mutex_destroy(&pObj->mutex);
-  
+
+  // wait for all sqlObjs created according to this connect closed
+  while(1) {
+    pthread_mutex_lock(&pObj->mutex);
+    void* p = pObj->sqlList;
+    pthread_mutex_unlock(&pObj->mutex);
+
+    if (p == NULL) {
+      break;
+    }
+  }
+
   if (pObj->pDnodeConn != NULL) {
     rpcClose(pObj->pDnodeConn);
     pObj->pDnodeConn = NULL;
   }
-  
+
+  pthread_mutex_destroy(&pObj->mutex);
+
   tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
   taosTFree(pObj);
 }
@@ -1721,6 +1757,9 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
 
   tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
+
+  uint64_t p = (uint64_t) pNew;
+  pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
   return pNew;
 }
 
@@ -1960,6 +1999,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex);
   }
 
+  uint64_t p = (uint64_t) pNew;
+  pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10);
   return pNew;
 
 _error:
@@ -2101,11 +2142,6 @@ bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
   return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
 }
 
-bool tscResultsetFetchCompleted(TAOS_RES *result) {
-  SSqlRes* pRes = result;
-  return pRes->completed; 
-}
-
 char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
 
 /**
diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c
index 07cdea1380..e86db2021a 100644
--- a/src/plugins/http/src/httpSql.c
+++ b/src/plugins/http/src/httpSql.c
@@ -50,10 +50,6 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
     }
   }
 
-  // if (tscResultsetFetchCompleted(result)) {
-  //   isContinue = false;
-  // }
-
   if (isContinue) {
     // retrieve next batch of rows
     httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s",
@@ -223,15 +219,6 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
     }
   }
 
-#if 0
-  // todo refactor
-  if (tscResultsetFetchCompleted(result)) {
-    httpDebug("context:%p, fd:%d, ip:%s, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->ipstr,
-              pContext->user);
-    isContinue = false;
-  }
-#endif
-
   if (isContinue) {
     // retrieve next batch of rows
     httpDebug("context:%p, fd:%d, ip:%s, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd,
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index 5fece58ef7..24a19e83d0 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -248,6 +248,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
 
+  // todo apply the lastkey of table check to avoid to load header file
   for (int32_t i = 0; i < sizeOfGroup; ++i) {
     SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
 
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index ab489e2e46..7fda057483 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -165,7 +165,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
     return NULL;
   }
   
-  // set free cache node callback function for hash table
+  // set free cache node callback function
   pCacheObj->freeFp = fn;
   pCacheObj->refreshTime = refreshTimeInSeconds * 1000;
   pCacheObj->extendLifespan = extendLifespan;
@@ -322,7 +322,12 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
 }
 
 void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
-  if (pCacheObj == NULL || (*data) == NULL || (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0)) {
+  if (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) {
+    return;
+  }
+
+  if (pCacheObj == NULL || (*data) == NULL) {
+    uError("cache:%s, NULL data to release", pCacheObj->name);
     return;
   }
   
diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim
index 255e00ca41..bd0d3c1a12 100644
--- a/tests/script/general/parser/groupby.sim
+++ b/tests/script/general/parser/groupby.sim
@@ -423,6 +423,8 @@ if $data97 != @group_tb0@ then
   return -1
 endi
 
+print ---------------------------------> group by binary|nchar data add cases
+
 
 #=========================== group by multi tags ======================
 sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int);

From 6e3affb9231a1fe8e250aa51d91a08600a3a0a1f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 13 Sep 2020 00:52:14 +0800
Subject: [PATCH 19/56] [td-1319]

---
 src/client/src/tscServer.c | 22 +++++++-----
 src/client/src/tscSql.c    | 73 +++++++++++++++++++++++---------------
 src/client/src/tscUtil.c   | 50 +++++++++++++++++++++++---
 3 files changed, 103 insertions(+), 42 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index eda6db8a2b..26fe19d8be 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -248,6 +248,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
 
+  assert(*pSql->self == pSql);
+
   if (pObj->signature != pObj) {
     tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
 
@@ -263,6 +265,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
         pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
 
+    void** p1 = p;
+    taosCacheRelease(tscObjCache, (void**) &p1, false);
+
     taosCacheRelease(tscObjCache, (void**) &p, true);
     rpcFreeCont(rpcMsg->pCont);
     return;
@@ -368,21 +373,20 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
 
-  bool shouldFree = false;
+  bool shouldFree = tscShouldBeFreed(pSql);;
   if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code;
-    
-    shouldFree = tscShouldBeFreed(pSql);
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
-
-    if (shouldFree) {
-      void** p1 = p;
-      taosCacheRelease(tscObjCache, (void **)&p1, true);
-      tscDebug("%p sqlObj is automatically freed", pSql);
-    }
   }
 
+  void** p1 = p;
   taosCacheRelease(tscObjCache, (void**) &p, false);
+
+  if (shouldFree) {
+    taosCacheRelease(tscObjCache, (void **)&p1, true);
+    tscDebug("%p sqlObj is automatically freed", pSql);
+  }
+
   rpcFreeCont(rpcMsg->pCont);
 }
 
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 33996307ad..58859e944c 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -26,6 +26,7 @@
 #include "tsclient.h"
 #include "ttokendef.h"
 #include "tutil.h"
+#include "tscProfile.h"
 
 static bool validImpl(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -536,39 +537,53 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 // send free message to vnode to free qhandle and corresponding resources in vnode
-//static bool tscKillQueryInVnode(SSqlObj* pSql) {
-//  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
-//
-//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-//  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-//
-//  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-//    return false;
-//  }
-//
-//  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL) &&
-//      (pCmd->command == TSDB_SQL_SELECT ||
-//       pCmd->command == TSDB_SQL_SHOW ||
-//       pCmd->command == TSDB_SQL_RETRIEVE ||
-//       pCmd->command == TSDB_SQL_FETCH)) {
-//
-//    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-//    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
-//    tscProcessSql(pSql);
-//    return true;
-//  }
-//
-//  return false;
-//}
+static bool tscKillQueryInDnode(SSqlObj* pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+
+  if (pRes == NULL || pRes->qhandle == 0) {
+    return true;
+  }
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    return true;
+  }
+
+  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  tscRemoveFromSqlList(pSql);
+
+  int32_t cmd = pCmd->command;
+  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
+      (cmd == TSDB_SQL_SELECT ||
+       cmd == TSDB_SQL_SHOW ||
+       cmd == TSDB_SQL_RETRIEVE ||
+       cmd == TSDB_SQL_FETCH)) {
+    pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
+    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
+    tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]);
+
+    tscProcessSql(pSql);
+    return false;
+  }
+
+  return true;
+}
 
 void taos_free_result(TAOS_RES *res) {
-  if (res == NULL) {
+  SSqlObj* pSql = (SSqlObj*) res;
+  if (pSql == NULL || pSql->signature != pSql) {
+    tscError("%p already released sqlObj", res);
     return;
   }
 
-  SSqlObj* pSql = (SSqlObj*) res;
-  taosCacheRelease(tscObjCache, (void**) &pSql->self, true);
+  assert(pSql->self != 0 && *pSql->self == pSql);
+
+  bool freeNow = tscKillQueryInDnode(pSql);
+  if (freeNow) {
+    tscDebug("%p free sqlObj in cache", pSql);
+    taosCacheRelease(tscObjCache, (void**) &pSql->self, true);
+  }
 }
 
 //static void doFreeResult(TAOS_RES *res) {
@@ -596,7 +611,7 @@ void taos_free_result(TAOS_RES *res) {
 //  }
 //
 //  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-//  if (!tscKillQueryInVnode(pSql)) {
+//  if (!tscKillQueryInDnode(pSql)) {
 //    tscFreeSqlObj(pSql);
 //    tscDebug("%p sqlObj is freed by app", pSql);
 //  }
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 47abe60ddd..7d4369816a 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -363,6 +363,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   
   taosTFree(pSql->pSubs);
   pSql->numOfSubs = 0;
+  pSql->self = 0;
   
   tscResetSqlCmdObj(pCmd, false);
 }
@@ -390,12 +391,36 @@ void tscFreeSqlObjInCache(void *pSql) {
   tscFreeSqlObj(*p);
 }
 
+static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    return false;
+  }
+
+  int32_t cmd = pCmd->command;
+  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
+      (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || cmd == TSDB_SQL_RETRIEVE || cmd == TSDB_SQL_FETCH)) {
+    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
+    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
+    tscProcessSql(pSql);
+    return true;
+  }
+
+  return false;
+}
+
 void tscFreeSqlObj(SSqlObj* pSql) {
   if (pSql == NULL || pSql->signature != pSql) {
     return;
   }
 
-  tscDebug("%p start to free sql object", pSql);
+  tscDebug("%p start to free sqlObj", pSql);
+
   tscFreeSubobj(pSql);
   tscPartiallyFreeSqlObj(pSql);
 
@@ -412,6 +437,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tsem_destroy(&pSql->rspSem);
 
   free(pSql);
+  tscDebug("%p free sqlObj completed", pSql);
 }
 
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
@@ -423,7 +449,10 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
   taosTFree(pDataBlock->params);
 
   // free the refcount for metermeta
-  taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false);
+  if (pDataBlock->pTableMeta != NULL) {
+    taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false);
+  }
+
   taosTFree(pDataBlock);
 }
 
@@ -478,7 +507,10 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
   // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache
   if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) {
     tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
-    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+
+    if (pTableMetaInfo->pTableMeta != NULL) {
+      taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+    }
 
     pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta);
   } else {
@@ -758,6 +790,13 @@ void tscCloseTscObj(STscObj* pObj) {
     if (p == NULL) {
       break;
     }
+
+    tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
+    taosMsleep(100);
+
+    // todo fix me!! two threads call taos_free_result will cause problem.
+    tscDebug("%p free :%p", pObj, p);
+    taos_free_result(p);
   }
 
   if (pObj->pDnodeConn != NULL) {
@@ -1703,7 +1742,10 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
     return;
   }
 
-  taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
+  if (pTableMetaInfo->pTableMeta != NULL) {
+    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
+  }
+
   taosTFree(pTableMetaInfo->vgroupList);
   
   tscColumnListDestroy(pTableMetaInfo->tagColList);

From a39fbb1fd6df0bf072a3035cabfe7eeb305139a1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 13:23:12 +0800
Subject: [PATCH 20/56] [td-1319]

---
 src/client/inc/tsclient.h     |   3 +-
 src/client/src/tscAsync.c     |   8 ++-
 src/client/src/tscLocal.c     |   2 +-
 src/client/src/tscSQLParser.c |   3 +-
 src/client/src/tscServer.c    |  60 +++++++++--------
 src/client/src/tscSql.c       |  68 +++++++++----------
 src/client/src/tscStream.c    |   4 +-
 src/client/src/tscSystem.c    |  14 ++--
 src/client/src/tscUtil.c      | 122 +++++++++++++++++++---------------
 src/util/src/tcache.c         |  50 +++++++++-----
 10 files changed, 189 insertions(+), 145 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 11b7815586..74a0e8c11c 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -334,6 +334,7 @@ typedef struct STscObj {
   struct SSqlStream *streamList;
   void*              pDnodeConn;
   pthread_mutex_t    mutex;
+  T_REF_DECLARE();
 } STscObj;
 
 typedef struct SSqlObj {
@@ -503,7 +504,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
   }
 }
 
-extern SCacheObj*    tscCacheHandle;
+extern SCacheObj*    tscMetaCache;
 extern SCacheObj*    tscObjCache;
 extern void *    tscTmr;
 extern void *    tscQhandle;
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 5e9aa1b1f8..09610575f6 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -51,6 +51,11 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   pSql->fp        = fp;
   pSql->fetchFp   = fp;
 
+  uint64_t handle = (uint64_t) pSql;
+  pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
+
+  T_REF_INC(pSql->pTscObj);
+
   pSql->sqlstr = calloc(1, sqlLen + 1);
   if (pSql->sqlstr == NULL) {
     tscError("%p failed to malloc sql string buffer", pSql);
@@ -64,9 +69,6 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   tscDebugL("%p SQL: %s", pSql, pSql->sqlstr);
   pCmd->curSql = pSql->sqlstr;
 
-  uint64_t handle = (uint64_t) pSql;
-  pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
-
   int32_t code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
   
diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index 4b6174e13d..50f82ae662 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -429,7 +429,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
     pRes->qhandle = 0x1;
     pRes->numOfRows = 0;
   } else if (pCmd->command == TSDB_SQL_RESET_CACHE) {
-    taosCacheEmpty(tscCacheHandle);
+    taosCacheEmpty(tscMetaCache);
     pRes->code = TSDB_CODE_SUCCESS;
   } else if (pCmd->command == TSDB_SQL_SERV_VERSION) {
     pRes->code = tscProcessServerVer(pSql);
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index de00827c3a..e8f1a2d758 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1181,13 +1181,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
       } END_TRY
 
     len = tbufTell(&bw);
-    char* c = tbufGetData(&bw, true);
+    char* c = tbufGetData(&bw, false);
 
     // set the serialized binary string as the parameter of arithmetic expression
     addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
 
     insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
 
+    tbufCloseWriter(&bw);
     taosArrayDestroy(colList);
     tExprTreeDestroy(&pNode, NULL);
   } else {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 26fe19d8be..1b6a77bdc7 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -373,17 +373,17 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
 
-  bool shouldFree = tscShouldBeFreed(pSql);;
+  bool shouldFree = tscShouldBeFreed(pSql);
   if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code;
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
   }
 
   void** p1 = p;
-  taosCacheRelease(tscObjCache, (void**) &p, false);
+  taosCacheRelease(tscObjCache, (void**) &p1, false);
 
-  if (shouldFree) {
-    taosCacheRelease(tscObjCache, (void **)&p1, true);
+  if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
+    taosCacheRelease(tscObjCache, (void **)&p, true);
     tscDebug("%p sqlObj is automatically freed", pSql);
   }
 
@@ -1718,7 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
   assert(pTableMetaInfo->pTableMeta == NULL);
 
-  pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name,
+  pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
       strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
   
   // todo handle out of memory case
@@ -1830,7 +1830,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
     //    int32_t size = (int32_t)(rsp - ((char *)pMeta));  // Consistent with STableMeta in cache
     //
     //    pMeta->index = 0;
-    //    (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
+    //    (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
     //  }
   }
   
@@ -1917,12 +1917,14 @@ int tscProcessShowRsp(SSqlObj *pSql) {
   key[0] = pCmd->msgType + 'a';
   strcpy(key + 1, "showlist");
 
-  taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false);
-  
+  if (pTableMetaInfo->pTableMeta != NULL) {
+    taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false);
+  }
+
   size_t size = 0;
   STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
   
-  pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size,
+  pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size,
       tsTableMetaKeepTimer * 1000);
   SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
 
@@ -1981,6 +1983,8 @@ static void createHBObj(STscObj* pObj) {
   pSql->pTscObj = pObj;
   pSql->signature = pSql;
   pObj->pHb = pSql;
+  T_REF_INC(pObj);
+
   tscAddSubqueryInfo(&pObj->pHb->cmd);
 
   tscDebug("%p HB is allocated, pObj:%p", pObj->pHb, pObj);
@@ -2025,14 +2029,14 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
 
 int tscProcessDropDbRsp(SSqlObj *pSql) {
   pSql->pTscObj->db[0] = 0;
-  taosCacheEmpty(tscCacheHandle);
+  taosCacheEmpty(tscMetaCache);
   return 0;
 }
 
 int tscProcessDropTableRsp(SSqlObj *pSql) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
 
-  STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
+  STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
   if (pTableMeta == NULL) { /* not in cache, abort */
     return 0;
   }
@@ -2045,10 +2049,10 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
    * instead.
    */
   tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
-  taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
+  taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
 
   if (pTableMetaInfo->pTableMeta) {
-    taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
+    taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
   }
 
   return 0;
@@ -2057,21 +2061,21 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
 int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
 
-  STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
+  STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
   if (pTableMeta == NULL) { /* not in cache, abort */
     return 0;
   }
 
   tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name);
-  taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
+  taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
 
   if (pTableMetaInfo->pTableMeta) {
     bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
-    taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
+    taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
 
     if (isSuperTable) {  // if it is a super table, reset whole query cache
       tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
-      taosCacheEmpty(tscCacheHandle);
+      taosCacheEmpty(tscMetaCache);
     }
   }
 
@@ -2156,6 +2160,12 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->signature = pNew;
   pNew->cmd.command = TSDB_SQL_META;
 
+  T_REF_INC(pNew->pTscObj);
+
+  // TODO add test case on x86 platform
+  uint64_t adr = (uint64_t) pNew;
+  pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000);
+
   tscAddSubqueryInfo(&pNew->cmd);
 
   SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
@@ -2179,10 +2189,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
 
-  // TODO add test case on x86 platform
-  uint64_t adr = (uint64_t) pNew;
-  pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000);
-
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated
@@ -2196,10 +2202,10 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
 
   // If this STableMetaInfo owns a table meta, release it first
   if (pTableMetaInfo->pTableMeta != NULL) {
-    taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false);
+    taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false);
   }
   
-  pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
+  pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
   if (pTableMetaInfo->pTableMeta != NULL) {
     STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
     tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns,
@@ -2234,7 +2240,7 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) {
              tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta);
   }
 
-  taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
+  taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
   return getTableMetaFromMgmt(pSql, pTableMetaInfo);
 }
 
@@ -2264,7 +2270,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
   pNew->signature = pNew;
 
   pNew->cmd.command = TSDB_SQL_STABLEVGROUP;
-  
+
+  // TODO TEST IT
   SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
   if (pNewQueryInfo == NULL) {
     tscFreeSqlObj(pNew);
@@ -2274,7 +2281,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
-    STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta);
+    STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta);
     tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList);
   }
 
@@ -2284,6 +2291,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
   }
 
   pNewQueryInfo->numOfTables = pQueryInfo->numOfTables;
+  T_REF_INC(pNew->pTscObj);
 
   uint64_t p = (uint64_t) pNew;
   pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 58859e944c..025f0a2b14 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -102,6 +102,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   pObj->signature = pObj;
   pObj->pDnodeConn = pDnodeConn;
+  T_REF_INIT_VAL(pObj, 1);
 
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@@ -155,6 +156,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     *taos = pObj;
   }
 
+  T_REF_INC(pSql->pTscObj);
+
   uint64_t key = (uint64_t) pSql;
   pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
   tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
@@ -261,6 +264,31 @@ void taos_close(TAOS *taos) {
     tscFreeSqlObj(pObj->pHb);
   }
 
+  // free all sqlObjs created by using this connect before free the STscObj
+  while(1) {
+    pthread_mutex_lock(&pObj->mutex);
+    void* p = pObj->sqlList;
+    pthread_mutex_unlock(&pObj->mutex);
+
+    if (p == NULL) {
+      break;
+    }
+
+    tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
+    taosMsleep(100);
+
+    // todo fix me!! two threads call taos_free_result will cause problem.
+    tscDebug("%p free :%p", pObj, p);
+    taos_free_result(p);
+  }
+
+  int32_t ref = T_REF_DEC(pObj);
+  assert(ref >= 0);
+
+  if (ref > 0) {
+    return;
+  }
+
   tscCloseTscObj(pObj);
 }
 
@@ -537,7 +565,7 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 // send free message to vnode to free qhandle and corresponding resources in vnode
-static bool tscKillQueryInDnode(SSqlObj* pSql) {
+static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   SSqlRes* pRes = &pSql->res;
 
@@ -561,7 +589,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) {
        cmd == TSDB_SQL_FETCH)) {
     pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-    tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]);
+    tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql, sqlCmd[pCmd->command]);
 
     tscProcessSql(pSql);
     return false;
@@ -577,46 +605,16 @@ void taos_free_result(TAOS_RES *res) {
     return;
   }
 
-  assert(pSql->self != 0 && *pSql->self == pSql);
+//  assert(pSql->self != 0 && *pSql->self == pSql);
 
   bool freeNow = tscKillQueryInDnode(pSql);
   if (freeNow) {
     tscDebug("%p free sqlObj in cache", pSql);
-    taosCacheRelease(tscObjCache, (void**) &pSql->self, true);
+    SSqlObj** p = pSql->self;
+    taosCacheRelease(tscObjCache, (void**) &p, true);
   }
 }
 
-//static void doFreeResult(TAOS_RES *res) {
-//  SSqlObj *pSql = (SSqlObj *)res;
-//
-//  if (pSql == NULL || pSql->signature != pSql) {
-//    tscDebug("%p sqlObj has been freed", pSql);
-//    return;
-//  }
-//
-//  // The semaphore can not be changed while freeing async sub query objects.
-//  SSqlRes *pRes = &pSql->res;
-//  if (pRes == NULL || pRes->qhandle == 0) {
-//    tscFreeSqlObj(pSql);
-//    tscDebug("%p SqlObj is freed by app, qhandle is null", pSql);
-//    return;
-//  }
-//
-//  // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node
-//  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-//  if (pQueryInfo == NULL) {
-//    tscFreeSqlObj(pSql);
-//    tscDebug("%p SqlObj is freed by app", pSql);
-//    return;
-//  }
-//
-//  pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-//  if (!tscKillQueryInDnode(pSql)) {
-//    tscFreeSqlObj(pSql);
-//    tscDebug("%p sqlObj is freed by app", pSql);
-//  }
-//}
-
 int taos_errno(TAOS_RES *tres) {
   SSqlObj *pSql = (SSqlObj *) tres;
   if (pSql == NULL || pSql->signature != pSql) {
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index 79e0011093..71ba76dc6c 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -167,7 +167,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
              retryDelay);
 
     STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
-    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true);
+    taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true);
     taosTFree(pTableMetaInfo->vgroupList);
   
     tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
@@ -275,7 +275,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
 
     // release the metric/meter meta information reference, so data in cache can be updated
 
-    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+    taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
     tscFreeSqlResult(pSql);
     taosTFree(pSql->pSubs);
     pSql->numOfSubs = 0;
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 2c7fcf05c9..82ce1d3679 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -30,7 +30,7 @@
 #include "tlocale.h"
 
 // global, not configurable
-SCacheObj*  tscCacheHandle;
+SCacheObj*  tscMetaCache;
 SCacheObj*  tscObjCache;
 void *  tscTmr;
 void *  tscQhandle;
@@ -145,9 +145,9 @@ void taos_init_imp(void) {
   refreshTime = refreshTime > 10 ? 10 : refreshTime;
   refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
-  if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
-    tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime, false, tscFreeSqlObjInCache, "sqlObjHandle");
+  if (tscMetaCache == NULL) {
+    tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
+    tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime/2, false, tscFreeSqlObjInCache, "sqlObj");
   }
 
   tscDebug("client is initialized successfully");
@@ -156,9 +156,9 @@ void taos_init_imp(void) {
 void taos_init() { pthread_once(&tscinit, taos_init_imp); }
 
 void taos_cleanup() {
-  if (tscCacheHandle != NULL) {
-    taosCacheCleanup(tscCacheHandle);
-    tscCacheHandle = NULL;
+  if (tscMetaCache != NULL) {
+    taosCacheCleanup(tscMetaCache);
+    tscMetaCache = NULL;
 
     taosCacheCleanup(tscObjCache);
     tscObjCache = NULL;
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 7d4369816a..7f2c1716a6 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -344,8 +344,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   }
 
   SSqlCmd* pCmd = &pSql->cmd;
-  STscObj* pObj = pSql->pTscObj;
-
   int32_t cmd = pCmd->command;
   if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
       cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
@@ -353,11 +351,11 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   }
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-  if (pObj->signature == pObj) {
+//  if (pObj->signature == pObj) {
     //pthread_mutex_lock(&pObj->mutex);
     taosTFree(pSql->sqlstr);
     //pthread_mutex_unlock(&pObj->mutex);
-  }
+//  }
   
   tscFreeSqlResult(pSql);
   
@@ -384,42 +382,61 @@ static void tscFreeSubobj(SSqlObj* pSql) {
   pSql->numOfSubs = 0;
 }
 
+//static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
+//  SSqlCmd* pCmd = &pSql->cmd;
+//  SSqlRes* pRes = &pSql->res;
+//
+//  if (pRes == NULL || pRes->qhandle == 0) {
+//    return true;
+//  }
+//
+//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+//  if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+//    return true;
+//  }
+//
+//  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+//  tscRemoveFromSqlList(pSql);
+//
+//  int32_t cmd = pCmd->command;
+//  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
+//      (cmd == TSDB_SQL_SELECT ||
+//       cmd == TSDB_SQL_SHOW ||
+//       cmd == TSDB_SQL_RETRIEVE ||
+//       cmd == TSDB_SQL_FETCH)) {
+//    pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
+//    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
+//    tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]);
+//
+//    tscProcessSql(pSql);
+//    return false;
+//  }
+//
+//  return true;
+//}
+
+/**
+ * The free operation will cause the pSql to be removed from hash table and free it in
+ * the function of processmsgfromserver is impossible in this case, since it will fail
+ * to retrieve pSqlObj in hashtable.
+ *
+ * @param pSql
+ */
 void tscFreeSqlObjInCache(void *pSql) {
   assert(pSql != NULL);
-  SSqlObj** p = (SSqlObj**) pSql;
+  SSqlObj** p = (SSqlObj**)pSql;
 
+  assert((*p)->self != 0 && (*p)->self == (p));
   tscFreeSqlObj(*p);
 }
 
-static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
-  SSqlCmd* pCmd = &pSql->cmd;
-  SSqlRes* pRes = &pSql->res;
-
-  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
-  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    return false;
-  }
-
-  int32_t cmd = pCmd->command;
-  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
-      (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || cmd == TSDB_SQL_RETRIEVE || cmd == TSDB_SQL_FETCH)) {
-    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
-    tscProcessSql(pSql);
-    return true;
-  }
-
-  return false;
-}
-
 void tscFreeSqlObj(SSqlObj* pSql) {
   if (pSql == NULL || pSql->signature != pSql) {
     return;
   }
 
   tscDebug("%p start to free sqlObj", pSql);
+  STscObj* pTscObj = pSql->pTscObj;
 
   tscFreeSubobj(pSql);
   tscPartiallyFreeSqlObj(pSql);
@@ -438,6 +455,13 @@ void tscFreeSqlObj(SSqlObj* pSql) {
 
   free(pSql);
   tscDebug("%p free sqlObj completed", pSql);
+
+  int32_t ref = T_REF_DEC(pTscObj);
+  assert(ref >= 0);
+
+  if (ref == 0) {
+    tscCloseTscObj(pTscObj);
+  }
 }
 
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
@@ -450,7 +474,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
 
   // free the refcount for metermeta
   if (pDataBlock->pTableMeta != NULL) {
-    taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false);
+    taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
   }
 
   taosTFree(pDataBlock);
@@ -509,10 +533,10 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
     tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
 
     if (pTableMetaInfo->pTableMeta != NULL) {
-      taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+      taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
     }
 
-    pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta);
+    pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta);
   } else {
     assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0);
   }
@@ -583,7 +607,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
    * due to operation such as drop database. So here we add the reference count directly instead of invoke
    * taosGetDataFromCache, which may return NULL value.
    */
-  dataBuf->pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMeta);
+  dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta);
   assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL);
 
   *dataBlocks = dataBuf;
@@ -777,28 +801,10 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
 // TODO: all subqueries should be freed correctly before close this connection.
 void tscCloseTscObj(STscObj* pObj) {
   assert(pObj != NULL);
-  
+
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
 
-  // wait for all sqlObjs created according to this connect closed
-  while(1) {
-    pthread_mutex_lock(&pObj->mutex);
-    void* p = pObj->sqlList;
-    pthread_mutex_unlock(&pObj->mutex);
-
-    if (p == NULL) {
-      break;
-    }
-
-    tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
-    taosMsleep(100);
-
-    // todo fix me!! two threads call taos_free_result will cause problem.
-    tscDebug("%p free :%p", pObj, p);
-    taos_free_result(p);
-  }
-
   if (pObj->pDnodeConn != NULL) {
     rpcClose(pObj->pDnodeConn);
     pObj->pDnodeConn = NULL;
@@ -1743,7 +1749,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
   }
 
   if (pTableMetaInfo->pTableMeta != NULL) {
-    taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
+    taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
   }
 
   taosTFree(pTableMetaInfo->vgroupList);
@@ -1769,6 +1775,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   }
 
   pNew->pTscObj = pSql->pTscObj;
+  T_REF_INC(pNew->pTscObj);
+
   pNew->signature = pNew;
 
   SSqlCmd* pCmd = &pNew->cmd;
@@ -1800,6 +1808,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
 
   tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
 
+  T_REF_INC(pNew->pTscObj);
+
   uint64_t p = (uint64_t) pNew;
   pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
   return pNew;
@@ -1890,6 +1900,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   pNew->pTscObj = pSql->pTscObj;
   pNew->signature = pNew;
+  T_REF_INC(pNew->pTscObj);
 
   pNew->sqlstr = strdup(pSql->sqlstr);
   if (pNew->sqlstr == NULL) {
@@ -1994,14 +2005,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   STableMetaInfo* pFinalInfo = NULL;
 
   if (pPrevSql == NULL) {
-    STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
+    STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
     assert(pTableMeta != NULL);
 
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.
     STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
 
-    STableMeta*  pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta);
+    STableMeta*  pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta);
     
     SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList;
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
@@ -2041,6 +2052,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex);
   }
 
+  T_REF_INC(pNew->pTscObj);
+
   uint64_t p = (uint64_t) pNew;
   pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10);
   return pNew;
@@ -2154,6 +2167,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo,  const char* s
   return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
   
 }
+
 int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) {
   const char* msgFormat1 = "invalid SQL: %s";
   const char* msgFormat2 = "invalid SQL: \'%s\' (%s)";
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 7fda057483..4d737ebe66 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -71,7 +71,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
  * @param pCacheObj    Cache object
  * @param pNode   Cache slot object
  */
-static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
+static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode);
 
 /**
  * remove nodes in trash with refCount == 0 in cache
@@ -80,7 +80,7 @@ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
  * @param force   force model, if true, remove data in trash without check refcount.
  *                may cause corruption. So, forece model only applys before cache is closed
  */
-static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force);
+static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force);
 
 /**
  * release node
@@ -222,7 +222,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
 
           taosTFree(p);
         } else {
-          taosAddToTrash(pCacheObj, p);
+          taosAddToTrashcan(pCacheObj, p);
           uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
         }
       }
@@ -322,11 +322,11 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
 }
 
 void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
-  if (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) {
+  if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) {
     return;
   }
 
-  if (pCacheObj == NULL || (*data) == NULL) {
+  if ((*data) == NULL) {
     uError("cache:%s, NULL data to release", pCacheObj->name);
     return;
   }
@@ -399,19 +399,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
               "others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data);
 
           assert(p->pTNodeHeader == NULL);
-          taosAddToTrash(pCacheObj, p);
+          taosAddToTrashcan(pCacheObj, p);
         } else {
           uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key,
                  pNode->data, ref);
           if (ref > 0) {
             assert(pNode->pTNodeHeader == NULL);
 
-            taosAddToTrash(pCacheObj, pNode);
+            taosAddToTrashcan(pCacheObj, pNode);
           } else {  // ref == 0
             atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
 
             int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
-            uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes",
+            uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes",
                    pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize);
 
             if (pCacheObj->freeFp) {
@@ -432,6 +432,26 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
     char* key = pNode->key;
     char* p = pNode->data;
 
+//    int32_t ref = T_REF_VAL_GET(pNode);
+//
+//    if (ref == 1 && inTrashcan) {
+//      // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be
+//      // destroyed by refresh worker if decrease ref count before removing it from linked-list.
+//      assert(pNode->pTNodeHeader->pData == pNode);
+//
+//      __cache_wr_lock(pCacheObj);
+//      doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
+//      __cache_unlock(pCacheObj);
+//
+//      ref = T_REF_DEC(pNode);
+//      assert(ref == 0);
+//
+//      doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader);
+//    } else {
+//      ref = T_REF_DEC(pNode);
+//      assert(ref >= 0);
+//    }
+
     int32_t ref = T_REF_DEC(pNode);
     uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan);
   }
@@ -452,7 +472,7 @@ static bool travHashTableEmptyFn(void* param, void* data) {
   if (T_REF_VAL_GET(pNode) == 0) {
     taosCacheReleaseNode(pCacheObj, pNode);
   } else { // do add to trashcan
-    taosAddToTrash(pCacheObj, pNode);
+    taosAddToTrashcan(pCacheObj, pNode);
   }
 
   // this node should be remove from hash table
@@ -463,7 +483,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
   SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
 
   taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
-  taosTrashCanEmpty(pCacheObj, false);
+  taosTrashcanEmpty(pCacheObj, false);
 }
 
 void taosCacheCleanup(SCacheObj *pCacheObj) {
@@ -503,7 +523,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *
   return pNewNode;
 }
 
-void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
+void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   if (pNode->inTrashcan) { /* node is already in trash */
     assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode);
     return;
@@ -525,11 +545,11 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   pCacheObj->numOfElemsInTrash++;
   __cache_unlock(pCacheObj);
 
-  uDebug("cache:%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data,
+  uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data,
       pCacheObj->numOfElemsInTrash);
 }
 
-void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
+void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
   __cache_wr_lock(pCacheObj);
 
   if (pCacheObj->numOfElemsInTrash == 0) {
@@ -573,7 +593,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
 
   // todo memory leak if there are object with refcount greater than 0 in hash table?
   taosHashCleanup(pCacheObj->pHashTable);
-  taosTrashCanEmpty(pCacheObj, true);
+  taosTrashcanEmpty(pCacheObj, true);
 
   __cache_lock_destroy(pCacheObj);
   
@@ -648,7 +668,7 @@ void* taosCacheTimedRefresh(void *handle) {
       doCacheRefresh(pCacheObj, now, NULL);
     }
 
-    taosTrashCanEmpty(pCacheObj, false);
+    taosTrashcanEmpty(pCacheObj, false);
   }
 
   return NULL;

From 2bf6b3de2d462c188df4589ea3ce3446d4834329 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 13:54:52 +0800
Subject: [PATCH 21/56] [td-225] update test

---
 src/util/tests/cacheTest.cpp | 42 ++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp
index 51221e0b35..0a4791f6a9 100644
--- a/src/util/tests/cacheTest.cpp
+++ b/src/util/tests/cacheTest.cpp
@@ -12,65 +12,65 @@ int32_t tsMaxMeterConnections = 200;
 // test cache
 TEST(testCase, client_cache_test) {
   const int32_t REFRESH_TIME_IN_SEC = 2;
-  SCacheObj* tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test");
+  SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test");
 
   const char* key1 = "test1";
   char data1[] = "test11";
 
-  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data1, strlen(data1)+1, 1);
+  char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1);
   sleep(REFRESH_TIME_IN_SEC+1);
 
   printf("obj is still valid: %s\n", cachedObj);
 
   char data2[] = "test22";
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj, false);
 
   /* the object is cleared by cache clean operation */
-  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data2, strlen(data2)+1, 20);
+  cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data2, strlen(data2)+1, 20);
   printf("after updated: %s\n", cachedObj);
 
   printf("start to remove data from cache\n");
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj, false);
   printf("end of removing data from cache\n");
 
   const char* key3 = "test2";
   const char* data3 = "kkkkkkk";
 
-  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, strlen(key3), data3, strlen(data3) + 1, 1);
+  char* cachedObj2 = (char*) taosCachePut(tscMetaCache, key3, strlen(key3), data3, strlen(data3) + 1, 1);
   printf("%s\n", cachedObj2);
 
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false);
 
   sleep(3);
-  char* d = (char*) taosCacheAcquireByKey(tscCacheHandle, key3, strlen(key3));
+  char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3));
 //    assert(d == NULL);
 
   char key5[] = "test5";
   char data5[] = "data5kkkkk";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data5, strlen(data5) + 1, 20);
+  cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data5, strlen(data5) + 1, 20);
 
   const char* data6= "new Data after updated";
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false);
 
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data6, strlen(data6) + 1, 20);
+  cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data6, strlen(data6) + 1, 20);
   printf("%s\n", cachedObj2);
 
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true);
 
   const char* data7 = "add call update procedure";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data7, strlen(data7) + 1, 20);
+  cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data7, strlen(data7) + 1, 20);
   printf("%s\n=======================================\n\n", cachedObj2);
 
-  char* cc = (char*) taosCacheAcquireByKey(tscCacheHandle, key5, strlen(key5));
+  char* cc = (char*) taosCacheAcquireByKey(tscMetaCache, key5, strlen(key5));
 
-  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
-  taosCacheRelease(tscCacheHandle, (void**) &cc, false);
+  taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true);
+  taosCacheRelease(tscMetaCache, (void**) &cc, false);
 
   const char* data8 = "ttft";
   const char* key6 = "key6";
 
-  char* ft = (char*) taosCachePut(tscCacheHandle, key6, strlen(key6), data8, strlen(data8), 20);
-  taosCacheRelease(tscCacheHandle, (void**) &ft, false);
+  char* ft = (char*) taosCachePut(tscMetaCache, key6, strlen(key6), data8, strlen(data8), 20);
+  taosCacheRelease(tscMetaCache, (void**) &ft, false);
 
   /**
    * 140ns
@@ -78,14 +78,14 @@ TEST(testCase, client_cache_test) {
   uint64_t startTime = taosGetTimestampUs();
   printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime);
   for(int32_t i=0; i<1000; ++i) {
-    char* dd = (char*) taosCacheAcquireByKey(tscCacheHandle, key6, strlen(key6));
+    char* dd = (char*) taosCacheAcquireByKey(tscMetaCache, key6, strlen(key6));
     if (dd != NULL) {
 //      printf("get the data\n");
     } else {
       printf("data has been released\n");
     }
 
-    taosCacheRelease(tscCacheHandle, (void**) &dd, false);
+    taosCacheRelease(tscMetaCache, (void**) &dd, false);
   }
 
   uint64_t endTime = taosGetTimestampUs();
@@ -93,7 +93,7 @@ TEST(testCase, client_cache_test) {
 
   printf("End of Test, %" PRIu64 "\nTotal Elapsed Time:%" PRIu64 " us.avg:%f us\n", endTime, el, el/1000.0);
 
-  taosCacheCleanup(tscCacheHandle);
+  taosCacheCleanup(tscMetaCache);
 }
 
 TEST(testCase, cache_resize_test) {

From 7d56810a6ed816e3d93079c67fac76932bd3e45d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 14:37:58 +0800
Subject: [PATCH 22/56] [td-225] update test

---
 src/query/src/qParserImpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index 1e8ceffae6..c1b0bd5706 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -179,7 +179,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt
 tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
   tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr));
 
-  if (pRight != NULL && pLeft != NULL) {
+  if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) {
     char* endPos = pRight->token.z + pRight->token.n;
     pExpr->token.z = pLeft->token.z;
     pExpr->token.n = (uint32_t)(endPos - pExpr->token.z);

From fbb76bb9b216a8414520c0ecc3db1327152923f4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 15:54:46 +0800
Subject: [PATCH 23/56] [td-1319]

---
 src/client/src/tscSql.c  | 34 ++++++++++++++++------------------
 src/client/src/tscUtil.c |  4 ++--
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 025f0a2b14..f02e587455 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -265,22 +265,22 @@ void taos_close(TAOS *taos) {
   }
 
   // free all sqlObjs created by using this connect before free the STscObj
-  while(1) {
-    pthread_mutex_lock(&pObj->mutex);
-    void* p = pObj->sqlList;
-    pthread_mutex_unlock(&pObj->mutex);
-
-    if (p == NULL) {
-      break;
-    }
-
-    tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
-    taosMsleep(100);
-
-    // todo fix me!! two threads call taos_free_result will cause problem.
-    tscDebug("%p free :%p", pObj, p);
-    taos_free_result(p);
-  }
+//  while(1) {
+//    pthread_mutex_lock(&pObj->mutex);
+//    void* p = pObj->sqlList;
+//    pthread_mutex_unlock(&pObj->mutex);
+//
+//    if (p == NULL) {
+//      break;
+//    }
+//
+//    tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
+//    taosMsleep(100);
+//
+//    // todo fix me!! two threads call taos_free_result will cause problem.
+//    tscDebug("%p free :%p", pObj, p);
+//    taos_free_result(p);
+//  }
 
   int32_t ref = T_REF_DEC(pObj);
   assert(ref >= 0);
@@ -605,8 +605,6 @@ void taos_free_result(TAOS_RES *res) {
     return;
   }
 
-//  assert(pSql->self != 0 && *pSql->self == pSql);
-
   bool freeNow = tscKillQueryInDnode(pSql);
   if (freeNow) {
     tscDebug("%p free sqlObj in cache", pSql);
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 7f2c1716a6..979e153356 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -366,7 +366,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   tscResetSqlCmdObj(pCmd, false);
 }
 
-static void tscFreeSubobj(SSqlObj* pSql) {
+static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
   if (pSql->numOfSubs == 0) {
     return;
   }
@@ -438,7 +438,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscDebug("%p start to free sqlObj", pSql);
   STscObj* pTscObj = pSql->pTscObj;
 
-  tscFreeSubobj(pSql);
+//  tscFreeSubobj(pSql);
   tscPartiallyFreeSqlObj(pSql);
 
   pSql->signature = NULL;

From dc8b2e281f5cf2b56ab56f93a868d5f5329a26e7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 16:16:25 +0800
Subject: [PATCH 24/56] [td-1391]update the error msg.

---
 src/client/src/tscSQLParser.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index e8f1a2d758..056ea258bd 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -360,8 +360,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
+      // additional msg has been attached already
       if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) {
-        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+        return TSDB_CODE_TSC_INVALID_SQL;
       }
 
       return tscGetTableMeta(pSql, pTableMetaInfo);
@@ -712,8 +713,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
 
 int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) {
   const char* msg1 = "name too long";
-  const char* msg2 = "invalid db name";
-  const char *msg = msg1;
+  const char* msg2 = "current database name is invalid";
 
   SSqlCmd* pCmd = &pSql->cmd;
   int32_t  code = TSDB_CODE_SUCCESS;
@@ -728,17 +728,24 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
   if (hasSpecifyDB(pzTableName)) {
     // db has been specified in sql string so we ignore current db path
     code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL);
+    if (code != 0) {
+      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
   } else {  // get current DB name first, then set it into path
     SStrToken t = {0};
     getCurrentDBName(pSql, &t);
     if (t.n == 0) {
-      msg = msg2;
+      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
     }
+
     code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL);
+    if (code != 0) {
+      invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
   }
+
   if (code != TSDB_CODE_SUCCESS) {
-    invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
-    free(oldName);  
+    taosTFree(oldName);
     return code;
   }
 

From 0c43959191a2b09e017343123129f13e48a09e2f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 16:55:08 +0800
Subject: [PATCH 25/56] [td-1391] fix bugs in release obj in submit processing.

---
 src/client/src/tscSubquery.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index e9ec272ea4..8d908b2506 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1903,7 +1903,9 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
     pParentObj->res.code = pSql->res.code;
   }
 
+  taos_free_result(tres);
   taosTFree(pSupporter);
+
   if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
     return;
   }

From 4bed0a0f3910463a6031b0a4b94332faf155381c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 17:16:23 +0800
Subject: [PATCH 26/56] [td-1391] update the error msg output info.

---
 src/util/src/tsocket.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 61896a86df..498fc6af6b 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -16,7 +16,7 @@
 #include "os.h"
 #include "tulog.h"
 #include "tsocket.h"
-#include "tutil.h"
+#include "taoserror.h"
 
 int taosGetFqdn(char *fqdn) {
   char hostname[1024];
@@ -56,7 +56,13 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
     freeaddrinfo(result);
     return ip;
   } else {
-    uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+    if (ret == EAI_SYSTEM) {
+      uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+    } else {
+      uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+    }
+
     return 0xFFFFFFFF;
   }
 }

From eaa6ca2701d4c02d1f1df9357e6e33d471c0ab13 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 17:44:29 +0800
Subject: [PATCH 27/56] [td-1391] upate the check of vgroup info.

---
 src/client/src/tscServer.c | 5 +++--
 src/util/src/tsocket.c     | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 1b6a77bdc7..32a4df4963 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1677,8 +1677,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   pMetaMsg->contLen = htons(pMetaMsg->contLen);
   pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
 
-  if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) {
-    tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid);
+  if (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0) {
+    tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps,
+        pMetaMsg->vgroup.vgId, pMetaMsg->sid);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 498fc6af6b..479eeaa754 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -60,7 +60,7 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
       uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
     } else {
-      uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+      uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
     }
 
     return 0xFFFFFFFF;

From 89c417d6fa2b0c3b40ff7cc60ba7cf428402fd99 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 17:51:41 +0800
Subject: [PATCH 28/56] [td-1391]

---
 src/client/src/tscServer.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 32a4df4963..f1ef394b8f 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1677,9 +1677,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   pMetaMsg->contLen = htons(pMetaMsg->contLen);
   pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
 
-  if (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0) {
-    tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps,
-        pMetaMsg->vgroup.vgId, pMetaMsg->sid);
+  if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) &&
+      (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
+    tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
+             pMetaMsg->sid);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 

From 5a641b058c755e51db3a7895209e33678ab60f95 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 18:01:41 +0800
Subject: [PATCH 29/56] [td-1391] update test

---
 src/client/src/tscServer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index f1ef394b8f..106cac9595 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1678,9 +1678,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
 
   if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) &&
-      (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
-    tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
-             pMetaMsg->sid);
+      (pMetaMsg->sid < 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
+    tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
+             pMetaMsg->sid, pMetaMsg->tableId);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 

From 62d6f6c9a324eae9fef02d9ce309c2b7830743ef Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 18:02:16 +0800
Subject: [PATCH 30/56] [td-1391] update test

---
 src/client/src/tscServer.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 106cac9595..5639719c15 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1677,8 +1677,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   pMetaMsg->contLen = htons(pMetaMsg->contLen);
   pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
 
-  if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) &&
-      (pMetaMsg->sid < 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
+  if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) &&
+      (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
     tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
              pMetaMsg->sid, pMetaMsg->tableId);
     return TSDB_CODE_TSC_INVALID_VALUE;

From 5ff1dbe5f3ab140704aa60697fe19c3b3d2aced2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 21:41:42 +0800
Subject: [PATCH 31/56] [td-1391]add free subobj

---
 src/client/src/tscSubquery.c |  2 --
 src/client/src/tscUtil.c     | 35 +----------------------------------
 2 files changed, 1 insertion(+), 36 deletions(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 8d908b2506..e9ec272ea4 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1903,9 +1903,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
     pParentObj->res.code = pSql->res.code;
   }
 
-  taos_free_result(tres);
   taosTFree(pSupporter);
-
   if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
     return;
   }
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 979e153356..f06d36e0c6 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -382,39 +382,6 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
   pSql->numOfSubs = 0;
 }
 
-//static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
-//  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
-//
-//  if (pRes == NULL || pRes->qhandle == 0) {
-//    return true;
-//  }
-//
-//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-//  if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-//    return true;
-//  }
-//
-//  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-//  tscRemoveFromSqlList(pSql);
-//
-//  int32_t cmd = pCmd->command;
-//  if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
-//      (cmd == TSDB_SQL_SELECT ||
-//       cmd == TSDB_SQL_SHOW ||
-//       cmd == TSDB_SQL_RETRIEVE ||
-//       cmd == TSDB_SQL_FETCH)) {
-//    pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
-//    pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-//    tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]);
-//
-//    tscProcessSql(pSql);
-//    return false;
-//  }
-//
-//  return true;
-//}
-
 /**
  * The free operation will cause the pSql to be removed from hash table and free it in
  * the function of processmsgfromserver is impossible in this case, since it will fail
@@ -438,7 +405,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscDebug("%p start to free sqlObj", pSql);
   STscObj* pTscObj = pSql->pTscObj;
 
-//  tscFreeSubobj(pSql);
+  tscFreeSubobj(pSql);
   tscPartiallyFreeSqlObj(pSql);
 
   pSql->signature = NULL;

From 9d1619c3b90b0948232913af83d8699bd40f09ab Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 14 Sep 2020 23:37:45 +0800
Subject: [PATCH 32/56] [td-1391] refactor

---
 src/client/src/tscSql.c  | 2 ++
 src/client/src/tscUtil.c | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index f02e587455..4a001a2ce4 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -286,9 +286,11 @@ void taos_close(TAOS *taos) {
   assert(ref >= 0);
 
   if (ref > 0) {
+    tscDebug("%p %d remain sqlObjs, do not close dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
     return;
   }
 
+  tscDebug("%p all sqlObj are freed, free tscObj, dnodeConn:%p", pObj, pObj->pDnodeConn);
   tscCloseTscObj(pObj);
 }
 
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index f06d36e0c6..9b3f5927ab 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -772,6 +772,7 @@ void tscCloseTscObj(STscObj* pObj) {
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
 
+  void* p = pObj->pDnodeConn;
   if (pObj->pDnodeConn != NULL) {
     rpcClose(pObj->pDnodeConn);
     pObj->pDnodeConn = NULL;
@@ -779,7 +780,7 @@ void tscCloseTscObj(STscObj* pObj) {
 
   pthread_mutex_destroy(&pObj->mutex);
 
-  tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
+  tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
   taosTFree(pObj);
 }
 

From 53b2d31c389ae6e493bae3374de7d971bc519d60 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 03:24:05 +0000
Subject: [PATCH 33/56] minor changes

---
 tests/test/c/CMakeLists.txt |   3 +
 tests/test/c/cacheTest.c    | 136 ++++++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 tests/test/c/cacheTest.c

diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 5339203ec2..ffab39d41c 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -39,5 +39,8 @@ IF (TD_LINUX)
   
   add_executable(httpTest httpTest.c)
   target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
+
+  add_executable(cacheTest cacheTest.c)
+  target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
 ENDIF()
 
diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c
new file mode 100644
index 0000000000..091a6662c8
--- /dev/null
+++ b/tests/test/c/cacheTest.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tcache.h"
+#include "tulog.h"
+#include "tutil.h"
+
+#define MAX_REFRESH_TIME_SEC 2
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+int32_t tsKeepTimeInSec = 3;
+int32_t tsNumOfRows = 10000;
+int32_t tsSizeOfRow = 64 * 1024 * 100;
+void *  tsCacheObj = NULL;
+
+typedef int64_t CacheTestKey;
+typedef struct CacheTestRow {
+  int32_t index;
+  void ** ppRow;
+  void *  data;
+} CacheTestRow;
+
+CacheTestRow *initRow(int32_t index) {
+  CacheTestRow *row = calloc(sizeof(CacheTestRow), 1);
+  row->index = index;
+  row->data = malloc(tsSizeOfRow * sizeof(int8_t));
+  return row;
+}
+
+void detroyRow(void *data) {
+  CacheTestRow *row = *(CacheTestRow **)data;
+  free(row->data);
+  free(row);
+}
+
+void initCache() {
+  tsCacheObj = taosCacheInit(TSDB_DATA_TYPE_BIGINT, MAX_REFRESH_TIME_SEC, true, detroyRow, "cachetest");
+}
+
+void putRowInCache() {
+  for (int index = 0; index < tsNumOfRows; ++index) {
+    CacheTestRow *row = initRow(index);
+    uint64_t      key = (uint64_t)row;
+    void **ppRow = taosCachePut(tsCacheObj, &key, sizeof(int64_t), &row, sizeof(int64_t), tsKeepTimeInSec * 1000);
+    row->ppRow = ppRow;
+    taosCacheRelease(tsCacheObj, (void **)&ppRow, false);
+  }
+}
+
+void cleanupCache() {
+  taosCacheCleanup(tsCacheObj);
+}
+
+void initGetMemory() {
+  osInit();
+  taos_init();
+}
+
+float getProcMemory() {
+  float procMemoryUsedMB = 0;
+  taosGetProcMemory(&procMemoryUsedMB);
+  return procMemoryUsedMB;
+}
+
+void doTest() {
+  initCache();
+  pPrint("%s initialize procMemory %f MB %s", GREEN, getProcMemory(), NC);
+
+  putRowInCache();
+  pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC);
+
+  int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000;
+  taosMsleep(sleepMs);
+  pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC);
+
+  cleanupCache();
+  taosMsleep(sleepMs);
+  pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC);
+}
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance of cache\n");
+
+  printf("%s%s\n", indent, "-k");
+  printf("%s%s%s%d\n", indent, indent, "KeepTimeInSec, default is ", tsKeepTimeInSec);
+  printf("%s%s\n", indent, "-n");
+  printf("%s%s%s%d\n", indent, indent, "NumOfRows, default is ", tsNumOfRows);
+  printf("%s%s\n", indent, "-s");
+  printf("%s%s%s%d\n", indent, indent, "SizeOfData, default is ", tsSizeOfRow);
+
+  exit(EXIT_SUCCESS);
+}
+
+void parseArgument(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-k") == 0) {
+      tsKeepTimeInSec = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      tsNumOfRows = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s") == 0) {
+      tsSizeOfRow = atoi(argv[++i]);
+    } else {
+    }
+  }
+
+  pPrint("%s KeepTimeInSec:%d %s", GREEN, tsKeepTimeInSec, NC);
+  pPrint("%s NumOfRows:%d %s", GREEN, tsNumOfRows, NC);
+  pPrint("%s SizeOfData:%d %s", GREEN, tsSizeOfRow, NC);
+}
+
+int main(int argc, char *argv[]) {
+  initGetMemory();
+  parseArgument(argc, argv);
+  doTest();
+}

From 80eddb19df849558db6dab972bb5af2810cce387 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 11:53:46 +0800
Subject: [PATCH 34/56] minor changes

---
 tests/test/c/cacheTest.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c
index 091a6662c8..341b64638e 100644
--- a/tests/test/c/cacheTest.c
+++ b/tests/test/c/cacheTest.c
@@ -26,9 +26,10 @@
 #define NC "\033[0m"
 
 int32_t tsKeepTimeInSec = 3;
-int32_t tsNumOfRows = 10000;
-int32_t tsSizeOfRow = 64 * 1024 * 100;
+int32_t tsNumOfRows = 1000000;
+int32_t tsSizeOfRow = 64 * 1024;
 void *  tsCacheObj = NULL;
+int32_t destroyTimes = 0;
 
 typedef int64_t CacheTestKey;
 typedef struct CacheTestRow {
@@ -48,6 +49,10 @@ void detroyRow(void *data) {
   CacheTestRow *row = *(CacheTestRow **)data;
   free(row->data);
   free(row);
+  destroyTimes++;
+  if (destroyTimes % 50000 == 0) {
+    pPrint("%s ===> destroyTimes:%d %s", GREEN, destroyTimes, NC);
+  }
 }
 
 void initCache() {
@@ -86,13 +91,15 @@ void doTest() {
   putRowInCache();
   pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC);
 
-  int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000;
+  int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3 + 10) * 1000 + tsKeepTimeInSec * 1000;
   taosMsleep(sleepMs);
   pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC);
 
-  cleanupCache();
+  //cleanupCache();
   taosMsleep(sleepMs);
   pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC);
+
+  pPrint("%s finally destroyTimes:%d %s", GREEN, destroyTimes, NC);
 }
 
 void printHelp() {

From 11cc33c575e67eba20ff89eac017523d5a378ed0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 15 Sep 2020 14:02:33 +0800
Subject: [PATCH 35/56] [td-1446]

---
 src/client/src/tscSql.c                   |  4 +-
 src/tsdb/src/tsdbRead.c                   | 56 ++++++++++++++---------
 tests/script/general/parser/testSuite.sim |  1 -
 tests/script/general/parser/topbot.sim    | 51 +++++++++++++++++++--
 4 files changed, 82 insertions(+), 30 deletions(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 4a001a2ce4..69bc69cd4a 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -286,11 +286,11 @@ void taos_close(TAOS *taos) {
   assert(ref >= 0);
 
   if (ref > 0) {
-    tscDebug("%p %d remain sqlObjs, do not close dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
+    tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
     return;
   }
 
-  tscDebug("%p all sqlObj are freed, free tscObj, dnodeConn:%p", pObj, pObj->pDnodeConn);
+  tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
   tscCloseTscObj(pObj);
 }
 
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index 24a19e83d0..3a39d5fc49 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -389,9 +389,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-              "-%" PRId64 ", lastKey:%" PRId64 ", %p",
+              "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
               pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast,
-              pCheckInfo->lastKey, pHandle->qinfo);
+              pCheckInfo->lastKey, pMem->numOfRows, pHandle->qinfo);
 
     if (ASCENDING_TRAVERSE(order)) {
       assert(pCheckInfo->lastKey <= key);
@@ -411,9 +411,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
-              "-%" PRId64 ", lastKey:%" PRId64 ", %p",
+              "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
               pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast,
-              pCheckInfo->lastKey, pHandle->qinfo);
+              pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qinfo);
 
     if (ASCENDING_TRAVERSE(order)) {
       assert(pCheckInfo->lastKey <= key);
@@ -735,6 +735,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo);
 static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end);
 static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols);
 static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle);
@@ -791,9 +792,10 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
      * Here the buffer is not enough, so only part of file block can be loaded into memory buffer
      */
     assert(pQueryHandle->outputCapacity >= binfo.rows);
+    int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo);
 
-    if ((cur->pos == 0 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-        (cur->pos == (binfo.rows - 1) && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) {
+    if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+        (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) {
       pQueryHandle->realNumOfRows = binfo.rows;
 
       cur->rows = binfo.rows;
@@ -809,7 +811,6 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
         cur->pos = -1;
       }
     } else { // partially copy to dest buffer
-      int32_t endPos = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.rows - 1): 0;
       copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos);
       cur->mixBlock = true;
     }
@@ -1204,6 +1205,29 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
             cur->win.ekey, cur->rows, pQueryHandle->qinfo);
 }
 
+int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) {
+  // NOTE: reverse the order to find the end position in data block
+  int32_t endPos = -1;
+  int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
+
+  SQueryFilePos* cur = &pQueryHandle->cur;
+  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+
+  if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) {
+    endPos = pBlockInfo->rows - 1;
+    cur->mixBlock = (cur->pos != 0);
+  } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) {
+    endPos = 0;
+    cur->mixBlock = (cur->pos != pBlockInfo->rows - 1);
+  } else {
+    assert(pCols->numOfRows > 0);
+    endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
+    cur->mixBlock = true;
+  }
+
+  return endPos;
+}
+
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
 static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
@@ -1225,19 +1249,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
 
   STable* pTable = pCheckInfo->pTableObj;
-  int32_t endPos = cur->pos;
-
-  if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
-    endPos = blockInfo.rows - 1;
-    cur->mixBlock = (cur->pos != 0);
-  } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) {
-    endPos = 0;
-    cur->mixBlock = (cur->pos != blockInfo.rows - 1);
-  } else {
-    assert(pCols->numOfRows > 0);
-    endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
-    cur->mixBlock = true;
-  }
+  int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo);
 
   tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d,"
             "end:%d, %p",
@@ -1339,8 +1351,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   }
 
   cur->blockCompleted =
-      (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-       ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+      (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+       ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
 
   if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
     SWAP(cur->win.skey, cur->win.ekey, TSKEY);
diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim
index f42254981c..3dd80b8e38 100644
--- a/tests/script/general/parser/testSuite.sim
+++ b/tests/script/general/parser/testSuite.sim
@@ -105,7 +105,6 @@ run general/parser/timestamp.sim
 sleep 2000
 run general/parser/sliding.sim
 
-
 #sleep 2000
 #run general/parser/repeatStream.sim
 #sleep 2000
diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim
index 2faee55460..3e906f3a28 100644
--- a/tests/script/general/parser/topbot.sim
+++ b/tests/script/general/parser/topbot.sim
@@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0
 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200
 
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
+sleep 1000
 sql connect
 
 $dbPrefix = tb_db
@@ -25,7 +25,7 @@ $stb = $stbPrefix . $i
 
 sql drop database $db -x step1
 step1:
-sql create database $db cache 16
+sql create database $db cache 16 maxrows 4096 keep 36500
 print ====== create tables
 sql use $db
 sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int)
@@ -132,15 +132,15 @@ sleep 5000
 system sh/exec.sh -n dnode1 -s start
 print ================== server restart completed
 sql connect
-sleep 3000
+sleep 1000
 
 sql select count(*) from t1.test where ts>10000 and ts<90000 interval(5000a)
 if $rows != 3 then
  return -1
 endi
 
-print =========>td-1308
-sql create database db;
+print ==============>td-1308
+sql create database db keep 36500;
 sql use db;
 
 sql create table stb (ts timestamp, c1 int, c2 binary(10)) tags(t1 binary(10));
@@ -158,4 +158,45 @@ if $rows != 1 then
   return -1
 endi
 
+print =======================>td-1446
+sql create table t(ts timestamp, k int)
+$ts = 6000
+while $ts < 7000
+  sql insert into t values ( $ts , $ts )
+  $ts = $ts + 1
+endw
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sleep 1000
+sql use db;
+
+$ts = 1000
+while $ts < 5096
+  sql insert into t values ( $ts , $ts )
+  $ts = $ts + 1
+endw
+
+sql select * from t where ts < 6500
+if $rows != 4596 then
+  print expect 4596, actual: $rows
+  return -1
+endi
+
+sql select * from t where ts < 7000
+if $rows != 5096 then
+  return -1
+endi
+
+sql select * from t where ts <= 6000
+if $rows != 4097 then
+  return -1
+endi
+
+sql select * from t where ts <= 6001
+if $rows != 4098 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

From c233807ddc9f0e8492a5c3e9b6fd63e5df2fd4ec Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 15 Sep 2020 14:30:40 +0800
Subject: [PATCH 36/56] [TECO-20] <doc> remove docker icon

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 411dddb9f7..522fc0ebc1 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,6 @@
 [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
 [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
-[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine)
 [![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine)
 
 [![TDengine](TDenginelogo.png)](https://www.taosdata.com)

From bffd30ef80ddb2162cfcca2b7da696d916c221d1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 15 Sep 2020 15:21:22 +0800
Subject: [PATCH 37/56] [td-1290]

---
 src/query/src/qExecutor.c | 131 ++++++++++++++++----------------------
 1 file changed, 56 insertions(+), 75 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d48d7d5ea1..6b67c7ceb9 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1951,36 +1951,36 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
 
   // todo handle the case the the order irrelevant query type mixed up with order critical query type
   // descending order query for last_row query
-  if (isFirstLastRowQuery(pQuery) && !QUERY_IS_ASC_QUERY(pQuery)) {
+  if (isFirstLastRowQuery(pQuery)) {
     qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery),
            pQuery->order.order, TSDB_ORDER_ASC);
 
-    SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
     pQuery->order.order = TSDB_ORDER_ASC;
-    assert (pQuery->window.skey <= pQuery->window.ekey);
+    if (pQuery->window.skey > pQuery->window.ekey) {
+      SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
+    }
+
+    return;
+  }
+
+  if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) {
+    pQuery->order.order = TSDB_ORDER_ASC;
+    if (pQuery->window.skey > pQuery->window.ekey) {
+      SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
+    }
 
     doExchangeTimeWindow(pQInfo, &pQuery->window);
     return;
   }
 
-  if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && !QUERY_IS_ASC_QUERY(pQuery)) {
-    pQuery->order.order = TSDB_ORDER_ASC;
-    SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
-    assert (pQuery->window.skey <= pQuery->window.ekey);
-
-    doExchangeTimeWindow(pQInfo, &pQuery->window);
-    return;
-  }
-
-  if (isPointInterpoQuery(pQuery) && (pQuery->intervalTime == 0) && !QUERY_IS_ASC_QUERY(pQuery)) {
-    qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey,
-           pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey);
-    SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
+  if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) {
+    if (!QUERY_IS_ASC_QUERY(pQuery)) {
+      qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey,
+             pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey);
+      SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
+    }
 
     pQuery->order.order = TSDB_ORDER_ASC;
-
-    assert (pQuery->window.skey <= pQuery->window.ekey);
-    doExchangeTimeWindow(pQInfo, &pQuery->window);
     return;
   }
 
@@ -2365,16 +2365,16 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
           memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes));
           pQuery->sdata[i] = (tFilePage *)tmp;
         }
-        
+
         // set the pCtx output buffer position
         pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes;
-        
+
         int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
         if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
           pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
         }
       }
-      
+
       qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv),
              newSize, pRec->capacity, newSize - pRec->rows);
 
@@ -2920,11 +2920,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     STableQueryInfo *item = taosArrayGetP(pGroup, i);
 
     SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid);
+    pageList = list;
+    tid = TSDB_TABLEID(item->pTable)->tid;
 
     if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) {
       pTableList[numOfTables++] = item;
-      tid = TSDB_TABLEID(item->pTable)->tid;
-      pageList = list;
     }
   }
 
@@ -3357,7 +3357,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
-      
+
       memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes));
       pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes;
 
@@ -4354,32 +4354,6 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
   return true;
 }
 
-static void freeTableQueryInfo(STableGroupInfo* pTableGroupInfo) {
-  if (pTableGroupInfo->pGroupList == NULL) {
-    assert(pTableGroupInfo->numOfTables == 0);
-  } else {
-    size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
-    for (int32_t i = 0; i < numOfGroups; ++i) {
-      SArray *p = taosArrayGetP(pTableGroupInfo->pGroupList, i);
-
-      size_t num = taosArrayGetSize(p);
-      for(int32_t j = 0; j < num; ++j) {
-        STableQueryInfo* item = taosArrayGetP(p, j);
-        destroyTableQueryInfo(item);
-      }
-
-      taosArrayDestroy(p);
-    }
-
-    taosArrayDestroy(pTableGroupInfo->pGroupList);
-    pTableGroupInfo->pGroupList = NULL;
-    pTableGroupInfo->numOfTables = 0;
-  }
-
-  taosHashCleanup(pTableGroupInfo->map);
-  pTableGroupInfo->map = NULL;
-}
-
 static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
@@ -4415,22 +4389,20 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
   terrno = TSDB_CODE_SUCCESS;
   if (isFirstLastRowQuery(pQuery)) {
     pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
-    if (pRuntimeEnv->pQueryHandle == NULL) {  // no data in current stable, clear all
-      freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
-    } else { // update the query time window
-      pQuery->window = cond.twindow;
 
-      size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
-      for (int32_t i = 0; i < numOfGroups; ++i) {
-        SArray *group = GET_TABLEGROUP(pQInfo, i);
+    // update the query time window
+    pQuery->window = cond.twindow;
 
-        size_t t = taosArrayGetSize(group);
-        for (int32_t j = 0; j < t; ++j) {
-          STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
+    size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
+    for(int32_t i = 0; i < numOfGroups; ++i) {
+      SArray *group = GET_TABLEGROUP(pQInfo, i);
 
-          pCheckInfo->win = pQuery->window;
-          pCheckInfo->lastKey = pCheckInfo->win.skey;
-        }
+      size_t t = taosArrayGetSize(group);
+      for (int32_t j = 0; j < t; ++j) {
+        STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
+
+        pCheckInfo->win = pQuery->window;
+        pCheckInfo->lastKey = pCheckInfo->win.skey;
       }
     }
   } else if (isPointInterpoQuery(pQuery)) {
@@ -4484,12 +4456,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
     return code;
   }
 
-  if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
-    qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
-    setQueryStatus(pQuery, QUERY_COMPLETED);
-    return TSDB_CODE_SUCCESS;
-  }
-
   pQInfo->tsdb = tsdb;
   pQInfo->vgId = vgId;
 
@@ -4612,7 +4578,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery*           pQuery = pRuntimeEnv->pQuery;
   SQueryCostInfo*   summary  = &pRuntimeEnv->summary;
-  
+
   int64_t st = taosGetTimestampMs();
 
   TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
@@ -4622,7 +4588,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
 
   while (tsdbNextDataBlock(pQueryHandle)) {
     summary->totalBlocks += 1;
-    
+
     if (IS_QUERY_KILLED(pQInfo)) {
       longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
     }
@@ -4659,7 +4625,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
 
     summary->totalRows += blockInfo.rows;
     stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey);
-  
+
     qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, "
            "lastKey:%" PRId64,
            pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
@@ -6383,10 +6349,25 @@ static void freeQInfo(SQInfo *pQInfo) {
     taosTFree(pQuery);
   }
 
-  freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
+  // todo refactor, extract method to destroytableDataInfo
+  if (pQInfo->tableqinfoGroupInfo.pGroupList != NULL) {
+    int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
+    for (int32_t i = 0; i < numOfGroups; ++i) {
+      SArray *p = GET_TABLEGROUP(pQInfo, i);
+
+      size_t num = taosArrayGetSize(p);
+      for(int32_t j = 0; j < num; ++j) {
+        STableQueryInfo* item = taosArrayGetP(p, j);
+        destroyTableQueryInfo(item);
+      }
+
+      taosArrayDestroy(p);
+    }
+  }
 
   taosTFree(pQInfo->pBuf);
-
+  taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList);
+  taosHashCleanup(pQInfo->tableqinfoGroupInfo.map);
   tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
   taosArrayDestroy(pQInfo->arrTableIdInfo);
 

From 4d6e0ef40e11118cd24f880f7cc1d296685fed27 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 15 Sep 2020 16:09:48 +0800
Subject: [PATCH 38/56] [td-1454]

---
 src/client/src/tscSubquery.c           | 15 +++++++--------
 tests/script/general/parser/topbot.sim | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index e9ec272ea4..7ea5d4183b 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -2198,16 +2198,15 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
     // calculate the result from several other columns
     if (pSup->pArithExprInfo != NULL) {
       if (pRes->pArithSup == NULL) {
-        SArithmeticSupport *sas = (SArithmeticSupport *) calloc(1, sizeof(SArithmeticSupport));
-        sas->offset     = 0;
-        sas->pArithExpr = pSup->pArithExprInfo;
-        sas->numOfCols  = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
-        sas->exprList   = pQueryInfo->exprList;
-        sas->data       = calloc(sas->numOfCols, POINTER_BYTES);
-
-        pRes->pArithSup = sas;
+        pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport));
       }
 
+      pRes->pArithSup->offset     = 0;
+      pRes->pArithSup->pArithExpr = pSup->pArithExprInfo;
+      pRes->pArithSup->numOfCols  = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
+      pRes->pArithSup->exprList   = pQueryInfo->exprList;
+      pRes->pArithSup->data       = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES);
+
       if (pRes->buffer[i] == NULL) {
         TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
         pRes->buffer[i] = malloc(field->bytes);
diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim
index 3e906f3a28..5c575b6163 100644
--- a/tests/script/general/parser/topbot.sim
+++ b/tests/script/general/parser/topbot.sim
@@ -199,4 +199,18 @@ if $rows != 4098 then
   return -1
 endi
 
+print ======================>td-1454
+sql select count(*)/10, count(*)+99 from t
+if $rows != 1  then
+  return -1
+endi
+
+if $data00 != 509.600000000 then
+  return -1
+endi
+
+if $data01 != 5195.000000000 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

From 41a21ca29ebefe2bd69b9c5a96c53498730efed0 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 16:23:37 +0800
Subject: [PATCH 39/56] TD-1225

---
 src/client/src/tscParseInsert.c          |  2 +-
 src/plugins/http/src/httpServer.c        |  6 ++--
 tests/script/general/http/autocreate.sim |  5 +++
 tests/script/general/http/chunked.sim    | 37 +++++++++++++++++++++
 tests/script/general/http/gzip.sim       | 27 ++++++++++++++++
 tests/script/general/http/testSuite.sim  |  2 ++
 tests/script/jenkins/basic.txt           |  2 ++
 tests/test/c/cacheTest.c                 | 11 ++++---
 tests/tsim/inc/sim.h                     |  2 ++
 tests/tsim/src/simExe.c                  | 41 ++++++++++++++++++++++++
 tests/tsim/src/simParse.c                | 14 ++++++++
 11 files changed, 141 insertions(+), 8 deletions(-)
 create mode 100644 tests/script/general/http/chunked.sim
 create mode 100644 tests/script/general/http/gzip.sim

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 09eb8f167e..18926a84b9 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe
   int32_t   index = 0;
   SStrToken sToken;
 
-  int16_t numOfRows = 0;
+  int32_t numOfRows = 0;
 
   SSchema *pSchema = tscGetTableSchema(pTableMeta);
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index 2d95a0ac72..d85a236cb1 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -317,10 +317,10 @@ static bool httpReadData(HttpContext *pContext) {
   pContext->lastAccessTime = taosGetTimestampSec();
 
   char    buf[HTTP_STEP_SIZE + 1] = {0};
-  int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf));
+  int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE);
   if (nread > 0) {
     buf[nread] = '\0';
-    httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread);
+    httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf);
     int32_t ok = httpParseBuf(pParser, buf, nread);
 
     if (ok) {
@@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) {
       httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos);
       return false;
     } else {
-      httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str);
+      httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos);
       return true;
     }
   } else if (nread < 0) {
diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim
index 6a005b028a..98d64ab839 100644
--- a/tests/script/general/http/autocreate.sim
+++ b/tests/script/general/http/autocreate.sim
@@ -24,5 +24,10 @@ print curl 127.0.0.1:7111/rest/sql -----> $system_content
 #  return -1
 #endi
 
+sql select * from db.win_cpu_windows_1_processor
+print rows: $rows
+if $rows != 1 then
+  return -1
+endi
 
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim
new file mode 100644
index 0000000000..8673655d96
--- /dev/null
+++ b/tests/script/general/http/chunked.sim
@@ -0,0 +1,37 @@
+system sh/stop_dnodes.sh
+sleep 3000
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032
+system sh/exec.sh -n dnode1 -s start
+
+sleep 3000
+sql connect
+
+print ============================ dnode1 start
+
+print ===============  step1 - prepare data
+sql create database d1
+sql use d1
+
+sql create table table_rest (ts timestamp, i int) 
+print sql length is 270KB
+restful d1 table_rest 1591072800 10000
+restful d1 table_rest 1591172800 10000
+restful d1 table_rest 1591272800 10000
+restful d1 table_rest 1591372800 10000
+restful d1 table_rest 1591472800 10000
+restful d1 table_rest 1591572800 10000
+restful d1 table_rest 1591672800 10000
+restful d1 table_rest 1591772800 10000
+restful d1 table_rest 1591872800 10000
+restful d1 table_rest 1591972800 10000
+
+sql select * from table_rest;
+print rows: $rows
+if $rows != 100000 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim
new file mode 100644
index 0000000000..0289e337a6
--- /dev/null
+++ b/tests/script/general/http/gzip.sim
@@ -0,0 +1,27 @@
+system sh/stop_dnodes.sh
+sleep 3000
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c wallevel -v 0
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032
+system sh/exec.sh -n dnode1 -s start
+
+sleep 3000
+sql connect
+
+print ============================ dnode1 start
+
+print ===============  step1 - prepare data
+sql create database d1
+sql use d1
+
+sql create table table_rest (ts timestamp, i int) 
+print sql length is 270KB
+restful d1 table_rest 1591072800 10000 gzip
+sql select * from table_rest;
+print rows: $rows
+if $rows != 10000 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim
index d91e9f452d..f35362bf07 100644
--- a/tests/script/general/http/testSuite.sim
+++ b/tests/script/general/http/testSuite.sim
@@ -1,3 +1,5 @@
+run general/http/autocreate.sim
+run general/http/chunked.sim
 run general/http/restful.sim
 run general/http/restful_insert.sim
 run general/http/restful_limit.sim
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 063d11bd9d..adb22aa265 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -77,6 +77,8 @@ cd ../../../debug; make
 ./test.sh -f general/field/smallint.sim
 ./test.sh -f general/field/tinyint.sim
 
+./test.sh -f general/http/autocreate.sim
+./test.sh -f general/http/chunked.sim
 ./test.sh -f general/http/restful.sim
 ./test.sh -f general/http/restful_insert.sim
 ./test.sh -f general/http/restful_limit.sim
diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c
index 341b64638e..54aca0038e 100644
--- a/tests/test/c/cacheTest.c
+++ b/tests/test/c/cacheTest.c
@@ -14,6 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
+#include <malloc.h>
 #include "os.h"
 #include "taos.h"
 #include "tcache.h"
@@ -91,15 +92,17 @@ void doTest() {
   putRowInCache();
   pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC);
 
-  int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3 + 10) * 1000 + tsKeepTimeInSec * 1000;
+  int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000;
   taosMsleep(sleepMs);
   pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC);
 
-  //cleanupCache();
+  cleanupCache();
   taosMsleep(sleepMs);
   pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC);
-
-  pPrint("%s finally destroyTimes:%d %s", GREEN, destroyTimes, NC);
+  
+  malloc_trim(0);
+  taosMsleep(sleepMs);
+  pPrint("%s after malloc_trim, procMemory %f MB %s", GREEN, getProcMemory(), NC);
 }
 
 void printHelp() {
diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h
index 6f3bc7099d..58e58a442c 100644
--- a/tests/tsim/inc/sim.h
+++ b/tests/tsim/inc/sim.h
@@ -84,6 +84,7 @@ enum {
   SIM_CMD_SQL,
   SIM_CMD_SQL_ERROR,
   SIM_CMD_SQL_SLOW,
+  SIM_CMD_RESTFUL,
   SIM_CMD_TEST,
   SIM_CMD_RETURN,
   SIM_CMD_END
@@ -172,6 +173,7 @@ bool simExecuteReturnCmd(SScript *script, char *option);
 bool simExecuteSqlCmd(SScript *script, char *option);
 bool simExecuteSqlErrorCmd(SScript *script, char *rest);
 bool simExecuteSqlSlowCmd(SScript *script, char *option);
+bool simExecuteRestfulCmd(SScript *script, char *rest);
 void simVisuallizeOption(SScript *script, char *src, char *dst);
 
 #endif
\ No newline at end of file
diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index adc2fd0b9d..463dc33c7c 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -915,6 +915,47 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) {
   return simExecuteSqlImpCmd(script, rest, isSlow);
 }
 
+bool simExecuteRestfulCmd(SScript *script, char *rest) {
+  FILE *fp = NULL;
+  char filename[256];
+  sprintf(filename, "%s/tmp.sql", tsScriptDir);  
+  fp = fopen(filename, "w");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to open file: %s\n", filename);
+    return false;
+  }
+
+  char db[64] = {0};
+  char tb[64] = {0};
+  char gzip[32] = {0};
+  int32_t ts;
+  int32_t times;
+  sscanf(rest, "%s %s %d %d %s", db, tb, &ts, &times, gzip);
+  
+  fprintf(fp, "insert into %s.%s values ", db, tb);
+  for (int i = 0; i < times; ++i) {
+    fprintf(fp, "(%d000, %d)", ts + i, ts);
+  }
+  fprintf(fp, "  \n");
+  fflush(fp);
+  fclose(fp);
+
+  char cmd[1024] = {0};
+  if (strcmp(gzip, "gzip") == 0) {
+    sprintf(cmd,
+            "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header "
+            "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql",
+            filename);
+  } else {
+    sprintf(cmd,
+            "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header "
+            "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql",
+            filename);
+  }
+
+  return simExecuteSystemCmd(script, cmd);
+}
+
 bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
   char buf[3000];
   SCmdLine *line = &script->lines[script->linePos];
diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c
index 8dcf83806f..2e6121304f 100644
--- a/tests/tsim/src/simParse.c
+++ b/tests/tsim/src/simParse.c
@@ -721,6 +721,12 @@ bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) {
   return true;
 }
 
+bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) {
+  simParseSqlCmd(rest, pCmd, lineNum);
+  cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL;
+  return true;
+}
+
 bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) {
   int expLen;
 
@@ -1020,6 +1026,14 @@ void simInitsimCmdList() {
   simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd;
   simAddCmdIntoHash(&(simCmdList[cmdno]));
 
+  cmdno = SIM_CMD_RESTFUL;
+  simCmdList[cmdno].cmdno = cmdno;
+  strcpy(simCmdList[cmdno].name, "restful");
+  simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name);
+  simCmdList[cmdno].parseCmd = simParseRestfulCmd;
+  simCmdList[cmdno].executeCmd = simExecuteRestfulCmd;
+  simAddCmdIntoHash(&(simCmdList[cmdno]));
+
   /* test is only an internal command */
   cmdno = SIM_CMD_TEST;
   simCmdList[cmdno].cmdno = cmdno;

From 2372e4decc278be058a643376eb498213081365b Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 17:26:48 +0800
Subject: [PATCH 40/56] TD-1451

---
 src/plugins/http/src/httpParser.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c
index cad5dd8f91..0c3204687a 100644
--- a/src/plugins/http/src/httpParser.c
+++ b/src/plugins/http/src/httpParser.c
@@ -238,6 +238,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
     httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive);
   }
 
+#if 0
   else if (0 == strcasecmp(key, "Content-Encoding")) {
     if (0 == strcmp(val, "gzip")) {
       parser->contentChunked = 1;
@@ -245,8 +246,9 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
     }
     return 0;
   }
+  #endif
 
-  else if (0 == strcasecmp(key, "Transfer-Encoding")) {
+  else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) {
     if (strstr(val, "gzip")) {
       parser->transferGzip = 1;
       ehttp_gzip_conf_t      conf = {0};

From 53c4f51116cd1cf66752f7ea12b4b4ec65f92288 Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Tue, 15 Sep 2020 18:21:40 +0800
Subject: [PATCH 41/56] [TD-1427]

---
 cmake/install.inc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/install.inc b/cmake/install.inc
index 7a92a396e3..0531d40048 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -13,7 +13,7 @@ ELSEIF (TD_WINDOWS)
   IF (NOT TD_GODLL)
     #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
     #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
-    #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
+    INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
     INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
     INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
     INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)

From 0015100d2bda2cafa1d69ad43018eb05619772d1 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 15:40:10 +0000
Subject: [PATCH 42/56] TD-1364

---
 src/common/src/tglobal.c            | 11 -----
 src/plugins/http/src/httpRestJson.c | 28 +++++-------
 src/plugins/http/src/httpServer.c   | 69 +++++++++++++++--------------
 src/util/src/tlog.c                 |  2 +-
 4 files changed, 49 insertions(+), 61 deletions(-)

diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 96e8fb26c6..7e46f58a93 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -957,17 +957,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  // http configs
-  cfg.option = "httpCacheSessions";
-  cfg.ptr = &tsHttpCacheSessions;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 1;
-  cfg.maxValue = 100000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   cfg.option = "httpEnableRecordSql";
   cfg.ptr = &tsHttpEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c
index 26f0441519..f8912331a3 100644
--- a/src/plugins/http/src/httpRestJson.c
+++ b/src/plugins/http/src/httpRestJson.c
@@ -87,15 +87,12 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
   JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
   if (jsonBuf == NULL) return false;
 
-  cmd->numOfRows += numOfRows;
-
   int32_t     num_fields = taos_num_fields(result);
   TAOS_FIELD *fields = taos_fetch_fields(result);
 
   for (int32_t k = 0; k < numOfRows; ++k) {
     TAOS_ROW row = taos_fetch_row(result);
     if (row == NULL) {
-      cmd->numOfRows--;
       continue;
     }
     int32_t* length = taos_fetch_lengths(result);
@@ -151,24 +148,23 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
     }
 
     // data row array end
-    httpJsonToken(jsonBuf, JsonArrEnd);
-  }
+    httpJsonToken(jsonBuf, JsonArrEnd);   
+    cmd->numOfRows ++;
 
-  if (cmd->numOfRows >= tsRestRowLimit) {
-    httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext,
-              pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit);
-    return false;
-  } else {
     if (pContext->fd <= 0) {
-      httpError("context:%p, fd:%d, user:%s, connection is closed, abort retrieve", pContext, pContext->fd,
-                pContext->user);
+      httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user);
+      return false;
+    }
+
+    if (cmd->numOfRows >= tsRestRowLimit) {
+      httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext,
+                pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit);
       return false;
-    } else {
-      httpDebug("context:%p, fd:%d, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->user,
-                cmd->numOfRows);
-      return true;
     }
   }
+
+  httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows);
+  return true;
 }
 
 bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index d85a236cb1..f0a7249b51 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -315,45 +315,48 @@ static bool httpReadData(HttpContext *pContext) {
 
   pContext->accessTimes++;
   pContext->lastAccessTime = taosGetTimestampSec();
+  char buf[HTTP_STEP_SIZE + 1] = {0};
 
-  char    buf[HTTP_STEP_SIZE + 1] = {0};
-  int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE);
-  if (nread > 0) {
-    buf[nread] = '\0';
-    httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf);
-    int32_t ok = httpParseBuf(pParser, buf, nread);
+  while (1) {
+    int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE);
+    if (nread > 0) {
+      buf[nread] = '\0';
+      httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf);
+      int32_t ok = httpParseBuf(pParser, buf, nread);
 
-    if (ok) {
-      httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, pParser->parseCode);
-      httpSendErrorResp(pContext, pParser->parseCode);
-      httpNotifyContextClose(pContext);
-      return false;
-    }
+      if (ok) {
+        httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok,
+                  pParser->parseCode);
+        httpSendErrorResp(pContext, pParser->parseCode);
+        httpNotifyContextClose(pContext);
+        return false;
+      }
 
-    if (pParser->parseCode) {
-      httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode);
-      httpSendErrorResp(pContext, pParser->parseCode);
-      httpNotifyContextClose(pContext);
-      return false;
-    }
+      if (pParser->parseCode) {
+        httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode);
+        httpSendErrorResp(pContext, pParser->parseCode);
+        httpNotifyContextClose(pContext);
+        return false;
+      }
 
-    if (!pParser->parsed) {
-      httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos);
-      return false;
+      if (!pParser->parsed) {
+        httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd);
+        continue;
+      } else {
+        httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos);
+        return true;
+      }
+    } else if (nread < 0) {
+      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+        httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno);
+        return false;  // later again
+      } else {
+        httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno);
+        return false;
+      }
     } else {
-      httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos);
-      return true;
-    }
-  } else if (nread < 0) {
-    if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
-      httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno);
-      return false;  // later again
-    } else {
-      httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno);
+      httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread);
       return false;
     }
-  } else {
-    httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread);
-    return false;
   }
 }
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index a8587de767..e5afe1b68e 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -433,7 +433,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
 
   va_list        argpointer;
   char           buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE];
-  int32_t            len;
+  int32_t        len;
   struct tm      Tm, *ptm;
   struct timeval timeSecs;
   time_t         curTime;

From 4454b077ff2c4fb444657a47fb827fd3274255b0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 15 Sep 2020 23:49:17 +0800
Subject: [PATCH 43/56] [td-1456]

---
 src/client/src/tscSQLParser.c                 | 29 +++++++++++------
 src/common/src/tname.c                        |  5 ++-
 tests/script/general/parser/lastrow_query.sim | 31 +++++++++++++++++++
 3 files changed, 53 insertions(+), 12 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 056ea258bd..517fb32bb3 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -2601,7 +2601,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
       continue;
     }
 
-    if (functionId == TSDB_FUNC_PRJ && pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+    if (functionId == TSDB_FUNC_PRJ && (pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->colInfo.flag))) {
       continue;
     }
 
@@ -5246,7 +5246,8 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
 
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-    if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) {
+    if ((pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) &&
+       !(pExpr->functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) {
       SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex];
       getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64Key, &pExpr->resType,
                         &pExpr->resBytes, &pExpr->interBytes, tagLength, true);
@@ -5354,16 +5355,23 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
   const char* msg1 = "only one selectivity function allowed in presence of tags function";
   const char* msg3 = "aggregation function should not be mixed up with projection";
 
-  bool    tagColExists = false;
+  bool    tagTsColExists = false;
   int16_t numOfSelectivity = 0;
   int16_t numOfAggregation = 0;
 
+  // todo is 0??
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
+  if (!isSTable) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList);
   for (int32_t i = 0; i < numOfExprs; ++i) {
     SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i);
     if (pExpr->functionId == TSDB_FUNC_TAGPRJ ||
         (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) {
-      tagColExists = true;  // selectivity + ts/tag column
+      tagTsColExists = true;  // selectivity + ts/tag column
       break;
     }
   }
@@ -5384,7 +5392,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
     }
   }
 
-  if (tagColExists) {  // check if the selectivity function exists
+  if (tagTsColExists) {  // check if the selectivity function exists
     // When the tag projection function on tag column that is not in the group by clause, aggregation function and
     // selectivity function exist in select clause is not allowed.
     if (numOfAggregation > 0) {
@@ -5407,13 +5415,16 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
        * Otherwise, return with error code.
        */
       for (int32_t i = 0; i < numOfExprs; ++i) {
-        
-        int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
-        if (functionId == TSDB_FUNC_TAGPRJ) {
+        SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+        int16_t functionId = pExpr->functionId;
+        if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) {
           continue;
         }
 
-        if (((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) && (functionId != TSDB_FUNC_LAST_ROW)) {
+        if ((functionId == TSDB_FUNC_LAST_ROW) ||
+             (functionId == TSDB_FUNC_LAST_DST && (pExpr->colInfo.flag & TSDB_COL_NULL) != 0)) {
+          // do nothing
+        } else {
           return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
         }
       }
diff --git a/src/common/src/tname.c b/src/common/src/tname.c
index 49c9e6b726..bb18cb6847 100644
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@@ -62,10 +62,9 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const
   if (name != NULL) {
     tstrncpy(s.name, name, sizeof(s.name));
   } else {
-    size_t len = strdequote(exprStr->z);
-    size_t tlen = MIN(sizeof(s.name), len + 1);
-
+    size_t tlen = MIN(sizeof(s.name), exprStr->n + 1);
     tstrncpy(s.name, exprStr->z, tlen);
+    strdequote(s.name);
   }
 
   return s;
diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim
index 5fc47ed15d..1308ac119f 100644
--- a/tests/script/general/parser/lastrow_query.sim
+++ b/tests/script/general/parser/lastrow_query.sim
@@ -172,3 +172,34 @@ sql select last_row(*) from m1 where tbname in ('t1')
 if $rows != 0 then
   return -1
 endi
+
+sql insert into t1 values('2019-1-1 1:1:1', 1);
+print ===================> last_row query against normal table along with ts/tbname
+sql select last_row(*),ts,'k' from t1;
+if $rows != 1 then
+  return -1
+endi
+
+print ===================> last_row + user-defined column + normal tables
+sql select last_row(ts), 'abc', 1234.9384, ts from t1
+if $rows != 1 then
+  return -1
+endi
+
+if $data02 != 'abc' then
+  return -1
+endi
+
+if $data03 !=  1234.938400000 then
+  return -1
+endi
+
+if $data04 != @2019-01-01 01:01:01.000@ then
+  return -1
+endi
+
+print ===================> last_row + stable + ts/tag column + condition + udf
+sql select last_row(*), ts, 'abc', 123.981, tbname from m1
+if $rows != 1 then
+  return -1
+endi

From 5c055cbfcbb0cbe79f047acfad2478767bf59a89 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 00:04:38 +0800
Subject: [PATCH 44/56] [td-1456] update test

---
 tests/script/general/parser/lastrow_query.sim | 21 ++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim
index 1308ac119f..fd2ca3cb8f 100644
--- a/tests/script/general/parser/lastrow_query.sim
+++ b/tests/script/general/parser/lastrow_query.sim
@@ -186,15 +186,17 @@ if $rows != 1 then
   return -1
 endi
 
-if $data02 != 'abc' then
+if $data01 != @abc@ then
+  print expect abc, actual $data02
   return -1
 endi
 
-if $data03 !=  1234.938400000 then
+if $data02 !=  1234.938400000 then
   return -1
 endi
 
-if $data04 != @2019-01-01 01:01:01.000@ then
+if $data03 != @19-01-01 01:01:01.000@ then
+  print expect 19-01-01 01:01:01.000, actual:$data03
   return -1
 endi
 
@@ -203,3 +205,16 @@ sql select last_row(*), ts, 'abc', 123.981, tbname from m1
 if $rows != 1 then
   return -1
 endi
+
+if $data02 != @19-01-01 01:01:01.000@ then
+    return -1
+endi
+
+if $data03 != @abc@ then
+    return -1
+endi
+
+if $data04 != 123.981000000 then
+    print expect 123.981000000, actual: $data04
+    return -1
+endi
\ No newline at end of file

From 5b208fd47989814ac8556d7b7023f354d3b3f4e1 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 15 Sep 2020 16:16:02 +0000
Subject: [PATCH 45/56] TD-1291

---
 src/plugins/http/src/httpContext.c | 2 --
 src/plugins/http/src/httpServer.c  | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index c0c0c494de..9238869375 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -131,8 +131,6 @@ HttpContext *httpCreateContext(int32_t fd) {
 HttpContext *httpGetContext(void *ptr) {
   uint64_t handleVal = (uint64_t)ptr;
   HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(HttpContext *));
-  ASSERT(ppContext);
-  ASSERT(*ppContext);
 
   if (ppContext) {
     HttpContext *pContext = *ppContext;
diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index f0a7249b51..cdc6d79a75 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) {
 
       if (!pParser->parsed) {
         httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd);
-        continue;
+        return false;
       } else {
         httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos);
         return true;

From 8381148d805302870013ae41c7035acedc8c86c2 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 16 Sep 2020 00:55:38 +0000
Subject: [PATCH 46/56] TD-1364

---
 src/plugins/http/src/httpServer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index cdc6d79a75..f0a7249b51 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) {
 
       if (!pParser->parsed) {
         httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd);
-        return false;
+        continue;
       } else {
         httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos);
         return true;

From 02427c1ef3c974509ab6f5fc6fecc33adfb85050 Mon Sep 17 00:00:00 2001
From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com>
Date: Wed, 16 Sep 2020 09:07:35 +0800
Subject: [PATCH 47/56] Update Connections with other Tools-ch.md

grafanapulgin dir is changed
---
 .../webdocs/markdowndocs/Connections with other Tools-ch.md     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md
index d62a525e28..4e55f3c5e8 100644
--- a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md	
+++ b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md	
@@ -11,7 +11,7 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)
 
 ### 配置Grafana
 
-TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafana目录下。
+TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。
 
 以CentOS 7.2操作系统为例，将tdengine目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。
 

From 1446d990d42147e21b52b26b8c14a2abdf80b89c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 09:31:50 +0800
Subject: [PATCH 48/56] [td-1456]

---
 tests/script/general/parser/lastrow_query.sim | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim
index fd2ca3cb8f..e9d8ce413d 100644
--- a/tests/script/general/parser/lastrow_query.sim
+++ b/tests/script/general/parser/lastrow_query.sim
@@ -154,6 +154,7 @@ if $rows != 46 then
 endi
 
 print ========>td-1317, empty table last_row query crashed
+sql drop table if exists m1;
 sql create table m1(ts timestamp, k int) tags (a int);
 sql create table t1 using m1 tags(1);
 sql create table t2 using m1 tags(2);

From ce458cf037c0dae18a5d552baa8b136adb0d4fda Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Wed, 16 Sep 2020 11:00:48 +0800
Subject: [PATCH 49/56] [TD-1468]

---
 src/kit/shell/src/shellEngine.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 277dc45f8e..1d1ca1c42b 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -765,7 +765,9 @@ void read_history() {
   FILE *f = fopen(f_history, "r");
   if (f == NULL) {
 #ifndef WINDOWS
-    fprintf(stderr, "Failed to open file %s\n", f_history);
+    if (errno != ENOENT) {
+      fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno));
+    }
 #endif    
     return;
   }
@@ -792,7 +794,7 @@ void write_history() {
   FILE *f = fopen(f_history, "w");
   if (f == NULL) {
 #ifndef WINDOWS    
-    fprintf(stderr, "Failed to open file %s for write\n", f_history);
+    fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno));
 #endif    
     return;
   }

From 5e6006f533ae57240c2c6771cb95192dcf553487 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 11:28:17 +0800
Subject: [PATCH 50/56] [td-225] fix compiler errors.

---
 src/client/src/tscSubquery.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 7ea5d4183b..5e912d7f6e 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1947,16 +1947,22 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
 }
 
 int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
-  SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
-  
+  SSqlRes *pRes = &pSql->res;
+
   size_t size = taosArrayGetSize(pCmd->pDataBlocks);
   assert(size > 0);
 
   // the number of already initialized subqueries
   int32_t numOfSub = 0;
 
+  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
+  pState->numOfTotal = pSql->numOfSubs;
+  pState->numOfRemain = pSql->numOfSubs;
+
   pSql->numOfSubs = (uint16_t)size;
+  pRes->code = TSDB_CODE_SUCCESS;
+
   pSql->pSubs = calloc(size, POINTER_BYTES);
   if (pSql->pSubs == NULL) {
     goto _error;
@@ -1964,12 +1970,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
 
   tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size);
 
-  SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
-  pState->numOfTotal = pSql->numOfSubs;
-  pState->numOfRemain = pSql->numOfSubs;
- 
-  pRes->code = TSDB_CODE_SUCCESS;
-
   while(numOfSub < pSql->numOfSubs) {
     SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
     if (pSupporter == NULL) {

From b002f0be5107e27fa6c141ac1aed4f5cf2935119 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 12:54:00 +0800
Subject: [PATCH 51/56] [td-225]

---
 src/client/src/tscSubquery.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 5e912d7f6e..0706e10aa9 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1950,8 +1950,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
-  size_t size = taosArrayGetSize(pCmd->pDataBlocks);
-  assert(size > 0);
+  pSql->numOfSubs = taosArrayGetSize(pCmd->pDataBlocks);
+  assert(pSql->numOfSubs > 0);
+
+  pRes->code = TSDB_CODE_SUCCESS;
 
   // the number of already initialized subqueries
   int32_t numOfSub = 0;
@@ -1960,15 +1962,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   pState->numOfTotal = pSql->numOfSubs;
   pState->numOfRemain = pSql->numOfSubs;
 
-  pSql->numOfSubs = (uint16_t)size;
-  pRes->code = TSDB_CODE_SUCCESS;
-
-  pSql->pSubs = calloc(size, POINTER_BYTES);
+  pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES);
   if (pSql->pSubs == NULL) {
     goto _error;
   }
 
-  tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size);
+  tscDebug("%p submit data to %d vnode(s)", pSql, pSql->numOfSubs);
 
   while(numOfSub < pSql->numOfSubs) {
     SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
@@ -1999,8 +1998,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
       tscDebug("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
       numOfSub++;
     } else {
-      tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%" PRIzu ", code:%s", pSql, numOfSub,
-               size, tstrerror(pRes->code));
+      tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
+               pSql->numOfSubs, tstrerror(pRes->code));
       goto _error;
     }
   }

From d545eb9bb412901138b0573399fb9fc5815311e1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 15:01:10 +0800
Subject: [PATCH 52/56] [td-225] fix compiler error in win platform

---
 src/util/src/tsocket.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 479eeaa754..4cf73e6dff 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -56,13 +56,16 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
     freeaddrinfo(result);
     return ip;
   } else {
+#ifdef EAI_SYSTEM
     if (ret == EAI_SYSTEM) {
       uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
     } else {
       uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
     }
-
+#else
+    uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+#endif
     return 0xFFFFFFFF;
   }
 }

From 848a37aafe759b40d5ef1bd853ded2b09ac59c38 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 15:05:57 +0800
Subject: [PATCH 53/56] [td-225] fix compiler error in win platform

---
 src/query/src/qPercentile.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index ccc451108e..8d74dbbdc2 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -79,13 +79,13 @@ static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, d
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_SMALLINT:
     case TSDB_DATA_TYPE_INT:
-      range->iMinVal = minval;
-      range->iMaxVal = maxval;
+      range->iMinVal = (int32_t) minval;
+      range->iMaxVal = (int32_t) maxval;
       break;
 
     case TSDB_DATA_TYPE_BIGINT:
-      range->i64MinVal = minval;
-      range->i64MaxVal = maxval;
+      range->i64MinVal = (int64_t) minval;
+      range->i64MaxVal = (int64_t) maxval;
       break;
     case TSDB_DATA_TYPE_FLOAT:
     case TSDB_DATA_TYPE_DOUBLE:

From d7f409fd9ac27c8b7a2b2f952f608ab73936be0c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 15:09:43 +0800
Subject: [PATCH 54/56] [td-225] fix compiler error in win platform

---
 src/query/src/qPercentile.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index 8d74dbbdc2..3a8be781d5 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -740,8 +740,10 @@ double getPercentile(tMemBucket *pMemBucket, double percent) {
       case TSDB_DATA_TYPE_SMALLINT:
       case TSDB_DATA_TYPE_INT:
         return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal;
-      case TSDB_DATA_TYPE_BIGINT:
-        return fabs(percent - 100) < DBL_EPSILON? pRange->i64MaxVal:pRange->i64MinVal;
+      case TSDB_DATA_TYPE_BIGINT: {
+        double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal);
+        return v;
+      }
       case TSDB_DATA_TYPE_FLOAT:
       case TSDB_DATA_TYPE_DOUBLE:
         return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;

From b9db2aafb66a52d1e9f70b94878a568f4c1ffe34 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Wed, 16 Sep 2020 15:35:38 +0800
Subject: [PATCH 55/56] Update Taos error code

---
 documentation20/webdocs/markdowndocs/Taos Error Code-ch.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md
index 6ba0decfd3..a9a3dee432 100644
--- a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md	
+++ b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md	
@@ -1,7 +1,7 @@
 # TDengine 2.0 错误码以及对应的十进制码
 
 
-| Code                  |  bit  |   error code |  错误描述                  | 十进制错误码       |
+| 状态码                 |  模  |   错误码（十六进制） |  错误描述                  | 错误码（十进制）   |
 |-----------------------| :---: | :---------:  | :------------------------ | ---------------- |
 |TSDB_CODE_RPC_ACTION_IN_PROGRESS|	0	| 0x0001|	 "Action in progress"|	-2147483647|
 |TSDB_CODE_RPC_AUTH_REQUIRED|	0	| 0x0002	| "Authentication required"|	-2147483646|
@@ -87,7 +87,7 @@
 |TSDB_CODE_MND_INVALID_ACCT_OPTION|	0	| 0x0342	| "Invalid account options"|	-2147482814|
 |TSDB_CODE_MND_USER_ALREADY_EXIST|	0	| 0x0350	| "User already exists"|	-2147482800|
 |TSDB_CODE_MND_INVALID_USER	|0	| 0x0351	| "Invalid user"	|-2147482799|
-|TSDB_CODE_MND_INVALID_USER_FORMAT|	|0	 |0x0352	 |"Invalid user format"	|-2147482798|
+|TSDB_CODE_MND_INVALID_USER_FORMAT| 0	 |0x0352	 |"Invalid user format"	|-2147482798|
 |TSDB_CODE_MND_INVALID_PASS_FORMAT|	0|	 0x0353	| "Invalid password format"|	-2147482797|
 |TSDB_CODE_MND_NO_USER_FROM_CONN|	0	| 0x0354	| "Can not get user from conn"|	-2147482796|
 |TSDB_CODE_MND_TOO_MANY_USERS|	0	| 0x0355|	 "Too many users"|	-2147482795|
@@ -107,7 +107,7 @@
 |TSDB_CODE_MND_DB_NOT_SELECTED|	0	| 0x0380	| "Database not specified or available"|	-2147482752|
 |TSDB_CODE_MND_DB_ALREADY_EXIST|	0	| 0x0381	| "Database already exists"|	-2147482751|
 |TSDB_CODE_MND_INVALID_DB_OPTION|	0	| 0x0382	| "Invalid database options"|	-2147482750|
-|TSDB_CODE_MND_INVALID_DB|	|0	| 0x0383	| "Invalid database name"|	-2147482749|
+|TSDB_CODE_MND_INVALID_DB|	0	| 0x0383	| "Invalid database name"|	-2147482749|
 |TSDB_CODE_MND_MONITOR_DB_FORBIDDEN|	0	| 0x0384	| "Cannot delete monitor database"|	-2147482748|
 |TSDB_CODE_MND_TOO_MANY_DATABASES|	0|	 0x0385	| "Too many databases for account"|	-2147482747|
 |TSDB_CODE_MND_DB_IN_DROPPING|	0	| 0x0386|	 "Database not available"	|-2147482746|

From 6ea7e4ee84f8ab02f2b6a50b66fc1b525fc6aebc Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 16 Sep 2020 18:36:58 +0800
Subject: [PATCH 56/56] [td-1478]

---
 src/client/src/tscSubquery.c | 2 +-
 src/query/src/qExecutor.c    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 0706e10aa9..4246abf52d 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -92,7 +92,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
     STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
 
 #ifdef _DEBUG_VIEW
-    tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag);
+    tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
 #endif
 
     int32_t res = tVariantCompare(&elem1.tag, &elem2.tag);
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index b5adeafd23..b873714c49 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -2836,6 +2836,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   // all results have been return to client, try next group
   if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) {
     pGroupResInfo->numOfDataPages = 0;
+    pGroupResInfo->pos.pageId = 0;
     pGroupResInfo->pos.rowId = 0;
 
     // current results of group has been sent to client, try next group