From b42a8214da864b29ba39ea24984595b3ffcb3fb5 Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Fri, 8 May 2020 16:52:54 +0800
Subject: [PATCH 01/33] TD-191: WIP

---
 src/client/src/tscServer.c    |   3 +-
 src/client/src/tscSub.c       | 167 +++++++++++++---------------------
 src/client/src/tscUtil.c      |   5 +-
 src/query/inc/queryExecutor.h |   1 +
 src/query/src/queryExecutor.c | 116 +++++++++++++----------
 5 files changed, 135 insertions(+), 157 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index f30a494ee0..d1050d6fa6 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -624,7 +624,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
       STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
       pTableIdInfo->tid = htonl(pItem->tid);
       pTableIdInfo->uid = htobe64(pItem->uid);
-//      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
+      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
       pMsg += sizeof(STableIdInfo);
     }
   }
@@ -2354,6 +2354,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
     for (int i = 0; i < numOfTables; i++) {
       int64_t uid = htobe64(*(int64_t*)p);
       p += sizeof(int64_t);
+      p += sizeof(int32_t); // skip tid
       TSKEY key = htobe64(*(TSKEY*)p);
       p += sizeof(TSKEY);
       tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key);
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 856b678391..009372f3f7 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -44,8 +44,7 @@ typedef struct SSub {
   int                     interval;
   TAOS_SUBSCRIBE_CALLBACK fp;
   void *                  param;
-  int                     numOfTables;
-  SSubscriptionProgress * progress;
+  SArray* progress;
 } SSub;
 
 
@@ -58,41 +57,36 @@ static int tscCompareSubscriptionProgress(const void* a, const void* b) {
 }
 
 TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid) {
-  if (sub == NULL)
+  if (sub == NULL) {
     return 0;
-
+  }
   SSub* pSub = (SSub*)sub;
-  for (int s = 0, e = pSub->numOfTables; s < e;) {
-    int m = (s + e) / 2;
-    SSubscriptionProgress* p = pSub->progress + m;
-    if (p->uid > uid)
-      e = m;
-    else if (p->uid < uid)
-      s = m + 1;
-    else
-      return p->key;
+  if (pSub->progress == NULL) {
+    return 0;
   }
 
-  return 0;
+  SSubscriptionProgress target = {.uid = uid, .key = 0};
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  if (p == NULL) {
+    return 0;
+  }
+  return p->key;
 }
 
 void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   if( sub == NULL)
     return;
-
   SSub* pSub = (SSub*)sub;
-  for (int s = 0, e = pSub->numOfTables; s < e;) {
-    int m = (s + e) / 2;
-    SSubscriptionProgress* p = pSub->progress + m;
-    if (p->uid > uid)
-      e = m;
-    else if (p->uid < uid)
-      s = m + 1;
-    else {
-      if (ts >= p->key) p->key = ts;
-      break;
-    }
+
+  SSubscriptionProgress target = {.uid = uid, .key = ts};
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+  if (p != NULL) {
+    p->key = ts;
+    return;
   }
+
+  taosArrayPush(pSub->progress, &target);
+  taosArraySort(pSub->progress, tscCompareSubscriptionProgress);
 }
 
 
@@ -158,60 +152,40 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
   taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer);
 }
 
+static void waitParseComplete(void *param, TAOS_RES *tres, int code) {
+  assert(param != NULL);
+  SSqlObj *pSql = ((SSqlObj *)param);
+  
+  pSql->res.code = code;
+  sem_post(&pSql->rspSem);
+}
 
 int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
-  int code = (uint8_t)tsParseSql(pSub->pSql, false);
+  SSqlObj* pSql = pSub->pSql;
+
+  SSqlCmd* pCmd = &pSql->cmd;
+  tscAllocPayload( pCmd, TSDB_DEFAULT_PAYLOAD_SIZE );
+
+  pSql->fp = waitParseComplete;
+  pSql->param = pSql;
+  int code = tsParseSql(pSql, false);
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+    // wait for the callback function to post the semaphore
+    sem_wait(&pSql->rspSem);
+    code = pSql->res.code;
+  }
   if (code != TSDB_CODE_SUCCESS) {
-    tscError("failed to parse sql statement: %s", pSub->topic);
+    tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code));
     return 0;
   }
 
-  SSqlCmd* pCmd = &pSub->pSql->cmd;
   if (pCmd->command != TSDB_SQL_SELECT) {
     tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
     return 0;
   }
 
-  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0);
-  int numOfTables = 0;
-  if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
-//    SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
-//    for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
-//      SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
-//      numOfTables += pVnodeSidList->numOfSids;
-//    }
-  }
-
-  SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress));
-  if (progress == NULL) {
-    tscError("failed to allocate memory for progress: %s", pSub->topic);
-    return 0;
-  }
-
-  if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
-    numOfTables = 1;
-    int64_t uid = pTableMetaInfo->pTableMeta->uid;
-    progress[0].uid = uid;
-    progress[0].key = tscGetSubscriptionProgress(pSub, uid);
-  } else {
-//    SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta;
-//    numOfTables = 0;
-//    for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) {
-//      SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i);
-//      for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) {
-//        STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j);
-//        int64_t uid = pTableMetaInfo->uid;
-//        progress[numOfTables].uid = uid;
-//        progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid);
-//      }
-//    }
-    qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress);
-  }
-
-  free(pSub->progress);
-  pSub->numOfTables = numOfTables;
-  pSub->progress = progress;
-
+  taosArrayDestroy(pSub->progress);
+  pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
   pSub->lastSyncTime = taosGetTimestampMs();
 
   return 1;
@@ -255,23 +229,22 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
   }
 
   int numOfTables = atoi(buf);
-  SSubscriptionProgress* progress = calloc(numOfTables, sizeof(SSubscriptionProgress));
+  SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
   for (int i = 0; i < numOfTables; i++) {
     if (fgets(buf, sizeof(buf), fp) == NULL) {
       fclose(fp);
       free(progress);
       return 0;
     }
-    int64_t uid, key;
-    sscanf(buf, "%" SCNd64 ":%" SCNd64, &uid, &key);
-    progress[i].uid = uid;
-    progress[i].key = key;
+    SSubscriptionProgress p;
+    sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key);
+    taosArrayPush(progress, &p);
   }
 
   fclose(fp);
 
-  qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress);
-  pSub->numOfTables = numOfTables;
+  taosArraySort(progress, tscCompareSubscriptionProgress);
+  taosArrayDestroy(pSub->progress);
   pSub->progress = progress;
   tscTrace("subscription progress loaded, %d tables: %s", numOfTables, pSub->topic);
   return 1;
@@ -294,11 +267,9 @@ void tscSaveSubscriptionProgress(void* sub) {
   }
 
   fputs(pSub->pSql->sqlstr, fp);
-  fprintf(fp, "\n%d\n", pSub->numOfTables);
-  for (int i = 0; i < pSub->numOfTables; i++) {
-    int64_t uid = pSub->progress[i].uid;
-    TSKEY key = pSub->progress[i].key;
-    fprintf(fp, "%" PRId64 ":%" PRId64 "\n", uid, key);
+  for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) {
+    SSubscriptionProgress* p = taosArrayGet(pSub->progress, i);
+    fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key);
   }
 
   fclose(fp);
@@ -362,36 +333,20 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   for (int retry = 0; retry < 3; retry++) {
     tscRemoveFromSqlList(pSql);
 
-    if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) {
-      tscTrace("begin meter synchronization");
-      char* sqlstr = pSql->sqlstr;
-      pSql->sqlstr = NULL;
-      taos_free_result_imp(pSql, 0);
-      pSql->sqlstr = sqlstr;
-      taosCacheEmpty(tscCacheHandle);
-      if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
-      tscTrace("meter synchronization completed");
-    } else {
-      SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-      
-      uint32_t type = pQueryInfo->type;
-      taos_free_result_imp(pSql, 1);
-      pRes->numOfRows = 1;
-      pRes->numOfTotal = 0;
-      pRes->qhandle = 0;
-      pSql->cmd.command = TSDB_SQL_SELECT;
-      pQueryInfo->type = type;
-
-      tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
+    pSql->fp = waitParseComplete;
+    pSql->param = pSql;
+    tscDoQuery(pSql);
+    if (pRes->code == TSDB_CODE_ACTION_IN_PROGRESS) {
+      sem_wait(&pSql->rspSem);
     }
 
-    tscDoQuery(pSql);
-    if (pRes->code != TSDB_CODE_NOT_ACTIVE_TABLE) {
-      break;
+    if (pRes->code != TSDB_CODE_SUCCESS) {
+      continue;
     }
     // meter was removed, make sync time zero, so that next retry will
     // do synchronization first
     pSub->lastSyncTime = 0;
+    break;
   }
 
   if (pRes->code != TSDB_CODE_SUCCESS) {
@@ -421,7 +376,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   tscFreeSqlObj(pSub->pSql);
-  free(pSub->progress);
+  taosArrayDestroy(pSub->progress);
   memset(pSub, 0, sizeof(*pSub));
   free(pSub);
 }
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index c25e91a669..1a01904839 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1459,10 +1459,11 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) {
 }
 
 /*
- * the following three kinds of SqlObj should not be freed
+ * the following four kinds of SqlObj should not be freed
  * 1. SqlObj for stream computing
  * 2. main SqlObj
  * 3. heartbeat SqlObj
+ * 4. SqlObj for subscription
  *
  * If res code is error and SqlObj does not belong to above types, it should be
  * automatically freed for async query, ignoring that connection should be kept.
@@ -1475,7 +1476,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
   }
 
   STscObj* pTscObj = pSql->pTscObj;
-  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql) {
+  if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql || pSql->pSubscription != NULL) {
     return false;
   }
 
diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h
index 2088e5a49d..991b3b73f7 100644
--- a/src/query/inc/queryExecutor.h
+++ b/src/query/inc/queryExecutor.h
@@ -182,6 +182,7 @@ typedef struct SQInfo {
   SQueryRuntimeEnv runtimeEnv;
   int32_t          groupIndex;
   int32_t          offset;            // offset in group result set of subgroup, todo refactor
+  SArray*          arrTableIdInfo;
   
   T_REF_DECLARE()
   /*
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index fb19d0f50c..21511a1aa3 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -113,7 +113,6 @@ static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols);
 static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static bool hasMainOutput(SQuery *pQuery);
-static void createTableQueryInfo(SQInfo *pQInfo);
 static void buildTagQueryResult(SQInfo *pQInfo);
 
 static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTableId, STableQueryInfo *pTableQueryInfo);
@@ -3460,7 +3459,11 @@ static bool hasMainOutput(SQuery *pQuery) {
   return false;
 }
 
-STableQueryInfo *createTableQueryInfoImpl(SQueryRuntimeEnv *pRuntimeEnv, STableId tableId, STimeWindow win) {
+static STableQueryInfo *createTableQueryInfo(
+  SQueryRuntimeEnv *pRuntimeEnv,
+  STableId tableId,
+  STimeWindow win
+) {
   STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo));
 
   pTableQueryInfo->win = win;
@@ -3870,7 +3873,18 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
     data += bytes * numOfRows;
   }
 
-  
+  int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo);
+  *(int32_t*)data = htonl(numOfTables);
+  data += sizeof(int32_t);
+  for(int32_t i = 0; i < numOfTables; i++) {
+    STableIdInfo* pSrc = taosArrayGet(pQInfo->arrTableIdInfo, i);
+    STableIdInfo* pDst = (STableIdInfo*)data;
+    pDst->uid = htobe64(pSrc->uid);
+    pDst->tid = htonl(pSrc->tid);
+    pDst->key = htobe64(pSrc->key);
+    data += sizeof(STableIdInfo);
+  }
+
   // all data returned, set query over
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
     if (pQInfo->runtimeEnv.stableQuery && isIntervalQuery(pQuery)) {
@@ -4594,6 +4608,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
          * to ensure that, we can reset the query range once query on a meter is completed.
          */
         pQInfo->tableIndex++;
+        pInfo->lastKey = pQuery->lastKey;
+
+        STableIdInfo tidInfo;
+        tidInfo.uid = item->id.uid;
+        tidInfo.tid = item->id.tid;
+        tidInfo.key = pInfo->lastKey;
+        taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
+
         // if the buffer is full or group by each table, we need to jump out of the loop
         if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*||
             isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) {
@@ -4663,33 +4685,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       pQuery->limit.offset);
 }
 
-static void createTableQueryInfo(SQInfo *pQInfo) {
-  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) {
+  //  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
-  // todo make sure the table are added the reference count to gauranteed that all involved tables are valid
-  size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList);
-
-  int32_t index = 0;
-  for (int32_t i = 0; i < numOfGroups; ++i) {  // load all meter meta info
-    SArray *group = *(SArray **)taosArrayGet(pQInfo->groupInfo.pGroupList, i);
-
-    size_t s = taosArrayGetSize(group);
-    for (int32_t j = 0; j < s; ++j) {
-      SGroupItem* item = (SGroupItem *)taosArrayGet(group, j);
-
-      // STableQueryInfo has been created for each table
-      if (item->info != NULL) {
-        return;
-      }
-
-      STableQueryInfo* pInfo = createTableQueryInfoImpl(&pQInfo->runtimeEnv, item->id, pQuery->window);
-      pInfo->groupIdx = i;
-      pInfo->tableIndex = index;
-      
-      item->info = pInfo;
-      index += 1;
-    }
-  }
+  //  for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
+  //    STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo;
+  //    changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo);
+  //  }
 }
 
 static void doSaveContext(SQInfo *pQInfo) {
@@ -5196,20 +5198,10 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx
 static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) {
   assert(pQueryMsg->numOfTables > 0);
 
-  *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId));
+  *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo));
 
-  STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
-  pTableIdInfo->tid = htonl(pTableIdInfo->tid);
-  pTableIdInfo->uid = htobe64(pTableIdInfo->uid);
-  pTableIdInfo->key = htobe64(pTableIdInfo->key);
-
-  STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->tid};
-  taosArrayPush(*pTableIdList, &id);
-
-  pMsg += sizeof(STableIdInfo);
-
-  for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) {
-    pTableIdInfo = (STableIdInfo *)pMsg;
+  for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) {
+    STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg;
 
     pTableIdInfo->tid = htonl(pTableIdInfo->tid);
     pTableIdInfo->uid = htobe64(pTableIdInfo->uid);
@@ -5660,7 +5652,16 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
   }
 }
 
-static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
+
+static int compareTableIdInfo( const void* a, const void* b ) {
+  const STableIdInfo* x = (const STableIdInfo*)a;
+  const STableIdInfo* y = (const STableIdInfo*)b;
+  if (x->uid > y->uid) return 1;
+  if (x->uid < y->uid) return -1;
+  return 0;
+}
+
+static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
                                STableGroupInfo *groupInfo, SColumnInfo* pTagCols) {
   SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
   if (pQInfo == NULL) {
@@ -5753,6 +5754,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
   pQInfo->groupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
   pQInfo->groupInfo.numOfTables = groupInfo->numOfTables;
   
+  int tableIndex = 0;
+  STimeWindow window = pQueryMsg->window;
+  taosArraySort( pTableIdList, compareTableIdInfo );
   for(int32_t i = 0; i < numOfGroups; ++i) {
     SArray* pa = taosArrayGetP(groupInfo->pGroupList, i);
     size_t s = taosArrayGetSize(pa);
@@ -5760,13 +5764,26 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
     SArray* p1 = taosArrayInit(s, sizeof(SGroupItem));
     
     for(int32_t j = 0; j < s; ++j) {
-      SGroupItem item = { .id = *(STableId*) taosArrayGet(pa, j), .info = NULL, };
+      STableId id = *(STableId*) taosArrayGet(pa, j);
+      SGroupItem item = { .id = id };
+      // NOTE: compare STableIdInfo with STableId
+      // not a problem at present because we only use their 1st int64_t field
+      STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
+      if (pTableId != NULL ) {
+        window.skey = pTableId->key;
+      } else {
+        window.skey = 0;
+      }
+      item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window);
+      item.info->groupIdx = i;
+      item.info->tableIndex = tableIndex++;
       taosArrayPush(p1, &item);
     }
-    
     taosArrayPush(pQInfo->groupInfo.pGroupList, &p1);
   }
 
+  pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
+
   pQuery->pos = -1;
 
   pQuery->window = pQueryMsg->window;
@@ -5918,6 +5935,7 @@ static void freeQInfo(SQInfo *pQInfo) {
   }
   
   taosArrayDestroy(pQInfo->tableIdGroupInfo.pGroupList);
+  taosArrayDestroy(pQInfo->arrTableIdInfo);
   
   if (pQuery->pGroupbyExpr != NULL) {
     taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo);
@@ -6045,13 +6063,13 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
   if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) {
     isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
     
-    STableId *id = taosArrayGet(pTableIdList, 0);
+    STableIdInfo *id = taosArrayGet(pTableIdList, 0);
     if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }
   } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) {
     isSTableQuery = true;
-    STableId *id = taosArrayGet(pTableIdList, 0);
+    STableIdInfo *id = taosArrayGet(pTableIdList, 0);
     
     // group by normal column, do not pass the group by condition to tsdb to group table into different group
     int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
@@ -6070,7 +6088,7 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     assert(0);
   }
 
-  (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo);
+  (*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo);
   if ((*pQInfo) == NULL) {
     code = TSDB_CODE_SERV_OUT_OF_MEMORY;
     goto _over;
@@ -6168,6 +6186,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
+  size += sizeof(int32_t);
+  size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo);
   *contLen = size + sizeof(SRetrieveTableRsp);
 
   // todo handle failed to allocate memory

From 3e6471eb8575e67b53e8be1ae0fb6231a3fd2956 Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Sat, 9 May 2020 11:51:40 +0800
Subject: [PATCH 02/33] enhance exception handling

---
 src/client/inc/tscUtil.h |  4 ++++
 src/client/src/tscUtil.c | 23 +++++++++++++++++++++++
 src/util/inc/exception.h |  9 ++++++---
 src/util/src/exception.c | 26 +++++++++++++++++++++-----
 4 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index 71c24501d1..dd32854663 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -265,6 +265,10 @@ void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)());
 
+void* malloc_throw(size_t size);
+void* calloc_throw(size_t nmemb, size_t size);
+char* strdup_throw(const char* str);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 1a01904839..864239f8fb 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -2156,3 +2156,26 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column
   }
 }
 
+void* malloc_throw(size_t size) {
+  void* p = malloc(size);
+  if (p == NULL) {
+    THROW(TSDB_CODE_CLI_OUT_OF_MEMORY);
+  }
+  return p;
+}
+
+void* calloc_throw(size_t nmemb, size_t size) {
+  void* p = malloc(size);
+  if (p == NULL) {
+    THROW(TSDB_CODE_CLI_OUT_OF_MEMORY);
+  }
+  return p;
+}
+
+char* strdup_throw(const char* str) {
+  char* p = strdup(str);
+  if (p == NULL) {
+    THROW(TSDB_CODE_CLI_OUT_OF_MEMORY);
+  }
+  return p;
+}
diff --git a/src/util/inc/exception.h b/src/util/inc/exception.h
index 41f01d68dd..52cd03d830 100644
--- a/src/util/inc/exception.h
+++ b/src/util/inc/exception.h
@@ -73,6 +73,7 @@ void cleanupPush_void_ptr_bool  ( bool failOnly, void* func, void* arg1, bool ar
 void cleanupPush_void_ptr       ( bool failOnly, void* func, void* arg );
 void cleanupPush_int_int        ( bool failOnly, void* func, int arg );
 void cleanupPush_void           ( bool failOnly, void* func );
+void cleanupPush_int_ptr        ( bool failOnly, void* func, void* arg );
 
 int32_t cleanupGetActionCount();
 void cleanupExecuteTo( int32_t anchor, bool failed );
@@ -83,8 +84,10 @@ void cleanupExecute( SExceptionNode* node, bool failed );
 #define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg )             cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) )
 #define CLEANUP_PUSH_INT_INT( failOnly, func, arg )              cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) )
 #define CLEANUP_PUSH_VOID( failOnly, func )                      cleanupPush_void( (failOnly), (void*)(func) )
+#define CLEANUP_PUSH_INT_PTR( failOnly, func, arg )              cleanupPush_int_ptr( (failOnly), (void*)(func), (void*)(arg) )
 #define CLEANUP_PUSH_FREE( failOnly, arg )                       cleanupPush_void_ptr( (failOnly), free, (void*)(arg) )
 #define CLEANUP_PUSH_CLOSE( failOnly, arg )                      cleanupPush_int_int( (failOnly), close, (int)(arg) )
+#define CLEANUP_PUSH_FCLOSE( failOnly, arg )                     cleanupPush_int_ptr( (failOnly), fclose, (void*)(arg) )
 
 #define CLEANUP_GET_ANCHOR()          cleanupGetActionCount()
 #define CLEANUP_EXECUTE_TO( anchor, failed )  cleanupExecuteTo( (anchor), (failed) )
@@ -95,7 +98,7 @@ void cleanupExecute( SExceptionNode* node, bool failed );
 
 void exceptionPushNode( SExceptionNode* node );
 int32_t exceptionPopNode();
-void exceptionThrow( int code );
+void exceptionThrow( int32_t code );
 
 #define TRY(maxCleanupActions) do { \
     SExceptionNode exceptionNode = { 0 }; \
@@ -106,10 +109,10 @@ void exceptionThrow( int code );
     int caughtException = setjmp( exceptionNode.jb ); \
     if( caughtException == 0 )
 
-#define CATCH( code ) int code = exceptionPopNode(); \
+#define CATCH( code ) int32_t code = exceptionPopNode(); \
     if( caughtException == 1 )
 
-#define FINALLY( code ) int code = exceptionPopNode();
+#define FINALLY( code ) int32_t code = exceptionPopNode();
 
 #define END_TRY } while( 0 );
 
diff --git a/src/util/src/exception.c b/src/util/src/exception.c
index 7f8f91c784..3d2949c093 100644
--- a/src/util/src/exception.c
+++ b/src/util/src/exception.c
@@ -14,7 +14,7 @@ int32_t exceptionPopNode() {
     return node->code;
 }
 
-void exceptionThrow( int code ) {
+void exceptionThrow( int32_t code ) {
     expList->code = code;
     longjmp( expList->jb, 1 );
 }
@@ -38,21 +38,27 @@ static void cleanupWrapper_void_ptr( SCleanupAction* ca ) {
 
 static void cleanupWrapper_int_int( SCleanupAction* ca ) {
     int (*func)( int ) = ca->func;
-    func( (int)(intptr_t)(ca->arg1.Int) );
+    func( ca->arg1.Int );
 }
 
-static void cleanupWrapper_void_void( SCleanupAction* ca ) {
+static void cleanupWrapper_void( SCleanupAction* ca ) {
     void (*func)() = ca->func;
     func();
 }
 
+static void cleanupWrapper_int_ptr( SCleanupAction* ca ) {
+    int (*func)( void* ) = ca->func;
+    func( ca->arg1.Ptr );
+}
+
 typedef void (*wrapper)(SCleanupAction*);
 static wrapper wrappers[] = {
     cleanupWrapper_void_ptr_ptr,
     cleanupWrapper_void_ptr_bool,
     cleanupWrapper_void_ptr,
     cleanupWrapper_int_int,
-    cleanupWrapper_void_void,
+    cleanupWrapper_void,
+    cleanupWrapper_int_ptr,
 };
 
 
@@ -107,6 +113,15 @@ void cleanupPush_void( bool failOnly, void* func ) {
     ca->func = func;
 }
 
+void cleanupPush_int_ptr( bool failOnly, void* func, void* arg ) {
+    assert( expList->numCleanupAction < expList->maxCleanupAction );
+
+    SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
+    ca->wrapper = 5;
+    ca->failOnly = failOnly;
+    ca->func = func;
+    ca->arg1.Ptr = arg;
+}
 
 
 int32_t cleanupGetActionCount() {
@@ -118,8 +133,9 @@ static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed
     while( node->numCleanupAction > anchor ) {
         --node->numCleanupAction;
         SCleanupAction *ca = node->cleanupActions + node->numCleanupAction;
-        if( failed || !(ca->failOnly) )
+        if( failed || !(ca->failOnly) ) {
             wrappers[ca->wrapper]( ca );
+        }
     }
 }
 

From 891476044db1a05592e888eea082a21561a1bf6f Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Sat, 9 May 2020 11:54:51 +0800
Subject: [PATCH 03/33] add taosArrayClear

---
 src/util/inc/tarray.h | 6 ++++++
 src/util/src/tarray.c | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h
index 7edd032034..866bde0938 100644
--- a/src/util/inc/tarray.h
+++ b/src/util/inc/tarray.h
@@ -106,6 +106,12 @@ void taosArrayCopy(SArray* pDst, const SArray* pSrc);
  */
 SArray* taosArrayClone(const SArray* pSrc);
 
+/**
+ * clear the array (remove all element)
+ * @param pArray
+ */
+void taosArrayClear(SArray* pArray);
+
 /**
  * destroy array list
  * @param pArray
diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c
index 5ef4417710..5198597ff7 100755
--- a/src/util/src/tarray.c
+++ b/src/util/src/tarray.c
@@ -176,6 +176,11 @@ SArray* taosArrayClone(const SArray* pSrc) {
   return dst;
 }
 
+void taosArrayClear(SArray* pArray) {
+  assert( pArray != NULL );
+  pArray->size = 0;
+}
+
 void taosArrayDestroy(SArray* pArray) {
   if (pArray == NULL) {
     return;

From 50f432c6a3a8ad02734609b58f0a2d3e542589b3 Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Sat, 9 May 2020 12:49:46 +0800
Subject: [PATCH 04/33] fix typo

---
 src/client/inc/tscUtil.h        |  6 +++---
 src/client/src/tscAsync.c       |  2 +-
 src/client/src/tscLocal.c       |  4 ++--
 src/client/src/tscParseInsert.c |  4 ++--
 src/client/src/tscSQLParser.c   | 36 ++++++++++++++++-----------------
 src/client/src/tscServer.c      | 10 ++++-----
 src/client/src/tscStream.c      |  2 +-
 src/client/src/tscSubquery.c    |  4 ++--
 src/client/src/tscUtil.c        | 12 +++++------
 src/query/src/queryExecutor.c   |  9 ---------
 10 files changed, 40 insertions(+), 49 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index dd32854663..d110295369 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -31,13 +31,13 @@ extern "C" {
 #include "tscSecondaryMerge.h"
 #include "tsclient.h"
 
-#define UTIL_TABLE_IS_SUPERTABLE(metaInfo)  \
+#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
 #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
   
-#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo)\
-  (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
+#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\
+  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
 
 #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0)
 
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 9f207936df..4ceeb5b316 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -471,7 +471,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
 
     if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
 
-    if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex);
       pRes->code = code;
 
diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index 5ae226a577..2da786d1d8 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -122,7 +122,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
   int32_t numOfRows = tscGetNumOfColumns(pMeta);
   int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta);
 
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     numOfRows = numOfRows + tscGetNumOfTags(pMeta);
   }
 
@@ -161,7 +161,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     }
   }
 
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return 0;
   }
 
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 65555dba13..7e67ff82e9 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -796,7 +796,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return code;
     }
 
-    if (!UTIL_TABLE_IS_SUPERTABLE(pSTableMeterMetaInfo)) {
+    if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMeterMetaInfo)) {
       return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z);
     }
 
@@ -1097,7 +1097,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
       goto _error_clean;       // TODO: should _clean or _error_clean to async flow ????
     }
 
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL);
       goto _error_clean;
     }
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index a81ad19e4f..f82710669b 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1352,7 +1352,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
 
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     numOfTotalColumns = tinfo.numOfColumns + tinfo.numOfTags;
   } else {
     numOfTotalColumns = tinfo.numOfColumns;
@@ -1412,7 +1412,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
       STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
 
-      if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+      if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg1);
       }
 
@@ -1866,7 +1866,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
     
     case TK_TBID: {
       pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-      if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+      if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg7);
       }
     
@@ -2283,7 +2283,7 @@ bool validateIpAddress(const char* ip, size_t size) {
 int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return TSDB_CODE_INVALID_SQL;
   }
 
@@ -2322,7 +2322,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
 /* transfer the field-info back to original input format */
 void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return;
   }
   
@@ -2546,7 +2546,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
     }
     
     if (groupTag) {
-      if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+      if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
         return invalidSqlErrMsg(pQueryInfo->msg, msg9);
       }
 
@@ -3258,7 +3258,7 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum
   }
 
   // table to table/ super table to super table are allowed
-  if (UTIL_TABLE_IS_SUPERTABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPERTABLE(pRightMeterMeta)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPER_TABLE(pRightMeterMeta)) {
     invalidSqlErrMsg(pQueryInfo->msg, msg5);
     return false;
   }
@@ -3341,7 +3341,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
   } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) ||
              index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {  // query on tags
     // check for tag query condition
-    if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
     }
 
@@ -3701,7 +3701,7 @@ static int32_t validateJoinExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) {
   }
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {  // for stable join, tag columns
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {  // for stable join, tag columns
                                                    // must be present for join
     if (pCondExpr->pJoinExpr == NULL) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
@@ -3739,7 +3739,7 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) {
 
 static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     SColumnIndex index = {0};
 
     getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index);
@@ -4104,7 +4104,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
   }
 
   /* for super table query, set default ascending order for group output */
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
   }
 }
@@ -4130,7 +4130,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema
    *
    * for super table query, the order option must be less than 3.
    */
-  if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     if (pSortorder->nExpr > 1) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg0);
     }
@@ -4151,7 +4151,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema
   SSQLToken    columnName = {pVar->nLen, pVar->nType, pVar->pz};
   SColumnIndex index = {0};
 
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {  // super table query
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {  // super table query
     if (getColumnIndexByName(&columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
     }
@@ -4304,10 +4304,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
   if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN ||
       pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
-    if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg3);
     }
-  } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) {
+  } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) {
     return invalidSqlErrMsg(pQueryInfo->msg, msg4);
   } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) &&
              UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) {
@@ -4693,7 +4693,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
   }
 
   // todo refactor
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     if (!tscQueryTags(pQueryInfo)) {  // local handle the super table tag query
       if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
         if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) {
@@ -5629,7 +5629,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
     return code;
   }
 
-  bool isSTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
+  bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
   if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_INVALID_SQL;
   }
@@ -5773,7 +5773,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
   assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr);
   bool isSTable = false;
   
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     isSTable = true;
     code = tscGetSTableVgroupInfo(pSql, index);
     if (code != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index d1050d6fa6..9431a09bb0 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -500,7 +500,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   // todo valid the vgroupId at the client side
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     int32_t vgIndex = pTableMetaInfo->vgroupIndex;
     
     SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
@@ -565,7 +565,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   //STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
   // table query without tags values
-  //if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  //if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
   //}
   
@@ -577,10 +577,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
 
   STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
-  if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
+  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
     
     SCMVgroupInfo* pVgroupInfo = NULL;
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       int32_t index = pTableMetaInfo->vgroupIndex;
       assert(index >= 0);
   
@@ -2292,7 +2292,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
   taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
 
   if (pTableMetaInfo->pTableMeta) {
-    bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
+    bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
 
     taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
 //    taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index d7c22b2248..a00f856b2a 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -79,7 +79,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
 
   if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
 
-  if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     code = tscGetSTableVgroupInfo(pSql, 0);
     pSql->res.code = code;
 
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 7a1c08d056..073fffbc57 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -330,7 +330,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
     pNewQueryInfo->limit = pSupporter->limit;
   
     // fetch the join tag column
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, 0);
       assert(pQueryInfo->tagCond.joinInfo.hasJoin);
     
@@ -1096,7 +1096,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     tscInitQueryInfo(pNewQueryInfo);
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
     
-    if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // return the tableId & tag
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
       SSchema s = {0};
       SColumnIndex index = {0};
   
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 864239f8fb..2ef7342d08 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -157,7 +157,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   }
   
   // for select query super table, the super table vgroup list can not be null in any cases.
-  if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     assert(pTableMetaInfo->vgroupList != NULL);
   }
   
@@ -172,7 +172,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
 
   if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) &&
       pQueryInfo->command == TSDB_SQL_SELECT) {
-    return UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo);
+    return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
   }
 
   return false;
@@ -187,7 +187,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
    * 4. show queries, instead of a select query
    */
   size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
-  if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) ||
+  if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) ||
       pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) {
     return false;
   }
@@ -1348,7 +1348,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) {
     return false;
   }
 
-  if (colId == -1 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (colId == -1 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     return true;
   }
 
@@ -1861,7 +1861,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1);
-  if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
     assert(pFinalInfo->vgroupList != NULL);
   }
   
@@ -1996,7 +1996,7 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
 //  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   
 //  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-//  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
+//  if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
     return false;
 //  }
   
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index 21511a1aa3..d400d72a38 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -4685,15 +4685,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       pQuery->limit.offset);
 }
 
-static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) {
-  //  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-
-  //  for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) {
-  //    STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo;
-  //    changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo);
-  //  }
-}
-
 static void doSaveContext(SQInfo *pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;

From 9dd3e62b4b11b96e8f6db1b51ee09a64deace8cf Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Mon, 11 May 2020 17:48:52 +0800
Subject: [PATCH 05/33] build VgroupTableInfo

---
 src/client/inc/tsclient.h    |   3 +
 src/client/src/tscSub.c      | 222 +++++++++++++++++++++++------------
 src/client/src/tscSubquery.c | 100 ++++++----------
 src/client/src/tscUtil.c     |   2 +-
 src/tsdb/src/tsdbRead.c      |   2 +-
 5 files changed, 186 insertions(+), 143 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 6ea1ee6440..a7eec31388 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -438,6 +438,9 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
 
+int32_t tscCompareTidTags(const void* p1, const void* p2);
+void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 009372f3f7..5f20ee4fbc 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -61,14 +61,11 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid) {
     return 0;
   }
   SSub* pSub = (SSub*)sub;
-  if (pSub->progress == NULL) {
-    return 0;
-  }
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
   SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
   if (p == NULL) {
-    return 0;
+    return INT64_MIN;
   }
   return p->key;
 }
@@ -82,61 +79,90 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
   if (p != NULL) {
     p->key = ts;
-    return;
   }
+}
 
-  taosArrayPush(pSub->progress, &target);
-  taosArraySort(pSub->progress, tscCompareSubscriptionProgress);
+
+static void asyncCallback(void *param, TAOS_RES *tres, int code) {
+  assert(param != NULL);
+  SSqlObj *pSql = ((SSqlObj *)param);
+  
+  pSql->res.code = code;
+  sem_post(&pSql->rspSem);
 }
 
 
 static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) {
-  SSub* pSub = calloc(1, sizeof(SSub));
-  if (pSub == NULL) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for subscription");
-    return NULL;
-  }
+  SSub* pSub = NULL;
 
-  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
-  if (pSql == NULL) {
-    terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
-    tscError("failed to allocate SSqlObj for subscription");
-    goto _pSql_failed;
-  }
+  TRY( 8 ) {
+    SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj));
+    CLEANUP_PUSH_FREE(true, pSql);
+    SSqlCmd *pCmd = &pSql->cmd;
+    SSqlRes *pRes = &pSql->res;
 
-  pSql->signature = pSql;
-  pSql->pTscObj = pObj;
+    if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+      THROW(TAOS_SYSTEM_ERROR(errno));
+    }
+    CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem);
 
-  char* sqlstr = (char*)malloc(strlen(sql) + 1);
-  if (sqlstr == NULL) {
-    tscError("failed to allocate sql string for subscription");
-    goto failed;
-  }
-  strcpy(sqlstr, sql);
-  strtolower(sqlstr, sqlstr);
-  pSql->sqlstr = sqlstr;
+    pSql->signature = pSql;
+    pSql->param = pSql;
+    pSql->pTscObj = pObj;
+    pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
+    pSql->fp = asyncCallback;
 
-  tsem_init(&pSql->rspSem, 0, 0);
+    int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
+    if (code != TSDB_CODE_SUCCESS) {
+      THROW(code);
+    }
+    CLEANUP_PUSH_FREE(true, pCmd->payload);
 
-  SSqlRes *pRes = &pSql->res;
-  pRes->numOfRows = 1;
-  pRes->numOfTotal = 0;
+    pRes->qhandle = 0;
+    pRes->numOfRows = 1;
+
+    pSql->sqlstr = strdup_throw(sql);
+    CLEANUP_PUSH_FREE(true, pSql->sqlstr);
+    strtolower(pSql->sqlstr, pSql->sqlstr);
+
+    code = tsParseSql(pSql, false);
+    if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+      // wait for the callback function to post the semaphore
+      sem_wait(&pSql->rspSem);
+      code = pSql->res.code;
+    }
+    if (code != TSDB_CODE_SUCCESS) {
+      tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code));
+      THROW( code );
+    }
+
+    if (pSql->cmd.command != TSDB_SQL_SELECT) {
+      tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
+      THROW( -1 );  // TODO
+    }
+
+    pSub = calloc_throw(1, sizeof(SSub));
+    CLEANUP_PUSH_FREE(true, pSub);
+    pSql->pSubscription = pSub;
+    pSub->pSql = pSql;
+    pSub->signature = pSub;
+    strncpy(pSub->topic, topic, sizeof(pSub->topic));
+    pSub->topic[sizeof(pSub->topic) - 1] = 0;
+    pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
+    if (pSub->progress == NULL) {
+      THROW(TSDB_CODE_CLI_OUT_OF_MEMORY);
+    }
+
+    CLEANUP_EXECUTE();
+
+  } CATCH( code ) {
+    tscError("failed to create subscription object: %s", tstrerror(code));
+    CLEANUP_EXECUTE();
+    pSub = NULL;
+
+  } END_TRY
 
-  pSql->pSubscription = pSub;
-  pSub->pSql = pSql;
-  pSub->signature = pSub;
-  strncpy(pSub->topic, topic, sizeof(pSub->topic));
-  pSub->topic[sizeof(pSub->topic) - 1] = 0;
   return pSub;
-
-failed:
-  tfree(sqlstr);
-
-_pSql_failed:
-  tfree(pSql);
-  tfree(pSub);
-  return NULL;
 }
 
 
@@ -152,41 +178,67 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
   taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer);
 }
 
-static void waitParseComplete(void *param, TAOS_RES *tres, int code) {
-  assert(param != NULL);
-  SSqlObj *pSql = ((SSqlObj *)param);
-  
-  pSql->res.code = code;
-  sem_post(&pSql->rspSem);
+
+static SArray* getTableList( SSqlObj* pSql ) {
+  const char* p = strstr( pSql->sqlstr, " from " );
+  char* sql = alloca(strlen(p) + 32);
+  sprintf(sql, "select tbid(area)%s", p);
+  int code = taos_query( pSql->pTscObj, sql );
+  if (code != TSDB_CODE_SUCCESS) {
+    tscError("failed to retrieve table id: %s", tstrerror(code));
+    return NULL;
+  }
+
+  TAOS_RES* res = taos_use_result( pSql->pTscObj );
+  TAOS_ROW row;
+  SArray* result = taosArrayInit( 128, sizeof(STidTags) );
+  while ((row = taos_fetch_row(res))) {
+    STidTags tags;
+    memcpy(&tags, row[0], sizeof(tags));
+    taosArrayPush(result, &tags);
+  }
+
+  return result;
 }
 
-int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
+
+static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   SSqlObj* pSql = pSub->pSql;
 
   SSqlCmd* pCmd = &pSql->cmd;
-  tscAllocPayload( pCmd, TSDB_DEFAULT_PAYLOAD_SIZE );
 
-  pSql->fp = waitParseComplete;
-  pSql->param = pSql;
-  int code = tsParseSql(pSql, false);
-  if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
-    // wait for the callback function to post the semaphore
-    sem_wait(&pSql->rspSem);
-    code = pSql->res.code;
-  }
-  if (code != TSDB_CODE_SUCCESS) {
-    tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code));
-    return 0;
+  pSub->lastSyncTime = taosGetTimestampMs();
+
+  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
+  if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
+    STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
+    SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
+    if (p == NULL) {
+      taosArrayClear(pSub->progress);
+      taosArrayPush(pSub->progress, &target);
+    }
+    return 1;
   }
 
-  if (pCmd->command != TSDB_SQL_SELECT) {
-    tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
-    return 0;
+  SArray* tables = getTableList(pSql);
+  size_t numOfTables = taosArrayGetSize(tables);
+
+  SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
+  for( size_t i = 0; i < numOfTables; i++ ) {
+    STidTags* tt = taosArrayGet( tables, i );
+    SSubscriptionProgress p = { .uid = tt->uid };
+    p.key = tscGetSubscriptionProgress(pSub, tt->uid);
+    taosArrayPush(progress, &p);
   }
+  taosArraySort(progress, tscCompareSubscriptionProgress);
 
   taosArrayDestroy(pSub->progress);
-  pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
-  pSub->lastSyncTime = taosGetTimestampMs();
+  pSub->progress = progress;
+
+  taosArraySort( tables, tscCompareTidTags );
+  tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+  taosArrayDestroy(tables);
 
   return 1;
 }
@@ -229,11 +281,11 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
   }
 
   int numOfTables = atoi(buf);
-  SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
+  SArray* progress = pSub->progress;
+  taosArrayClear(progress);
   for (int i = 0; i < numOfTables; i++) {
     if (fgets(buf, sizeof(buf), fp) == NULL) {
       fclose(fp);
-      free(progress);
       return 0;
     }
     SSubscriptionProgress p;
@@ -244,8 +296,6 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
   fclose(fp);
 
   taosArraySort(progress, tscCompareSubscriptionProgress);
-  taosArrayDestroy(pSub->progress);
-  pSub->progress = progress;
   tscTrace("subscription progress loaded, %d tables: %s", numOfTables, pSub->topic);
   return 1;
 }
@@ -333,12 +383,28 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   for (int retry = 0; retry < 3; retry++) {
     tscRemoveFromSqlList(pSql);
 
-    pSql->fp = waitParseComplete;
+    if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) {
+      tscTrace("begin table synchronization");
+      if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
+      tscTrace("table synchronization completed");
+    } else {
+      SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+      
+      uint32_t type = pQueryInfo->type;
+      // taos_free_result_imp(pSql, 1);
+      pRes->numOfRows = 1;
+      //pRes->numOfTotal = 0;
+      pRes->qhandle = 0;
+      pSql->cmd.command = TSDB_SQL_SELECT;
+      pQueryInfo->type = type;
+
+      tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
+    }
+
+    pSql->fp = asyncCallback;
     pSql->param = pSql;
     tscDoQuery(pSql);
-    if (pRes->code == TSDB_CODE_ACTION_IN_PROGRESS) {
-      sem_wait(&pSql->rspSem);
-    }
+    sem_wait(&pSql->rspSem);
 
     if (pRes->code != TSDB_CODE_SUCCESS) {
       continue;
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 073fffbc57..fbe19fc030 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -463,77 +463,51 @@ static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj*
   }
 }
 
-int32_t tagsOrderCompar(const void* p1, const void* p2) {
-  STidTags* t1 = (STidTags*) p1;
-  STidTags* t2 = (STidTags*) p2;
+int32_t tscCompareTidTags(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) p1;
+  const STidTags* t2 = (const STidTags*) p2;
   
   if (t1->vgId != t2->vgId) {
-    return (t1->vgId > t2->vgId)? 1:-1;
-  } else {
-    if (t1->tid != t2->tid) {
-      return (t1->tid > t2->tid)? 1:-1;
-    } else {
-      return 0;
-    }
+    return (t1->vgId > t2->vgId) ? 1 : -1;
   }
+  if (t1->tid != t2->tid) {
+    return (t1->tid > t2->tid) ? 1 : -1;
+  }
+  return 0;
 }
 
-static void doBuildVgroupTableInfo(SArray* res, STableMetaInfo* pTableMetaInfo) {
-  SArray* pGroup = taosArrayInit(4, sizeof(SVgroupTableInfo));
-  
-  SArray* vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo));
-  int32_t size = taosArrayGetSize(res);
-  
-  STidTags* prev = taosArrayGet(res, 0);
-  int32_t prevVgId = prev->vgId;
-  
-  STableIdInfo item = {.uid = prev->uid, .tid = prev->tid, .key = INT64_MIN};
-  taosArrayPush(vgTableIdItem, &item);
-  
-  for(int32_t k = 1; k < size; ++k) {
-    STidTags* t1 = taosArrayGet(res, k);
-    if (prevVgId != t1->vgId) {
-      
-      SVgroupTableInfo info = {0};
-      
+void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) {
+  SArray* result = taosArrayInit( 4, sizeof(SVgroupTableInfo) );
+  SArray* vgTables = NULL;
+  STidTags* prev = NULL;
+
+  size_t numOfTables = taosArrayGetSize( tables );
+  for( size_t i = 0; i < numOfTables; i++ ) {
+    STidTags* tt = taosArrayGet( tables, i );
+
+    if( prev == NULL || tt->vgId != prev->vgId ) {
       SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
-      for(int32_t m = 0; m < pvg->numOfVgroups; ++m) {
-        if (prevVgId == pvg->vgroups[m].vgId) {
+
+      SVgroupTableInfo info = { 0 };
+      for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) {
+        if( tt->vgId == pvg->vgroups[m].vgId ) {
           info.vgInfo = pvg->vgroups[m];
           break;
         }
       }
-      
-      assert(info.vgInfo.numOfIps != 0);
-      info.itemList = vgTableIdItem;
-      taosArrayPush(pGroup, &info);
-      
-      vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo));
-      STableIdInfo item1 = {.uid = t1->uid, .tid = t1->tid, .key = INT64_MIN};
-      taosArrayPush(vgTableIdItem, &item1);
-      prevVgId = t1->vgId;
-    } else {
-      taosArrayPush(vgTableIdItem, &item);
+      assert( info.vgInfo.numOfIps != 0 );
+
+      vgTables = taosArrayInit( 4, sizeof(STableIdInfo) );
+      info.itemList = vgTables;
+      taosArrayPush( result, &info );
     }
+
+    STableIdInfo item = { .uid = tt->uid, .tid = tt->tid, .key = INT64_MIN };
+    taosArrayPush( vgTables, &item );
+    prev = tt;
   }
-  
-  if (taosArrayGetSize(vgTableIdItem) > 0) {
-    SVgroupTableInfo info = {0};
-    SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
-    
-    for(int32_t m = 0; m < pvg->numOfVgroups; ++m) {
-      if (prevVgId == pvg->vgroups[m].vgId) {
-        info.vgInfo = pvg->vgroups[m];
-        break;
-      }
-    }
-    
-    assert(info.vgInfo.numOfIps != 0);
-    info.itemList = vgTableIdItem;
-    taosArrayPush(pGroup, &info);
-  }
-  
-  pTableMetaInfo->pVgroupTables = pGroup;
+
+  pTableMetaInfo->pVgroupTables = result;
 }
 
 static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@@ -627,8 +601,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
       SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
       SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
   
-      qsort(p1->pIdTagList, p1->num, p1->tagSize, tagsOrderCompar);
-      qsort(p2->pIdTagList, p2->num, p2->tagSize, tagsOrderCompar);
+      qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
+      qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
       
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
@@ -668,11 +642,11 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         
         SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0);
         STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0);
-        doBuildVgroupTableInfo(s1, pTableMetaInfo1);
+        tscBuildVgroupTableInfo(pTableMetaInfo1, s1);
         
         SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0);
         STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
-        doBuildVgroupTableInfo(s2, pTableMetaInfo2);
+        tscBuildVgroupTableInfo(pTableMetaInfo2, s2);
   
         pSupporter->pState->numOfCompleted = 0;
         pSupporter->pState->code = 0;
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 2ef7342d08..85735b14e1 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -2165,7 +2165,7 @@ void* malloc_throw(size_t size) {
 }
 
 void* calloc_throw(size_t nmemb, size_t size) {
-  void* p = malloc(size);
+  void* p = calloc(nmemb, size);
   if (p == NULL) {
     THROW(TSDB_CODE_CLI_OUT_OF_MEMORY);
   }
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index 6576e7d7f1..a999502897 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -1522,7 +1522,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC
   }
   
   if (pTable->type != TSDB_SUPER_TABLE) {
-    uError("%p query normal tag not allowed, uid:%, tid:%d, name:%s" PRIu64,
+    uError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s",
            tsdb, uid, pTable->tableId.tid, pTable->name);
     
     return TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client

From a36937624ed78b49717d0753c2418961acaabe53 Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Tue, 12 May 2020 15:34:56 +0800
Subject: [PATCH 06/33] super table subscribe is done

---
 src/client/src/tscServer.c    | 14 ++++-----
 src/client/src/tscSub.c       | 41 ++++++++++++--------------
 src/query/src/queryExecutor.c | 55 ++++++++++++++++++++++-------------
 3 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 9431a09bb0..32eee08541 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -589,25 +589,25 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     } else {
       pVgroupInfo = &pTableMeta->vgroupInfo;
     }
-    
+
     tscSetDnodeIpList(pSql, pVgroupInfo);
     pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId);
-    
+
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
     pTableIdInfo->tid = htonl(pTableMeta->sid);
     pTableIdInfo->uid = htobe64(pTableMeta->uid);
     pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
-  
+
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
-    
+
     pMsg += sizeof(STableIdInfo);
   } else {
     int32_t index = pTableMetaInfo->vgroupIndex;
     int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
     assert(index >= 0 && index < numOfVgroups);
-  
+
     tscTrace("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups);
-    
+
     SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index);
     
     // set the vgroup info
@@ -624,7 +624,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
       STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
       pTableIdInfo->tid = htonl(pItem->tid);
       pTableIdInfo->uid = htobe64(pItem->uid);
-      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
+      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid));
       pMsg += sizeof(STableIdInfo);
     }
   }
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 5f20ee4fbc..8918338265 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -236,8 +236,10 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   taosArrayDestroy(pSub->progress);
   pSub->progress = progress;
 
-  taosArraySort( tables, tscCompareTidTags );
-  tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+  if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+    taosArraySort( tables, tscCompareTidTags );
+    tscBuildVgroupTableInfo( pTableMetaInfo, tables );
+  }
   taosArrayDestroy(tables);
 
   return 1;
@@ -274,16 +276,9 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
     return 0;
   }
 
-  if (fgets(buf, sizeof(buf), fp) == NULL || atoi(buf) < 0) {
-    tscTrace("invalid subscription progress file: %s", pSub->topic);
-    fclose(fp);
-    return 0;
-  }
-
-  int numOfTables = atoi(buf);
   SArray* progress = pSub->progress;
   taosArrayClear(progress);
-  for (int i = 0; i < numOfTables; i++) {
+  while( 1 ) {
     if (fgets(buf, sizeof(buf), fp) == NULL) {
       fclose(fp);
       return 0;
@@ -296,7 +291,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
   fclose(fp);
 
   taosArraySort(progress, tscCompareSubscriptionProgress);
-  tscTrace("subscription progress loaded, %d tables: %s", numOfTables, pSub->topic);
+  tscTrace("subscription progress loaded, %d tables: %s", taosArrayGetSize(progress), pSub->topic);
   return 1;
 }
 
@@ -317,6 +312,7 @@ void tscSaveSubscriptionProgress(void* sub) {
   }
 
   fputs(pSub->pSql->sqlstr, fp);
+  fprintf(fp, "\n");
   for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) {
     SSubscriptionProgress* p = taosArrayGet(pSub->progress, i);
     fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key);
@@ -387,20 +383,19 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
       tscTrace("begin table synchronization");
       if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
       tscTrace("table synchronization completed");
-    } else {
-      SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-      
-      uint32_t type = pQueryInfo->type;
-      // taos_free_result_imp(pSql, 1);
-      pRes->numOfRows = 1;
-      //pRes->numOfTotal = 0;
-      pRes->qhandle = 0;
-      pSql->cmd.command = TSDB_SQL_SELECT;
-      pQueryInfo->type = type;
-
-      tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
     }
 
+    SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+    
+    uint32_t type = pQueryInfo->type;
+    tscFreeSqlResult(pSql);
+    pRes->numOfRows = 1;
+    pRes->qhandle = 0;
+    pSql->cmd.command = TSDB_SQL_SELECT;
+    pQueryInfo->type = type;
+
+    tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
+
     pSql->fp = asyncCallback;
     pSql->param = pSql;
     tscDoQuery(pSql);
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index d400d72a38..8bd2052ef8 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -4184,11 +4184,9 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
     if (!isSTableQuery && isFirstLastRowQuery(pQuery)) {  // in case of last_row query, invoke a different API.
       pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
     } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
+
       pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
     }
-  
-    // create the table query support structures
-    createTableQueryInfo(pQInfo);
   }
   
   pQInfo->tsdb = tsdb;
@@ -4608,12 +4606,11 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
          * to ensure that, we can reset the query range once query on a meter is completed.
          */
         pQInfo->tableIndex++;
-        pInfo->lastKey = pQuery->lastKey;
 
         STableIdInfo tidInfo;
         tidInfo.uid = item->id.uid;
         tidInfo.tid = item->id.tid;
-        tidInfo.key = pInfo->lastKey;
+        tidInfo.key = pQuery->current->lastKey;
         taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
 
         // if the buffer is full or group by each table, we need to jump out of the loop
@@ -5198,6 +5195,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p
     pTableIdInfo->uid = htobe64(pTableIdInfo->uid);
     pTableIdInfo->key = htobe64(pTableIdInfo->key);
 
+printf("createTableIdList: uid = %ld, key = %ld\n", pTableIdInfo->uid, pTableIdInfo->key);
     taosArrayPush(*pTableIdList, pTableIdInfo);
     pMsg += sizeof(STableIdInfo);
   }
@@ -5761,9 +5759,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
       // not a problem at present because we only use their 1st int64_t field
       STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
       if (pTableId != NULL ) {
+        printf("create QInfoImpl: %ld  %ld\n", pTableId->uid, pTableId->key);
         window.skey = pTableId->key;
       } else {
-        window.skey = 0;
+        window.skey = INT64_MIN;
       }
       item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window);
       item.info->groupIdx = i;
@@ -6060,20 +6059,36 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     }
   } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) {
     isSTableQuery = true;
-    STableIdInfo *id = taosArrayGet(pTableIdList, 0);
-    
-    // group by normal column, do not pass the group by condition to tsdb to group table into different group
-    int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
-    if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) {
-      numOfGroupByCols = 0;
-    }
-    
-    // todo handle the error
-    /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex,
-                                         numOfGroupByCols);
-    if (groupInfo.numOfTables == 0) {  // no qualified tables no need to do query
-      code = TSDB_CODE_SUCCESS;
-      goto _over;
+    // TODO: need a macro from TSDB to check if table is super table,
+    // also note there's possiblity that only one table in the super table
+    if (taosArrayGetSize(pTableIdList) == 1) {
+      STableIdInfo *id = taosArrayGet(pTableIdList, 0);
+      // if array size is 1 and assert super table
+
+      // group by normal column, do not pass the group by condition to tsdb to group table into different group
+      int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
+      if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) {
+        numOfGroupByCols = 0;
+      }
+      
+      // todo handle the error
+      /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex,
+                                          numOfGroupByCols);
+      if (groupInfo.numOfTables == 0) {  // no qualified tables no need to do query
+        code = TSDB_CODE_SUCCESS;
+        goto _over;
+      }
+    } else {
+      groupInfo.numOfTables = taosArrayGetSize(pTableIdList);
+      SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
+
+      SArray* sa = taosArrayInit(groupInfo.numOfTables, sizeof(STableId));
+      for(int32_t i = 0; i < groupInfo.numOfTables; ++i) {
+        STableIdInfo* tableId = taosArrayGet(pTableIdList, i);
+        taosArrayPush(sa, tableId);
+      }
+      taosArrayPush(pTableGroup, &sa);
+      groupInfo.pGroupList = pTableGroup;
     }
   } else {
     assert(0);

From 577cf92fdf3a6fa96ed3594145abae561c6bc7bd Mon Sep 17 00:00:00 2001
From: localvar <localvar@hotmail.com>
Date: Tue, 12 May 2020 16:49:43 +0800
Subject: [PATCH 07/33] single table subscribe is done

---
 src/query/src/queryExecutor.c | 14 ++++++-
 tests/examples/c/subscribe.c  | 73 +++++++++++++++++++++--------------
 2 files changed, 56 insertions(+), 31 deletions(-)

diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index 8bd2052ef8..24f0f53ca5 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -4185,6 +4185,12 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
       pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
     } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
 
+      if(pQInfo->groupInfo.numOfTables == 1) {
+        SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+        SGroupItem* pItem = taosArrayGet(pa, 0);
+        cond.twindow = pItem->info->win;
+      }
+
       pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
     }
   }
@@ -4903,6 +4909,12 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
     qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo,
         pQuery->current->lastKey, pQuery->window.ekey);
+  } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
+    STableIdInfo tidInfo;
+    tidInfo.uid = pQuery->current->id.uid;
+    tidInfo.tid = pQuery->current->id.tid;
+    tidInfo.key = pQuery->current->lastKey;
+    taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
   }
 
   if (!isTSCompQuery(pQuery)) {
@@ -5195,7 +5207,6 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p
     pTableIdInfo->uid = htobe64(pTableIdInfo->uid);
     pTableIdInfo->key = htobe64(pTableIdInfo->key);
 
-printf("createTableIdList: uid = %ld, key = %ld\n", pTableIdInfo->uid, pTableIdInfo->key);
     taosArrayPush(*pTableIdList, pTableIdInfo);
     pMsg += sizeof(STableIdInfo);
   }
@@ -5759,7 +5770,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
       // not a problem at present because we only use their 1st int64_t field
       STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
       if (pTableId != NULL ) {
-        printf("create QInfoImpl: %ld  %ld\n", pTableId->uid, pTableId->key);
         window.skey = pTableId->key;
       } else {
         window.skey = INT64_MIN;
diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c
index 0bf93f6f2d..f9acf2bb10 100644
--- a/tests/examples/c/subscribe.c
+++ b/tests/examples/c/subscribe.c
@@ -56,32 +56,46 @@ void run_test(TAOS* taos) {
   taos_query(taos, "drop database if exists test;");
   
   usleep(100000);
-  taos_query(taos, "create database test tables 5;");
+  //taos_query(taos, "create database test tables 5;");
+  taos_query(taos, "create database test;");
   usleep(100000);
   taos_query(taos, "use test;");
-  usleep(100000);
-  taos_query(taos, "create table meters(ts timestamp, a int, b binary(20)) tags(loc binary(20), area int);");
 
-  taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:00:00.000', 0, 'china');");
-  taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:01:00.000', 0, 'china');");
-  taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:02:00.000', 0, 'china');");
-  taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:00:00.000', 0, 'china');");
-  taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:01:00.000', 0, 'china');");
-  taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:02:00.000', 0, 'china');");
-  taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:03:00.000', 0, 'china');");
-  taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:00:00.000', 0, 'UK');");
-  taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:00.000', 0, 'UK');");
-  taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:01.000', 0, 'UK');");
-  taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:02.000', 0, 'UK');");
-  taos_query(taos, "insert into t3 using meters tags('tianjin', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t4 using meters tags('wuhan', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t5 using meters tags('jinan', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t6 using meters tags('haikou', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t7 using meters tags('nanjing', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t8 using meters tags('lanzhou', 0) values('2020-01-01 00:01:02.000', 0, 'china');");
-  taos_query(taos, "insert into t9 using meters tags('tokyo', 0) values('2020-01-01 00:01:02.000', 0, 'japan');");
+  usleep(100000);
+  taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+
+  taos_query(taos, "create table t0 using meters tags(0);");
+  taos_query(taos, "create table t1 using meters tags(1);");
+  taos_query(taos, "create table t2 using meters tags(2);");
+  taos_query(taos, "create table t3 using meters tags(3);");
+  taos_query(taos, "create table t4 using meters tags(4);");
+  taos_query(taos, "create table t5 using meters tags(5);");
+  taos_query(taos, "create table t6 using meters tags(6);");
+  taos_query(taos, "create table t7 using meters tags(7);");
+  taos_query(taos, "create table t8 using meters tags(8);");
+  taos_query(taos, "create table t9 using meters tags(9);");
+
+  taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0);");
+  taos_query(taos, "insert into t0 values('2020-01-01 00:01:00.000', 0);");
+  taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.000', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:00:00.000', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:01:00.000', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:02:00.000', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.000', 0);");
+  taos_query(taos, "insert into t2 values('2020-01-01 00:00:00.000', 0);");
+  taos_query(taos, "insert into t2 values('2020-01-01 00:01:00.000', 0);");
+  taos_query(taos, "insert into t2 values('2020-01-01 00:01:01.000', 0);");
+  taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t3 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t4 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t5 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t6 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t7 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t8 values('2020-01-01 00:01:02.000', 0);");
+  taos_query(taos, "insert into t9 values('2020-01-01 00:01:02.000', 0);");
 
   // super tables subscription
+  usleep(1000000);
 
   TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0);
   TAOS_RES* res = taos_consume(tsub);
@@ -90,23 +104,23 @@ void run_test(TAOS* taos) {
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 0);
 
-  taos_query(taos, "insert into t0 values('2020-01-01 00:03:00.000', 0, 'china');");
-  taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0, 'china');");
+  taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
+  taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 2);
 
-  taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0, 'UK');");
-  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0, 'UK');");
+  taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 2);
 
-  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0, 'china');");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 1);
 
   // keep progress information and restart subscription
   taos_unsubscribe(tsub, 1);
-  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0, 'china');");
+  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
   tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 24);
@@ -133,7 +147,7 @@ void run_test(TAOS* taos) {
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 0);
 
-  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0, 'china');");
+  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 1);
 
@@ -197,7 +211,7 @@ int main(int argc, char *argv[]) {
   // init TAOS
   taos_init();
 
-  TAOS* taos = taos_connect(host, user, passwd, "test", 0);
+  TAOS* taos = taos_connect(host, user, passwd, "", 0);
   if (taos == NULL) {
     printf("failed to connect to db, reason:%s\n", taos_errstr(taos));
     exit(1);
@@ -209,6 +223,7 @@ int main(int argc, char *argv[]) {
     exit(0);
   }
 
+  taos_query(taos, "use test;");
   TAOS_SUB* tsub = NULL;
   if (async) {
     // create an asynchronized subscription, the callback function will be called every 1s

From 98baaf51cd14ed8798bff173155d9240c6a1928f Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Tue, 12 May 2020 17:36:36 +0800
Subject: [PATCH 08/33] use tbname in function tbid()

---
 src/client/src/tscSub.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 8918338265..9a3f3d290c 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -182,7 +182,7 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
 static SArray* getTableList( SSqlObj* pSql ) {
   const char* p = strstr( pSql->sqlstr, " from " );
   char* sql = alloca(strlen(p) + 32);
-  sprintf(sql, "select tbid(area)%s", p);
+  sprintf(sql, "select tbid(tbname)%s", p);
   int code = taos_query( pSql->pTscObj, sql );
   if (code != TSDB_CODE_SUCCESS) {
     tscError("failed to retrieve table id: %s", tstrerror(code));

From 8cb168a3343e5d123ff255102617e5969e18a9d8 Mon Sep 17 00:00:00 2001
From: changshuaiqiang <sqchang@taosdata.com>
Date: Tue, 12 May 2020 18:20:05 +0800
Subject: [PATCH 09/33] test: add db cases

---
 tests/script/basicSuite.sim                    | 16 +++++++++++++++-
 tests/script/general/connection/connection.sim |  2 +-
 tests/script/general/db/vnodes.sim             |  2 ++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim
index d0bb58ec20..7dbe51ff38 100644
--- a/tests/script/basicSuite.sim
+++ b/tests/script/basicSuite.sim
@@ -19,4 +19,18 @@ run general/user/basic1.sim
 run general/user/pass_alter.sim
 run general/vector/single.sim
 run general/connection/connection.sim
-run general/user/authority.sim
+run general/user/testSuite.sim
+run general/db/basic.sim
+run general/db/basic1.sim
+run general/db/basic2.sim
+run general/db/basic3.sim
+run general/db/basic4.sim
+run general/db/basic5.sim
+run general/db/delete_reuse1.sim
+run general/db/delete_reuse2.sim
+run general/db/delete_reusevnode.sim
+run general/db/delete_writing1.sim
+run general/db/delete_writing2.sim
+run general/db/len.sim
+run general/db/repeat.sim
+run general/db/tables.sim
diff --git a/tests/script/general/connection/connection.sim b/tests/script/general/connection/connection.sim
index 71db20b285..abebbacbd9 100644
--- a/tests/script/general/connection/connection.sim
+++ b/tests/script/general/connection/connection.sim
@@ -16,6 +16,6 @@ sql connect
 
 print ============= step3
 sql close
-sql connect write
+sql connect
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim
index 684910884b..54d8aa77c3 100644
--- a/tests/script/general/db/vnodes.sim
+++ b/tests/script/general/db/vnodes.sim
@@ -33,6 +33,8 @@ endw
 
 print ========== step2
 sql select * from mt
+print $rows
+print $totalRows
 if $rows != $totalRows then
   return -1
 endi  

From cf06a3c8859149bab79cddcb3650100a81da4432 Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Tue, 12 May 2020 18:36:35 +0800
Subject: [PATCH 10/33] [td-225] fix bugs for 256 columns data check

---
 src/client/src/tscSQLParser.c |  4 ----
 src/client/src/tscServer.c    | 13 ++-----------
 src/client/src/tscUtil.c      |  6 ++++++
 src/inc/taosmsg.h             |  1 -
 src/mnode/src/mgmtTable.c     |  5 +++--
 5 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index a81ad19e4f..3dbd5d36bd 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -1297,10 +1297,6 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c
       pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ);
 }
 
-void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) {
-
-}
-
 static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
   SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex);
 
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 8efe89d28a..f72f53c2b1 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -544,8 +544,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 }
 
 /*
- * for meter query, simply return the size <= 1k
- * for metric query, estimate size according to meter tags
+ * for table query, simply return the size <= 1k
  */
 static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5;
@@ -556,15 +555,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
   int32_t exprSize = sizeof(SSqlFuncMsg) * numOfExprs;
   
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
-  // meter query without tags values
-  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
-    return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize;
-  }
-  
-  int32_t size = 4096;
-  return size;
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
 }
 
 static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index c25e91a669..bea61f12c4 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1701,6 +1701,12 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
   tfree(pTableMetaInfo->vgroupList);
   
   if (pTableMetaInfo->tagColList != NULL) {
+    size_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList);
+    for(int32_t i = 0; i < numOfTags; ++i) { // todo do NOT use the allocated object
+      SColumn* pCol = taosArrayGetP(pTableMetaInfo->tagColList, i);
+      tfree(pCol);
+    }
+    
     taosArrayDestroy(pTableMetaInfo->tagColList);
     pTableMetaInfo->tagColList = NULL;
   }
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index 8dac73606d..9d16299380 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -626,7 +626,6 @@ typedef struct {
 typedef struct STableMetaMsg {
   int32_t       contLen;
   char          tableId[TSDB_TABLE_ID_LEN + 1];   // table id
-  char          stableId[TSDB_TABLE_ID_LEN + 1];  // stable name if it is created according to super table
   uint8_t       numOfTags;
   uint8_t       precision;
   uint8_t       tableType;
diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c
index bfe357cf7c..3e11bd5f14 100644
--- a/src/mnode/src/mgmtTable.c
+++ b/src/mnode/src/mgmtTable.c
@@ -1206,8 +1206,10 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
 
 static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
   int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags;
+  assert(numOfCols <= TSDB_MAX_COLUMNS);
+  
   for (int32_t i = 0; i < numOfCols; ++i) {
-    strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN);
+    strncpy(pSchema->name, pTable->schema[i].name, TSDB_COL_NAME_LEN);
     pSchema->type  = pTable->schema[i].type;
     pSchema->bytes = htons(pTable->schema[i].bytes);
     pSchema->colId = htons(pTable->schema[i].colId);
@@ -1679,7 +1681,6 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
     pMeta->numOfTags    = (int8_t)pTable->superTable->numOfTags;
     pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns);
     pMeta->contLen      = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable);
-    strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId));
   } else {
     pMeta->sversion     = htons(pTable->sversion);
     pMeta->numOfTags    = 0;

From 196d4f1340127e5efd7d858c5226ef11c386ce2e Mon Sep 17 00:00:00 2001
From: changshuaiqiang <sqchang@taosdata.com>
Date: Tue, 12 May 2020 18:47:55 +0800
Subject: [PATCH 11/33] test: del basic

---
 tests/script/basicSuite.sim | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim
index 7dbe51ff38..a99edfd93d 100644
--- a/tests/script/basicSuite.sim
+++ b/tests/script/basicSuite.sim
@@ -18,19 +18,3 @@ run general/table/vgroup.sim
 run general/user/basic1.sim
 run general/user/pass_alter.sim
 run general/vector/single.sim
-run general/connection/connection.sim
-run general/user/testSuite.sim
-run general/db/basic.sim
-run general/db/basic1.sim
-run general/db/basic2.sim
-run general/db/basic3.sim
-run general/db/basic4.sim
-run general/db/basic5.sim
-run general/db/delete_reuse1.sim
-run general/db/delete_reuse2.sim
-run general/db/delete_reusevnode.sim
-run general/db/delete_writing1.sim
-run general/db/delete_writing2.sim
-run general/db/len.sim
-run general/db/repeat.sim
-run general/db/tables.sim

From 11c0f8ba32d1c2b3c44374e54d586df5e36048c1 Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Tue, 12 May 2020 19:23:21 +0800
Subject: [PATCH 12/33] [td-225] fix bugs at client

---
 src/client/inc/tscSecondaryMerge.h |  2 +-
 src/client/src/tscSecondaryMerge.c | 85 +++++++++++++++---------------
 src/client/src/tscSubquery.c       |  3 +-
 src/query/inc/qextbuffer.h         |  6 ---
 4 files changed, 45 insertions(+), 51 deletions(-)

diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscSecondaryMerge.h
index f6c7516754..ad743eeea3 100644
--- a/src/client/inc/tscSecondaryMerge.h
+++ b/src/client/inc/tscSecondaryMerge.h
@@ -116,7 +116,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
  * create local reducer to launch the second-stage reduce process at client site
  */
 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                           SColumnModel *finalModel, SSqlCmd *pSqlCmd, SSqlRes *pRes);
+                           SColumnModel *finalModel, SSqlObj* pSql);
 
 void tscDestroyLocalReducer(SSqlObj *pSql);
 
diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c
index 8c75ed6c35..71d50b56f9 100644
--- a/src/client/src/tscSecondaryMerge.c
+++ b/src/client/src/tscSecondaryMerge.c
@@ -55,7 +55,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
   }
 }
 
-static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pReducer, tOrderDescriptor *pDesc) {
+static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDescriptor *pDesc) {
   /*
    * the fields and offset attributes in pCmd and pModel may be different due to
    * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object.
@@ -96,13 +96,13 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
       pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf;
       pCtx->param[2].i64Key = pQueryInfo->order.order;
-      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
+      pCtx->param[2].nType  = TSDB_DATA_TYPE_BIGINT;
       pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
     }
 
     SResultInfo *pResInfo = &pReducer->pResInfo[i];
     pResInfo->bufLen = pExpr->interBytes;
-    pResInfo->interResultBuf = calloc(1, (size_t)pResInfo->bufLen);
+    pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen);
 
     pCtx->resultInfo = &pReducer->pResInfo[i];
     pCtx->resultInfo->superTableQ = true;
@@ -132,16 +132,15 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu
   }
 }
 
-/*
- * todo release allocated memory process with async process
- */
 void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                           SColumnModel *finalmodel, SSqlCmd *pCmd, SSqlRes *pRes) {
-  // offset of cmd in SSqlObj structure
-  char *pSqlObjAddr = (char *)pCmd - offsetof(SSqlObj, cmd);
-
+                           SColumnModel *finalmodel, SSqlObj* pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  
   if (pMemBuffer == NULL) {
-    tscError("%p pMemBuffer", pMemBuffer);
+    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer);
+  
+    tscError("%p pMemBuffer is NULL", pMemBuffer);
     pRes->code = TSDB_CODE_APP_ERROR;
     return;
   }
@@ -149,7 +148,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   if (pDesc->pColumnModel == NULL) {
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer);
 
-    tscError("%p no local buffer or intermediate result format model", pSqlObjAddr);
+    tscError("%p no local buffer or intermediate result format model", pSql);
     pRes->code = TSDB_CODE_APP_ERROR;
     return;
   }
@@ -158,7 +157,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   for (int32_t i = 0; i < numOfBuffer; ++i) {
     int32_t len = pMemBuffer[i]->fileMeta.flushoutData.nLength;
     if (len == 0) {
-      tscTrace("%p no data retrieved from orderOfVnode:%d", pSqlObjAddr, i + 1);
+      tscTrace("%p no data retrieved from orderOfVnode:%d", pSql, i + 1);
       continue;
     }
 
@@ -167,13 +166,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   if (numOfFlush == 0 || numOfBuffer == 0) {
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer);
-    tscTrace("%p retrieved no data", pSqlObjAddr);
+    tscTrace("%p retrieved no data", pSql);
 
     return;
   }
 
   if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) {
-    tscError("%p Invalid value of buffer capacity %d and page size %d ", pSqlObjAddr, pDesc->pColumnModel->capacity,
+    tscError("%p Invalid value of buffer capacity %d and page size %d ", pSql, pDesc->pColumnModel->capacity,
              pMemBuffer[0]->pageSize);
 
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer);
@@ -181,10 +180,11 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     return;
   }
 
-  size_t         nReducerSize = sizeof(SLocalReducer) + sizeof(void *) * numOfFlush;
-  SLocalReducer *pReducer = (SLocalReducer *)calloc(1, nReducerSize);
+  size_t size = sizeof(SLocalReducer) + POINTER_BYTES * numOfFlush;
+  
+  SLocalReducer *pReducer = (SLocalReducer *) calloc(1, size);
   if (pReducer == NULL) {
-    tscError("%p failed to create merge structure", pSqlObjAddr);
+    tscError("%p failed to create local merge structure, out of memory", pSql);
 
     tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer);
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@@ -199,48 +199,52 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->numOfVnode = numOfBuffer;
 
   pReducer->pDesc = pDesc;
-  tscTrace("%p the number of merged leaves is: %d", pSqlObjAddr, pReducer->numOfBuffer);
+  tscTrace("%p the number of merged leaves is: %d", pSql, pReducer->numOfBuffer);
 
   int32_t idx = 0;
   for (int32_t i = 0; i < numOfBuffer; ++i) {
     int32_t numOfFlushoutInFile = pMemBuffer[i]->fileMeta.flushoutData.nLength;
 
     for (int32_t j = 0; j < numOfFlushoutInFile; ++j) {
-      SLocalDataSource *pDS = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize);
-      if (pDS == NULL) {
-        tscError("%p failed to create merge structure", pSqlObjAddr);
+      SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize);
+      if (ds == NULL) {
+        tscError("%p failed to create merge structure", pSql);
         pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
         return;
       }
-      pReducer->pLocalDataSrc[idx] = pDS;
+      
+      pReducer->pLocalDataSrc[idx] = ds;
 
-      pDS->pMemBuffer = pMemBuffer[i];
-      pDS->flushoutIdx = j;
-      pDS->filePage.numOfElems = 0;
-      pDS->pageId = 0;
-      pDS->rowIdx = 0;
+      ds->pMemBuffer = pMemBuffer[i];
+      ds->flushoutIdx = j;
+      ds->filePage.numOfElems = 0;
+      ds->pageId = 0;
+      ds->rowIdx = 0;
 
-      tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSqlObjAddr, i + 1, idx + 1);
-      tExtMemBufferLoadData(pMemBuffer[i], &(pDS->filePage), j, 0);
+      tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSql, i + 1, idx + 1);
+      tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0);
 #ifdef _DEBUG_VIEW
-      printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", pDS->filePage.numOfElems);
+      printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.numOfElems);
       SSrcColumnInfo colInfo[256] = {0};
       SQueryInfo *   pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
       tscGetSrcColumnInfo(colInfo, pQueryInfo);
 
-      tColModelDisplayEx(pDesc->pColumnModel, pDS->filePage.data, pDS->filePage.numOfElems,
+      tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.numOfElems,
                          pMemBuffer[0]->numOfElemsPerPage, colInfo);
 #endif
-      if (pDS->filePage.numOfElems == 0) {  // no data in this flush
-        tscTrace("%p flush data is empty, ignore %d flush record", pSqlObjAddr, idx);
-        tfree(pDS);
+      
+      if (ds->filePage.numOfElems == 0) {  // no data in this flush, the index does not increase
+        tscTrace("%p flush data is empty, ignore %d flush record", pSql, idx);
+        tfree(ds);
         continue;
       }
+      
       idx += 1;
     }
   }
-  assert(idx >= pReducer->numOfBuffer);
+  
+  // no data actually, no need to merge result.
   if (idx == 0) {
     return;
   }
@@ -262,9 +266,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   // the input data format follows the old format, but output in a new format.
   // so, all the input must be parsed as old format
-  size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-  
-  pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx));
+  pReducer->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx));
   pReducer->rowSize = pMemBuffer[0]->nElemSize;
 
   tscRestoreSQLFuncForSTableQuery(pQueryInfo);
@@ -313,7 +315,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->pResInfo = calloc(size, sizeof(SResultInfo));
 
   tscCreateResPointerInfo(pRes, pQueryInfo);
-  tscInitSqlContext(pCmd, pRes, pReducer, pDesc);
+  tscInitSqlContext(pCmd, pReducer, pDesc);
 
   // we change the capacity of schema to denote that there is only one row in temp buffer
   pReducer->pDesc->pColumnModel->capacity = 1;
@@ -428,8 +430,7 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
     tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows);
 
     if (pPage->numOfElems == pModel->capacity) {
-      int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType);
-      if (ret != 0) {
+      if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) {
         return -1;
       }
     } else {
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 7a1c08d056..1e4a142a90 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1533,8 +1533,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
   tscClearInterpInfo(pPQueryInfo);
   
-  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
-                        &pPObj->cmd, &pPObj->res);
+  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pPObj);
   tscTrace("%p build loser tree completed", pPObj);
   
   pPObj->res.precision = pSql->res.precision;
diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qextbuffer.h
index a73948fe3b..9389e5a5ab 100644
--- a/src/query/inc/qextbuffer.h
+++ b/src/query/inc/qextbuffer.h
@@ -120,12 +120,6 @@ typedef struct tExtMemBuffer {
   EXT_BUFFER_FLUSH_MODEL flushModel;
 } tExtMemBuffer;
 
-//typedef struct tTagSchema {
-//  struct SSchema *pSchema;
-//  int32_t         numOfCols;
-//  int32_t         colOffset[];
-//} tTagSchema;
-
 /**
  *
  * @param inMemSize

From 932604173a30b6cfd94a6c4db740914dcc4f5af0 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 10:23:56 +0800
Subject: [PATCH 13/33] [TD-149] change hash imp in sdb

---
 src/mnode/src/mgmtSdb.c   | 79 +++++++++++++++++++++++++--------------
 src/mnode/src/mgmtTable.c |  3 ++
 2 files changed, 54 insertions(+), 28 deletions(-)

diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c
index c25f4457a9..3116ad39f3 100644
--- a/src/mnode/src/mgmtSdb.c
+++ b/src/mnode/src/mgmtSdb.c
@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
+#include "hash.h"
 #include "trpc.h"
 #include "tutil.h"
 #include "tbalance.h"
@@ -83,12 +84,6 @@ typedef struct {
 } SSdbRow;
 
 static SSdbObject tsSdbObj = {0};
-static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash};
-static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash};
-static void  (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash};
-static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData};
-static void  (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash};
-static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData};
 static int sdbWrite(void *param, void *data, int type);
 
 int32_t sdbGetId(void *handle) {
@@ -375,7 +370,11 @@ static SSdbRow *sdbGetRowMeta(void *handle, void *key) {
 
   if (handle == NULL) return NULL;
 
-  pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING) {
+    keySize = strlen((char *)key);
+  }
+  pMeta = taosHashGet(pTable->iHandle, key, keySize);
 
   return pMeta;
 }
@@ -387,13 +386,17 @@ void *sdbGetRow(void *handle, void *key) {
   if (handle == NULL) return NULL;
 
   pthread_mutex_lock(&pTable->mutex);
-  pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key);
+
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING) {
+    keySize = strlen((char *)key);
+  }
+  pMeta = taosHashGet(pTable->iHandle, key, keySize);
+
   if (pMeta) sdbIncRef(pTable, pMeta->row);
   pthread_mutex_unlock(&pTable->mutex);
 
-  if (pMeta == NULL) {
-    return NULL;
-  }
+  if (pMeta == NULL) return NULL;
 
   return pMeta->row;
 }
@@ -404,7 +407,13 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   rowMeta.row = pOper->pObj;
 
   pthread_mutex_lock(&pTable->mutex);
-  (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta);
+
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING) {
+    keySize = strlen((char *)pOper->pObj);
+  }
+  taosHashPut(pTable->iHandle, pOper->pObj, keySize, &rowMeta, sizeof(SSdbRow));
+
   sdbIncRef(pTable, pOper->pObj);
   pTable->numOfRows++;
 
@@ -427,7 +436,13 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
   (*pTable->deleteFp)(pOper);
   
   pthread_mutex_lock(&pTable->mutex);
-  (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
+
+  int32_t keySize = sizeof(int32_t);
+  if (pTable->keyType == SDB_KEY_STRING) {
+    keySize = strlen((char *)pOper->pObj);
+  }
+  taosHashRemove(pTable->iHandle, pOper->pObj, keySize);
+
   pTable->numOfRows--;
   pthread_mutex_unlock(&pTable->mutex);
 
@@ -637,18 +652,26 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
 
 void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
   SSdbTable *pTable = (SSdbTable *)handle;
-  SSdbRow * pMeta;
-
   *ppRow = NULL;
   if (pTable == NULL) return NULL;
 
-  pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
+  SHashMutableIterator *pIter = pNode;
+  if (pIter == NULL) {
+    pIter = taosHashCreateIter(pTable->iHandle);
+  }
+
+  if (!taosHashIterNext(pIter)) {
+    taosHashDestroyIter(pIter);
+    return NULL;
+  }
+
+  SSdbRow *pMeta = taosHashIterGet(pIter);
   if (pMeta == NULL) return NULL;
 
   *ppRow = pMeta->row;
   sdbIncRef(handle, pMeta->row);
 
-  return pNode;
+  return pIter;
 }
 
 void *sdbOpenTable(SSdbTableDesc *pDesc) {
@@ -669,10 +692,12 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   pTable->decodeFp     = pDesc->decodeFp;
   pTable->destroyFp    = pDesc->destroyFp;
   pTable->restoredFp   = pDesc->restoredFp;
-  
-  if (sdbInitIndexFp[pTable->keyType] != NULL) {
-    pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SSdbRow));
+
+  _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
+  if (pTable->keyType == SDB_KEY_STRING) {
+    hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   }
+  pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true);
 
   pthread_mutex_init(&pTable->mutex, NULL);
 
@@ -688,11 +713,10 @@ void sdbCloseTable(void *handle) {
   tsSdbObj.numOfTables--;
   tsSdbObj.tableList[pTable->tableId] = NULL;
 
-  void *pNode = NULL;
-  while (1) {
-    SSdbRow *pMeta;
-    pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
-    if (pMeta == NULL) break;
+  SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle);
+  while (taosHashIterNext(pIter)) {
+    SSdbRow *pMeta = taosHashIterGet(pIter);
+    if (pMeta == NULL) continue;
 
     SSdbOper oper = {
       .pObj = pMeta->row,
@@ -702,9 +726,8 @@ void sdbCloseTable(void *handle) {
     (*pTable->destroyFp)(&oper);
   }
 
-  if (sdbCleanUpIndexFp[pTable->keyType]) {
-    (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle);
-  }
+  taosHashDestroyIter(pIter);
+  taosHashCleanup(pTable->iHandle);
 
   pthread_mutex_destroy(&pTable->mutex);
   
diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c
index 624833c6e4..affd7a1073 100644
--- a/src/mnode/src/mgmtTable.c
+++ b/src/mnode/src/mgmtTable.c
@@ -812,6 +812,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
       dnodeSendMsgToDnode(&ipSet, &rpcMsg);
       mgmtDecVgroupRef(pVgroup);
     }
+    taosHashDestroyIter(pIter);
 
     mgmtDropAllChildTablesInStable(pStable);
   } 
@@ -1288,6 +1289,8 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
       mgmtDecVgroupRef(pVgroup);
     }
 
+    taosHashDestroyIter(pIter);
+
     pVgroupInfo->numOfVgroups = htonl(vgSize);
 
     // one table is done, try the next table

From 5f0a05163d6280dcf151afa1c0cf9e3c17e4fc6f Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Wed, 13 May 2020 11:52:33 +0800
Subject: [PATCH 14/33] [td-225] fix bugs for leastsquares query.

---
 src/client/inc/tscUtil.h                     |  2 +-
 src/client/src/tscFunctionImpl.c             |  6 +++-
 src/client/src/tscSQLParser.c                |  2 ++
 src/client/src/tscUtil.c                     | 27 ++++++++---------
 src/common/inc/tname.h                       |  1 +
 src/inc/taosdef.h                            |  5 ++--
 src/query/src/queryExecutor.c                | 31 ++++++++++----------
 src/tsdb/src/tsdbRead.c                      |  5 ++--
 tests/script/general/compute/leastsquare.sim | 12 ++++----
 9 files changed, 50 insertions(+), 41 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index 71c24501d1..2a37f3fc7c 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -183,7 +183,7 @@ void      tscSqlExprInfoDestroy(SArray* pExprInfo);
 
 SColumn* tscColumnClone(const SColumn* src);
 SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex);
-void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex);
+SArray* tscColumnListClone(const SArray* src, int16_t tableIndex);
 void tscColumnListDestroy(SArray* pColList);
 
 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 6fb8df2444..5675416e6b 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2904,7 +2904,11 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
   
   param[1][2] /= param[1][1];
   
-  sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]);
+  int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE;
+  size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}",
+      param[0][2], param[1][2]);
+  
+  varDataSetLen(pCtx->aOutputBuf, n);
   doFinalizer(pCtx);
 }
 
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 3dbd5d36bd..86a74f0115 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -3792,6 +3792,8 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
     
     tSQLExprDestroy(p1);
     tExprTreeDestroy(&p, NULL);
+    
+    taosArrayDestroy(colList);
   }
 
   pCondExpr->pTagCond = NULL;
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index bea61f12c4..171d40b932 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1209,18 +1209,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) {
   }
 }
 
-void tscColumnListDestroy(SArray* pColumnBaseInfo) {
-  if (pColumnBaseInfo == NULL) {
+void tscColumnListDestroy(SArray* pColumnList) {
+  if (pColumnList == NULL) {
     return;
   }
 
-  size_t num = taosArrayGetSize(pColumnBaseInfo);
+  size_t num = taosArrayGetSize(pColumnList);
   for (int32_t i = 0; i < num; ++i) {
-    SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i);
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
     tscColumnDestroy(pCol);
   }
 
-  taosArrayDestroy(pColumnBaseInfo);
+  taosArrayDestroy(pColumnList);
 }
 
 /*
@@ -1572,7 +1572,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
   
   assert(pQueryInfo->exprList == NULL);
   pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
-  pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
+  pQueryInfo->colList  = taosArrayInit(4, POINTER_BYTES);
 }
 
 int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
@@ -1666,8 +1666,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   assert(pTableMetaInfo != NULL);
 
   if (name != NULL) {
-    assert(strlen(name) <= TSDB_TABLE_ID_LEN);
-    strcpy(pTableMetaInfo->name, name);
+    strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN);
   }
 
   pTableMetaInfo->pTableMeta = pTableMeta;
@@ -1678,10 +1677,9 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
     memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
   }
 
-  if (pTagCols == NULL) {
-    pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
-  } else {
-    pTableMetaInfo->tagColList = taosArrayClone(pTagCols);
+  pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
+  if (pTagCols != NULL) {
+    tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
   }
   
   pQueryInfo->numOfTables += 1;
@@ -1794,7 +1792,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscFreeSqlObj(pNew);
     return NULL;
   }
-
+  
   tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex);
 
   // set the correct query type
@@ -1853,7 +1851,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   if (pPrevSql == NULL) {
     STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name);
-
+    // todo handle error
+    assert(pTableMeta != NULL);
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.
     STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h
index a629128324..810d7f492c 100644
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@@ -23,4 +23,5 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
+
 #endif  // TDENGINE_NAME_H
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 33a0e4f2c6..23436fe6a5 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -32,6 +32,9 @@ extern "C" {
 #define TSKEY int64_t
 #endif
 
+#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX};
+#define TSKEY_INITIAL_VAL    INT64_MIN
+
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
 typedef int32_t VarDataOffsetT;
 typedef int16_t VarDataLenT;
@@ -341,8 +344,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_DBS           100
 #define TSDB_MAX_VGROUPS       1000
 #define TSDB_MAX_SUPER_TABLES  100
-#define TSDB_MAX_NORMAL_TABLES 1000
-#define TSDB_MAX_CHILD_TABLES  100000
 
 #define TSDB_PORT_DNODESHELL 0 
 #define TSDB_PORT_DNODEDNODE 5 
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index fb19d0f50c..4b48d79316 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -507,7 +507,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
     w.ekey = pQuery->window.ekey;
   }
 
-  assert(ts >= w.skey && ts <= w.ekey && w.skey != 0);
+  assert(ts >= w.skey && ts <= w.ekey);
 
   return w;
 }
@@ -624,7 +624,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
     setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
   } else {  // set the current index to be the last unclosed window
     int32_t i = 0;
-    int64_t skey = 0;
+    int64_t skey = TSKEY_INITIAL_VAL;
 
     for (i = 0; i < pWindowResInfo->size; ++i) {
       SWindowResult *pResult = &pWindowResInfo->pResult[i];
@@ -642,7 +642,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
     }
 
     // all windows are closed, set the last one to be the skey
-    if (skey == 0) {
+    if (skey == TSKEY_INITIAL_VAL) {
       assert(i == pWindowResInfo->size);
       pWindowResInfo->curIndex = pWindowResInfo->size - 1;
     } else {
@@ -660,7 +660,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
     qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n);
   }
 
-  assert(pWindowResInfo->prevSKey != 0);
+  assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL);
 }
 
 static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn,
@@ -3080,6 +3080,9 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
       int32_t functId = pQuery->pSelectExpr[j].base.functionId;
 
       SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j];
+      if (pCtx->resultInfo == NULL) {
+        continue; // resultInfo is NULL, means no data checked in previous scan
+      }
 
       if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) ||
           ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) {
@@ -3590,7 +3593,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
   if (pTableQueryInfo->queryRangeSet) {
     pTableQueryInfo->lastKey = key;
   } else {
-//    pQuery->window.skey = key;
     pTableQueryInfo->win.skey = key;
     STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey};
 
@@ -3613,18 +3615,16 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
     pWindowResInfo->startTime = pTableQueryInfo->win.skey;  // windowSKey may be 0 in case of 1970 timestamp
 
-    if (pWindowResInfo->prevSKey == 0) {
-      if (QUERY_IS_ASC_QUERY(pQuery)) {
-        pWindowResInfo->prevSKey = w.skey;
-      } else {
+    if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
+      if (!QUERY_IS_ASC_QUERY(pQuery)) {
         assert(win.ekey == pQuery->window.skey);
-        pWindowResInfo->prevSKey = w.skey;
       }
+      
+      pWindowResInfo->prevSKey = w.skey;
     }
 
     pTableQueryInfo->queryRangeSet = 1;
     pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
-    pTableQueryInfo->win.skey = pTableQueryInfo->win.skey;
   }
 }
 
@@ -4057,10 +4057,11 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
    *    pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is
    *    not valid. otherwise, we only forward pQuery->limit.offset number of points
    */
-  assert(pRuntimeEnv->windowResInfo.prevSKey == 0);
+  assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL);
 
-  TSKEY           skey1, ekey1;
-  STimeWindow     w = {0};
+  TSKEY       skey1, ekey1;
+  STimeWindow w = TSWINDOW_INITIALIZER;
+  
   SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   STableQueryInfo *pTableQueryInfo = pQuery->current;
 
@@ -4730,7 +4731,7 @@ static void doRestoreContext(SQInfo *pQInfo) {
   SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
 
   if (pRuntimeEnv->pTSBuf != NULL) {
-    pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1;
+    SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
   }
 
   switchCtxOrder(pRuntimeEnv);
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index 84615ff8c2..6a5a4b01bb 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -1346,7 +1346,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
     int32_t type = 0;
     int32_t bytes = 0;
     
-    if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
+    if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {  // todo refactor extract method , to queryExecutor to generate tags values
       f1 = (char*) pTable1->name;
       f2 = (char*) pTable2->name;
       type = TSDB_DATA_TYPE_BINARY;
@@ -1354,7 +1354,8 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
     } else {
       STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
       bytes = pCol->bytes;
-  
+      type = pCol->type;
+      
       f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
       f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
     }
diff --git a/tests/script/general/compute/leastsquare.sim b/tests/script/general/compute/leastsquare.sim
index bbadd849cb..5a28e74bff 100644
--- a/tests/script/general/compute/leastsquare.sim
+++ b/tests/script/general/compute/leastsquare.sim
@@ -48,41 +48,41 @@ $tb = $tbPrefix . $i
 
 sql select leastsquares(tbcol, 1, 1) from $tb
 print ===> $data00
-if $data00 != @(1.000000, 1.000000)@ then 
+if $data00 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 
 print =============== step3
 sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m
 print ===> $data00
-if $data00 != @(1.000000, 1.000000)@ then 
+if $data00 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 
 print =============== step4
 sql select leastsquares(tbcol, 1, 1) as b from $tb
 print ===> $data00
-if $data00 != @(1.000000, 1.000000)@ then 
+if $data00 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 
 print =============== step5
 sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m)
 print ===> $data01 
-if $data01 != @(1.000000, 1.000000)@ then 
+if $data01 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 
 sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d)
 print ===> $data01
-if $data01 != @(1.000000, 1.000000)@ then 
+if $data01 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 
 print =============== step6
 sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m)
 print ===> $data01
-if $data01 != @(1.000000, 1.000000)@ then 
+if $data01 != @{slop:1.000000, intercept:1.000000}@ then
   return -1
 endi
 print ===> $rows

From c6aef80a396de85f1bdebbbf157770926769410c Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Wed, 13 May 2020 11:52:53 +0800
Subject: [PATCH 15/33] [td-225] refactor hash function

---
 src/util/inc/hash.h |   6 +-
 src/util/src/hash.c | 482 +++++++++++++++++++++++---------------------
 2 files changed, 250 insertions(+), 238 deletions(-)

diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h
index 1bbc8dcf5c..0a2cbe8baf 100644
--- a/src/util/inc/hash.h
+++ b/src/util/inc/hash.h
@@ -94,7 +94,7 @@ size_t taosHashGetSize(const SHashObj *pHashObj);
  * @param size
  * @return
  */
-int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *data, size_t size);
+int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
 
 /**
  * return the payload data with the specified key
@@ -104,7 +104,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *da
  * @param keyLen
  * @return
  */
-void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen);
+void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
 
 /**
  * remove item with the specified key
@@ -112,7 +112,7 @@ void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen);
  * @param key
  * @param keyLen
  */
-void taosHashRemove(SHashObj *pHashObj, const char *key, size_t keyLen);
+void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen);
 
 /**
  * clean up hash table
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 37e8123170..12f7b43b79 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -17,7 +17,6 @@
 
 #include "hash.h"
 #include "tulog.h"
-#include "ttime.h"
 #include "tutil.h"
 
 static FORCE_INLINE void __wr_lock(void *lock) {
@@ -90,154 +89,65 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
 
 /**
  * inplace update node in hash table
- * @param pHashObj      hash table object
- * @param pNode     data node
+ * @param pHashObj  hash table object
+ * @param pNode     hash data node
  */
-static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) {
-  if (pNode->prev1) {
-    pNode->prev1->next = pNode;
-  }
-
-  if (pNode->next) {
-    (pNode->next)->prev = pNode;
-  }
-}
+static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode);
 
 /**
- * get SHashNode from hashlist, nodes from trash are not included.
+ * Get SHashNode from hashlist, nodes from trash are not included.
  * @param pHashObj  Cache objection
  * @param key       key for hash
  * @param keyLen    key length
+ * @param hashVal   hash value by hash function
  * @return
  */
-static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const char *key, uint32_t keyLen, uint32_t *hashVal) {
-  uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
-
-  int32_t     slot = HASH_INDEX(hash, pHashObj->capacity);
-  SHashEntry *pEntry = pHashObj->hashList[slot];
-
-  SHashNode *pNode = pEntry->next;
-  while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
-      break;
-    }
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
-  }
-
-  // return the calculated hash value, to avoid calculating it again in other functions
-  if (hashVal != NULL) {
-    *hashVal = hash;
-  }
-
-  return pNode;
-}
+static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal);
 
 /**
- * resize the hash list if the threshold is reached
+ * Resize the hash list if the threshold is reached
  *
  * @param pHashObj
  */
-static void taosHashTableResize(SHashObj *pHashObj) {
-  if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) {
-    return;
-  }
-
-  // double the original capacity
-  SHashNode *pNode = NULL;
-  SHashNode *pNext = NULL;
-
-  int32_t newSize = pHashObj->capacity << 1u;
-  if (newSize > HASH_MAX_CAPACITY) {
-//    uTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached",
-//           pHashObj->capacity, HASH_MAX_CAPACITY);
-    return;
-  }
-
-//  int64_t st = taosGetTimestampUs();
-
-  SHashEntry **pNewEntry = realloc(pHashObj->hashList, sizeof(SHashEntry *) * newSize);
-  if (pNewEntry == NULL) {
-//    uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
-    return;
-  }
-
-  pHashObj->hashList = pNewEntry;
-  for (int32_t i = pHashObj->capacity; i < newSize; ++i) {
-    pHashObj->hashList[i] = calloc(1, sizeof(SHashEntry));
-  }
-
-  pHashObj->capacity = newSize;
-
-  for (int32_t i = 0; i < pHashObj->capacity; ++i) {
-    SHashEntry *pEntry = pHashObj->hashList[i];
-
-    pNode = pEntry->next;
-    if (pNode != NULL) {
-      assert(pNode->prev1 == pEntry && pEntry->num > 0);
-    }
-
-    while (pNode) {
-      int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
-      if (j == i) {  // this key resides in the same slot, no need to relocate it
-        pNode = pNode->next;
-      } else {
-        pNext = pNode->next;
-
-        // remove from current slot
-        assert(pNode->prev1 != NULL);
-
-        if (pNode->prev1 == pEntry) {  // first node of the overflow linked list
-          pEntry->next = pNode->next;
-        } else {
-          pNode->prev->next = pNode->next;
-        }
-
-        pEntry->num--;
-        assert(pEntry->num >= 0);
-
-        if (pNode->next != NULL) {
-          (pNode->next)->prev = pNode->prev;
-        }
-
-        // added into new slot
-        pNode->next = NULL;
-        pNode->prev1 = NULL;
-
-        SHashEntry *pNewIndexEntry = pHashObj->hashList[j];
-
-        if (pNewIndexEntry->next != NULL) {
-          assert(pNewIndexEntry->next->prev1 == pNewIndexEntry);
-
-          pNewIndexEntry->next->prev = pNode;
-        }
-
-        pNode->next = pNewIndexEntry->next;
-        pNode->prev1 = pNewIndexEntry;
-
-        pNewIndexEntry->next = pNode;
-        pNewIndexEntry->num++;
-
-        // continue
-        pNode = pNext;
-      }
-    }
-  }
-
-//  int64_t et = taosGetTimestampUs();
-//  uTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity,
-//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
-}
+static void taosHashTableResize(SHashObj *pHashObj);
 
 /**
- * @param capacity maximum slots available for hash elements
- * @param fn       hash function
+ * @param key      key of object for hash, usually a null-terminated string
+ * @param keyLen   length of key
+ * @param pData    actually data. Requires a consecutive memory block, no pointer is allowed in pData.
+ *                 Pointer copy causes memory access error.
+ * @param dsize    size of data
+ * @return         SHashNode
+ */
+static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal);
+
+/**
+ * Update the hash node
+ *
+ * @param pNode   hash node
+ * @param key     key for generate hash value
+ * @param keyLen  key length
+ * @param pData   actual data
+ * @param dsize   size of actual data
+ * @return        hash node
+ */
+static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize);
+
+/**
+ * insert the hash node at the front of the linked list
+ *
+ * @param pHashObj
+ * @param pNode
+ */
+static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode);
+
+/**
+ * Get the next element in hash table for iterator
+ * @param pIter
  * @return
  */
+static SHashNode *getNextHashNode(SHashMutableIterator *pIter);
+
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) {
   if (capacity == 0 || fn == NULL) {
     return NULL;
@@ -285,79 +195,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) {
   return pHashObj;
 }
 
-/**
- * @param key      key of object for hash, usually a null-terminated string
- * @param keyLen   length of key
- * @param pData    actually data. required a consecutive memory block, no pointer is allowed
- *                 in pData. Pointer copy causes memory access error.
- * @param size     size of block
- * @return         SHashNode
- */
-static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize,
-                                   uint32_t hashVal) {
-  size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1;  // one extra byte for null
-
-  SHashNode *pNewNode = calloc(1, totalSize);
-  if (pNewNode == NULL) {
-    uError("failed to allocate memory, reason:%s", strerror(errno));
-    return NULL;
-  }
-
-  memcpy(pNewNode->data, pData, dataSize);
-
-  pNewNode->key = pNewNode->data + dataSize;
-  memcpy(pNewNode->key, key, keyLen);
-  pNewNode->keyLen = keyLen;
-
-  pNewNode->hashVal = hashVal;
-
-  return pNewNode;
-}
-
-static SHashNode *doUpdateHashNode(SHashNode *pNode, const char *key, size_t keyLen, const char *pData,
-                                   size_t dataSize) {
-  size_t size = dataSize + sizeof(SHashNode) + keyLen;
-
-  SHashNode *pNewNode = (SHashNode *)realloc(pNode, size);
-  if (pNewNode == NULL) {
-    return NULL;
-  }
-
-  memcpy(pNewNode->data, pData, dataSize);
-
-  pNewNode->key = pNewNode->data + dataSize;
-
-  assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen);
-
-  memcpy(pNewNode->key, key, keyLen);
-  return pNewNode;
-}
-
-/**
- * insert the hash node at the front of the linked list
- *
- * @param pHashObj
- * @param pNode
- */
-static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) {
-  assert(pNode != NULL);
-
-  int32_t     index = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
-  SHashEntry *pEntry = pHashObj->hashList[index];
-
-  pNode->next = pEntry->next;
-
-  if (pEntry->next) {
-    pEntry->next->prev = pNode;
-  }
-
-  pEntry->next = pNode;
-  pNode->prev1 = pEntry;
-
-  pEntry->num++;
-  pHashObj->size++;
-}
-
 size_t taosHashGetSize(const SHashObj *pHashObj) {
   if (pHashObj == NULL) {
     return 0;
@@ -366,12 +203,7 @@ size_t taosHashGetSize(const SHashObj *pHashObj) {
   return pHashObj->size;
 }
 
-/**
- * add data node into hash table
- * @param pHashObj    hash object
- * @param pNode   hash node
- */
-int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *data, size_t size) {
+int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
   __wr_lock(pHashObj->lock);
 
   uint32_t   hashVal = 0;
@@ -402,7 +234,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *da
   return 0;
 }
 
-void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen) {
+void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
   __rd_lock(pHashObj->lock);
 
   uint32_t   hashVal = 0;
@@ -419,12 +251,7 @@ void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen) {
   }
 }
 
-/**
- * remove node in hash list
- * @param pHashObj
- * @param pNode
- */
-void taosHashRemove(SHashObj *pHashObj, const char *key, size_t keyLen) {
+void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
   __wr_lock(pHashObj->lock);
 
   uint32_t   val = 0;
@@ -518,23 +345,6 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
   return pIter;
 }
 
-static SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
-  assert(pIter != NULL);
-
-  pIter->entryIndex++;
-  while (pIter->entryIndex < pIter->pHashObj->capacity) {
-    SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
-    if (pEntry->next == NULL) {
-      pIter->entryIndex++;
-      continue;
-    }
-
-    return pEntry->next;
-  }
-
-  return NULL;
-}
-
 bool taosHashIterNext(SHashMutableIterator *pIter) {
   if (pIter == NULL) {
     return false;
@@ -617,3 +427,205 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
 
   return num;
 }
+
+void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) {
+  if (pNode->prev1) {
+    pNode->prev1->next = pNode;
+  }
+  
+  if (pNode->next) {
+    (pNode->next)->prev = pNode;
+  }
+}
+
+SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) {
+  uint32_t hash = (*pHashObj->hashFp)(key, keyLen);
+  
+  int32_t     slot = HASH_INDEX(hash, pHashObj->capacity);
+  SHashEntry *pEntry = pHashObj->hashList[slot];
+  
+  SHashNode *pNode = pEntry->next;
+  while (pNode) {
+    if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+      break;
+    }
+    
+    pNode = pNode->next;
+  }
+  
+  if (pNode) {
+    assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
+  }
+  
+  // return the calculated hash value, to avoid calculating it again in other functions
+  if (hashVal != NULL) {
+    *hashVal = hash;
+  }
+  
+  return pNode;
+}
+
+void taosHashTableResize(SHashObj *pHashObj) {
+  if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) {
+    return;
+  }
+  
+  // double the original capacity
+  SHashNode *pNode = NULL;
+  SHashNode *pNext = NULL;
+  
+  int32_t newSize = pHashObj->capacity << 1u;
+  if (newSize > HASH_MAX_CAPACITY) {
+//    uTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached",
+//           pHashObj->capacity, HASH_MAX_CAPACITY);
+    return;
+  }
+
+//  int64_t st = taosGetTimestampUs();
+  
+  SHashEntry **pNewEntry = realloc(pHashObj->hashList, sizeof(SHashEntry *) * newSize);
+  if (pNewEntry == NULL) {
+//    uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
+    return;
+  }
+  
+  pHashObj->hashList = pNewEntry;
+  for (int32_t i = pHashObj->capacity; i < newSize; ++i) {
+    pHashObj->hashList[i] = calloc(1, sizeof(SHashEntry));
+  }
+  
+  pHashObj->capacity = newSize;
+  
+  for (int32_t i = 0; i < pHashObj->capacity; ++i) {
+    SHashEntry *pEntry = pHashObj->hashList[i];
+    
+    pNode = pEntry->next;
+    if (pNode != NULL) {
+      assert(pNode->prev1 == pEntry && pEntry->num > 0);
+    }
+    
+    while (pNode) {
+      int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
+      if (j == i) {  // this key resides in the same slot, no need to relocate it
+        pNode = pNode->next;
+      } else {
+        pNext = pNode->next;
+        
+        // remove from current slot
+        assert(pNode->prev1 != NULL);
+        
+        if (pNode->prev1 == pEntry) {  // first node of the overflow linked list
+          pEntry->next = pNode->next;
+        } else {
+          pNode->prev->next = pNode->next;
+        }
+        
+        pEntry->num--;
+        assert(pEntry->num >= 0);
+        
+        if (pNode->next != NULL) {
+          (pNode->next)->prev = pNode->prev;
+        }
+        
+        // added into new slot
+        pNode->next = NULL;
+        pNode->prev1 = NULL;
+        
+        SHashEntry *pNewIndexEntry = pHashObj->hashList[j];
+        
+        if (pNewIndexEntry->next != NULL) {
+          assert(pNewIndexEntry->next->prev1 == pNewIndexEntry);
+          
+          pNewIndexEntry->next->prev = pNode;
+        }
+        
+        pNode->next = pNewIndexEntry->next;
+        pNode->prev1 = pNewIndexEntry;
+        
+        pNewIndexEntry->next = pNode;
+        pNewIndexEntry->num++;
+        
+        // continue
+        pNode = pNext;
+      }
+    }
+  }
+
+//  int64_t et = taosGetTimestampUs();
+//  uTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity,
+//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
+}
+
+SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
+  size_t totalSize = dsize + sizeof(SHashNode) + keyLen + 1;  // one extra byte for null
+  
+  SHashNode *pNewNode = calloc(1, totalSize);
+  if (pNewNode == NULL) {
+    uError("failed to allocate memory, reason:%s", strerror(errno));
+    return NULL;
+  }
+  
+  memcpy(pNewNode->data, pData, dsize);
+  
+  pNewNode->key = pNewNode->data + dsize;
+  memcpy(pNewNode->key, key, keyLen);
+  pNewNode->keyLen = keyLen;
+  
+  pNewNode->hashVal = hashVal;
+  
+  return pNewNode;
+}
+
+SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize) {
+  size_t size = dsize + sizeof(SHashNode) + keyLen;
+  
+  SHashNode *pNewNode = (SHashNode *)realloc(pNode, size);
+  if (pNewNode == NULL) {
+    return NULL;
+  }
+  
+  memcpy(pNewNode->data, pData, dsize);
+  
+  pNewNode->key = pNewNode->data + dsize;
+  
+  assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen);
+  
+  memcpy(pNewNode->key, key, keyLen);
+  return pNewNode;
+}
+
+void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) {
+  assert(pNode != NULL);
+  
+  int32_t     index = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
+  SHashEntry *pEntry = pHashObj->hashList[index];
+  
+  pNode->next = pEntry->next;
+  
+  if (pEntry->next) {
+    pEntry->next->prev = pNode;
+  }
+  
+  pEntry->next = pNode;
+  pNode->prev1 = pEntry;
+  
+  pEntry->num++;
+  pHashObj->size++;
+}
+
+SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
+  assert(pIter != NULL);
+  
+  pIter->entryIndex++;
+  while (pIter->entryIndex < pIter->pHashObj->capacity) {
+    SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
+    if (pEntry->next == NULL) {
+      pIter->entryIndex++;
+      continue;
+    }
+    
+    return pEntry->next;
+  }
+  
+  return NULL;
+}

From 4328ac123cd3542dec7d54a7629a637c7d28d3ec Mon Sep 17 00:00:00 2001
From: jtao1735 <jhtao@taosdata.com>
Date: Wed, 13 May 2020 04:04:55 +0000
Subject: [PATCH 16/33] tsdb notify the vnode about it status changes

---
 src/inc/tsdb.h            |  6 ++++--
 src/tsdb/src/tsdbMain.c   |  8 +++++---
 src/vnode/src/vnodeMain.c | 19 ++++++++++++-------
 3 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 4468ee4262..32e3541692 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -34,12 +34,14 @@ extern "C" {
 
 #define TSDB_INVALID_SUPER_TABLE_ID -1
 
+#define TSDB_STATUS_COMMIT_START 1
+#define TSDB_STATUS_COMMIT_OVER  2
+
 // --------- TSDB APPLICATION HANDLE DEFINITION
 typedef struct {
-  // WAL handle
   void *appH;
   void *cqH;
-  int (*walCallBack)(void *);
+  int (*notifyStatus)(void *, int status);
   int (*eventCallBack)(void *);
 } STsdbAppH;
 
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index a353f54353..df09b26c71 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -330,7 +330,7 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
 int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
-  if (pRepo->appH.walCallBack) pRepo->appH.walCallBack(pRepo->appH.appH);
+  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
   
   tsdbLockRepo(repo);
   if (pRepo->commit) {
@@ -942,7 +942,6 @@ static void tsdbFreeMemTable(SMemTable *pMemTable) {
 
 // Commit to file
 static void *tsdbCommitData(void *arg) {
-  printf("Starting to commit....\n");
   STsdbRepo * pRepo = (STsdbRepo *)arg;
   STsdbMeta * pMeta = pRepo->tsdbMeta;
   STsdbCache *pCache = pRepo->tsdbCache;
@@ -951,6 +950,8 @@ static void *tsdbCommitData(void *arg) {
   SRWHelper   whelper = {0};
   if (pCache->imem == NULL) return NULL;
 
+  tsdbPrint("vgId: %d, starting to commit....", pRepo->config.tsdbId);
+
   // Create the iterator to read from cache
   SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables);
   if (iters == NULL) {
@@ -974,6 +975,7 @@ static void *tsdbCommitData(void *arg) {
 
   // Do retention actions
   tsdbFitRetention(pRepo);
+  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
 
 _exit:
   tdFreeDataCols(pDataCols);
@@ -1176,4 +1178,4 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *
   magic = *size;
 
   return magic;
-}
\ No newline at end of file
+}
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 8d35be9640..ec5ac40d04 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -33,12 +33,11 @@ static int32_t  tsOpennedVnodes;
 static void    *tsDnodeVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
 static void     vnodeBuildVloadMsg(char *pNode, void * param);
-static int      vnodeWalCallback(void *arg);
 static int32_t  vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
 static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
 static int32_t  vnodeSaveVersion(SVnodeObj *pVnode);
 static bool     vnodeReadVersion(SVnodeObj *pVnode);
-static int      vnodeWalCallback(void *arg);
+static int      vnodeProcessTsdbStatus(void *arg, int status);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size);
 static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
@@ -206,7 +205,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   STsdbAppH appH = {0};
   appH.appH = (void *)pVnode;
-  appH.walCallBack = vnodeWalCallback;
+  appH.notifyStatus = vnodeProcessTsdbStatus;
   appH.cqH = pVnode->cq;
 
   sprintf(temp, "%s/tsdb", rootDir);
@@ -374,14 +373,20 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
   walClose(pVnode->wal);
   pVnode->wal = NULL;
 
-  vnodeSaveVersion(pVnode);
   vnodeRelease(pVnode);
 }
 
 // TODO: this is a simple implement
-static int vnodeWalCallback(void *arg) {
+static int vnodeProcessTsdbStatus(void *arg, int status) {
   SVnodeObj *pVnode = arg;
-  return walRenew(pVnode->wal);
+
+  if (status == TSDB_STATUS_COMMIT_START)
+    return walRenew(pVnode->wal);
+
+  if (status == TSDB_STATUS_COMMIT_OVER)
+    return vnodeSaveVersion(pVnode);
+
+  return 0; 
 }
 
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
@@ -414,7 +419,7 @@ static void vnodeNotifyFileSynced(void *ahandle) {
   tsdbCloseRepo(pVnode->tsdb);
   STsdbAppH appH = {0};
   appH.appH = (void *)pVnode;
-  appH.walCallBack = vnodeWalCallback;
+  appH.notifyStatus = vnodeProcessTsdbStatus;
   appH.cqH = pVnode->cq;
   pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
 }

From 4ee2cb13aace6f74c5c89bee2cbc255057cbbec1 Mon Sep 17 00:00:00 2001
From: jtao1735 <jhtao@taosdata.com>
Date: Wed, 13 May 2020 04:28:18 +0000
Subject: [PATCH 17/33] avoid a potential bug

---
 src/tsdb/src/tsdbMain.c   |  1 +
 src/vnode/inc/vnodeInt.h  |  1 +
 src/vnode/src/vnodeMain.c | 10 ++++++----
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index df09b26c71..f06230c16f 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -284,6 +284,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   pRepo->tsdbCache->curBlock = NULL;
   tsdbUnLockRepo(repo);
 
+  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
   tsdbCommitData((void *)repo);
 
   tsdbCloseFileH(pRepo->tsdbFileH);
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index b2dd69df37..d41957d8f4 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -38,6 +38,7 @@ typedef struct {
   int          status; 
   int8_t       role;   
   int64_t      version;
+  int64_t      savedVersion;
   void        *wqueue;
   void        *rqueue;
   void        *wal;
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index ec5ac40d04..2e55b6f93a 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -380,8 +380,10 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
 static int vnodeProcessTsdbStatus(void *arg, int status) {
   SVnodeObj *pVnode = arg;
 
-  if (status == TSDB_STATUS_COMMIT_START)
+  if (status == TSDB_STATUS_COMMIT_START) {
+    pVnode->savedVersion = pVnode->version; 
     return walRenew(pVnode->wal);
+  }
 
   if (status == TSDB_STATUS_COMMIT_OVER)
     return vnodeSaveVersion(pVnode);
@@ -690,14 +692,14 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"version\": %" PRId64 "\n", pVnode->version);
+  len += snprintf(content + len, maxLen - len, "  \"version\": %" PRId64 "\n", pVnode->savedVersion);
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
   fclose(fp);
   free(content);
 
-  vPrint("vgId:%d, save vnode version:%" PRId64 " successed", pVnode->vgId, pVnode->version);
+  vPrint("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->savedVersion);
 
   return 0;
 }
@@ -739,7 +741,7 @@ static bool vnodeReadVersion(SVnodeObj *pVnode) {
 
   ret = true;
 
-  vPrint("vgId:%d, read vnode version successed, version:%%" PRId64, pVnode->vgId, pVnode->version);
+  vPrint("vgId:%d, read vnode version succeed, version:%" PRId64, pVnode->vgId, pVnode->version);
 
 PARSE_OVER:
   free(content);

From a478499c2d466427d974e96432d7860b74d91b6f Mon Sep 17 00:00:00 2001
From: lihui <huili@taosdata.com>
Date: Wed, 13 May 2020 13:31:42 +0800
Subject: [PATCH 18/33] [TD-161]

---
 src/client/src/tscAsync.c | 13 +++++++------
 src/client/src/tscUtil.c  | 17 +++++++++++++----
 src/util/src/tcache.c     |  2 ++
 src/vnode/src/vnodeMain.c |  5 ++++-
 4 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 9f207936df..14f242ddd4 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -425,6 +425,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   
     if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+      
+      code = tscGetTableMeta(pSql, pTableMetaInfo);
+      pRes->code = code;
+
+      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+      
       assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL);
 
       SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
@@ -433,12 +439,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0);
 
-      tscTrace("%p get metricMeta during super table query successfully", pSql);
-      
-      code = tscGetTableMeta(pSql, pTableMetaInfo);
-      pRes->code = code;
-
-      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+      tscTrace("%p get metricMeta during super table query successfully", pSql);      
 
       code = tscGetSTableVgroupInfo(pSql, 0);
       pRes->code = code;
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 171d40b932..42c537eb0c 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -391,9 +391,11 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   }
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-  pthread_mutex_lock(&pObj->mutex);
-  tfree(pSql->sqlstr);
-  pthread_mutex_unlock(&pObj->mutex);
+  if (pObj->signature == pObj) {
+    pthread_mutex_lock(&pObj->mutex);
+    tfree(pSql->sqlstr);
+    pthread_mutex_unlock(&pObj->mutex);
+  }
   
   tscFreeSqlResult(pSql);
   
@@ -1864,7 +1866,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
   }
 
-  assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1);
+  if (pFinalInfo->pTableMeta == NULL) {
+    tscError("%p new subquery failed for get pMeterMeta is NULL from cache", pSql);
+    tscFreeSqlObj(pNew);
+    return NULL;
+  }
+  
+  assert(pNewQueryInfo->numOfTables == 1);
+  
   if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
     assert(pFinalInfo->vgroupList != NULL);
   }
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 02a5b93141..c715d06a70 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -439,6 +439,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz
       
       uTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes",
              key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize);
+    } else {
+      uError("key:%s failed to added into cache, out of memory", key);
     }
   } else {  // old data exists, update the node
     pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L);
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 8d35be9640..6e7976ff44 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -265,7 +265,10 @@ void vnodeRelease(void *pVnodeRaw) {
   int32_t    vgId = pVnode->vgId;
 
   int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  assert(refCount >= 0);
+  if (ref < 0) {
+    vTrace("vgId:%d, other thread is releasing vnode, refCount:%d", vgId, refCount);
+    return; // avoid two threads dec ref count
+  }
 
   if (refCount > 0) {
     vTrace("vgId:%d, release vnode, refCount:%d", vgId, refCount);

From 55e68f2fc8e6bf720e37f7e221bcbde0362ff4a7 Mon Sep 17 00:00:00 2001
From: lihui <huili@taosdata.com>
Date: Wed, 13 May 2020 13:38:31 +0800
Subject: [PATCH 19/33] [TD-161]

---
 src/vnode/src/vnodeMain.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 6e7976ff44..77afb7bf29 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -265,7 +265,7 @@ void vnodeRelease(void *pVnodeRaw) {
   int32_t    vgId = pVnode->vgId;
 
   int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  if (ref < 0) {
+  if (refCount < 0) {
     vTrace("vgId:%d, other thread is releasing vnode, refCount:%d", vgId, refCount);
     return; // avoid two threads dec ref count
   }

From 054d60fd44d26a703e1f83a70697c2eeaead84b5 Mon Sep 17 00:00:00 2001
From: lihui <huili@taosdata.com>
Date: Wed, 13 May 2020 14:16:26 +0800
Subject: [PATCH 20/33] [TD-161]

---
 src/client/src/tscAsync.c | 9 ++++-----
 src/vnode/src/vnodeMain.c | 5 +----
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 14f242ddd4..813c456152 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -425,11 +425,10 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   
     if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-      
-      code = tscGetTableMeta(pSql, pTableMetaInfo);
-      pRes->code = code;
-
-      if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+      if (pTableMetaInfo->pTableMeta == NULL){
+        code = tscGetTableMeta(pSql, pTableMetaInfo);
+        assert(code == TSDB_CODE_SUCCESS);      
+      }     
       
       assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL);
 
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 77afb7bf29..8d35be9640 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -265,10 +265,7 @@ void vnodeRelease(void *pVnodeRaw) {
   int32_t    vgId = pVnode->vgId;
 
   int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  if (refCount < 0) {
-    vTrace("vgId:%d, other thread is releasing vnode, refCount:%d", vgId, refCount);
-    return; // avoid two threads dec ref count
-  }
+  assert(refCount >= 0);
 
   if (refCount > 0) {
     vTrace("vgId:%d, release vnode, refCount:%d", vgId, refCount);

From ff9bdcfdb0345e1fdf7171612db89be0d43a5d40 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 14:54:05 +0800
Subject: [PATCH 21/33] [TD-149] replace sdb hash

---
 src/mnode/inc/mgmtAcct.h    |  2 +-
 src/mnode/inc/mgmtDb.h      |  2 +-
 src/mnode/inc/mgmtDef.h     |  2 +-
 src/mnode/inc/mgmtDnode.h   |  2 +-
 src/mnode/inc/mgmtMnode.h   |  2 +-
 src/mnode/inc/mgmtSdb.h     |  3 +-
 src/mnode/inc/mgmtTable.h   |  4 +-
 src/mnode/inc/mgmtUser.h    |  2 +-
 src/mnode/inc/mgmtVgroup.h  |  2 +-
 src/mnode/src/mgmtAcct.c    |  4 +-
 src/mnode/src/mgmtBalance.c |  7 +++-
 src/mnode/src/mgmtDb.c      | 17 ++++----
 src/mnode/src/mgmtDnode.c   | 33 ++++++++-------
 src/mnode/src/mgmtMnode.c   | 21 ++++++----
 src/mnode/src/mgmtProfile.c | 18 ++++----
 src/mnode/src/mgmtSdb.c     | 16 +++++--
 src/mnode/src/mgmtShell.c   | 13 ++++--
 src/mnode/src/mgmtTable.c   | 83 ++++++++++++++-----------------------
 src/mnode/src/mgmtUser.c    | 15 ++++---
 src/mnode/src/mgmtVgroup.c  | 30 +++++++-------
 src/util/inc/tcache.h       |  2 +
 src/util/src/tcache.c       | 13 ++++--
 22 files changed, 155 insertions(+), 138 deletions(-)

diff --git a/src/mnode/inc/mgmtAcct.h b/src/mnode/inc/mgmtAcct.h
index 0c0bb8d05c..136c28f2ae 100644
--- a/src/mnode/inc/mgmtAcct.h
+++ b/src/mnode/inc/mgmtAcct.h
@@ -25,7 +25,7 @@ extern "C" {
 int32_t mgmtInitAccts();
 void    mgmtCleanUpAccts();
 void *  mgmtGetAcct(char *acctName);
-void *  mgmtGetNextAcct(void *pNode, SAcctObj **pAcct);
+void *  mgmtGetNextAcct(void *pIter, SAcctObj **pAcct);
 void    mgmtIncAcctRef(SAcctObj *pAcct);
 void    mgmtDecAcctRef(SAcctObj *pAcct);
 void    mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);
diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h
index b00a2bdf3d..4ad46eee7b 100644
--- a/src/mnode/inc/mgmtDb.h
+++ b/src/mnode/inc/mgmtDb.h
@@ -32,7 +32,7 @@ int32_t mgmtInitDbs();
 void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
-void *  mgmtGetNextDb(void *pNode, SDbObj **pDb);
+void *  mgmtGetNextDb(void *pIter, SDbObj **pDb);
 void    mgmtIncDbRef(SDbObj *pDb);
 void    mgmtDecDbRef(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);
diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h
index 3ac2efb83b..f33531583a 100644
--- a/src/mnode/inc/mgmtDef.h
+++ b/src/mnode/inc/mgmtDef.h
@@ -223,7 +223,7 @@ typedef struct SAcctObj {
 typedef struct {
   int8_t   type;
   char     db[TSDB_DB_NAME_LEN + 1];
-  void *   pNode;
+  void *   pIter;
   int16_t  numOfColumns;
   int32_t  rowSize;
   int32_t  numOfRows;
diff --git a/src/mnode/inc/mgmtDnode.h b/src/mnode/inc/mgmtDnode.h
index 1d7116c6c0..c9e062255b 100644
--- a/src/mnode/inc/mgmtDnode.h
+++ b/src/mnode/inc/mgmtDnode.h
@@ -34,7 +34,7 @@ char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
 void    mgmtMonitorDnodeModule();
 
 int32_t mgmtGetDnodesNum();
-void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
+void *  mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mgmtIncDnodeRef(SDnodeObj *pDnode);
 void    mgmtDecDnodeRef(SDnodeObj *pDnode);
 void *  mgmtGetDnode(int32_t dnodeId);
diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h
index cb1d009c8c..b9a135346d 100644
--- a/src/mnode/inc/mgmtMnode.h
+++ b/src/mnode/inc/mgmtMnode.h
@@ -37,7 +37,7 @@ void    mgmtDropMnodeLocal(int32_t dnodeId);
 
 void *  mgmtGetMnode(int32_t mnodeId);
 int32_t mgmtGetMnodesNum();
-void *  mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
+void *  mgmtGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
 void    mgmtIncMnodeRef(struct SMnodeObj *pMnode);
 void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 
diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h
index 2b96eb4005..a97b1219a9 100644
--- a/src/mnode/inc/mgmtSdb.h
+++ b/src/mnode/inc/mgmtSdb.h
@@ -80,7 +80,8 @@ int32_t sdbDeleteRow(SSdbOper *pOper);
 int32_t sdbUpdateRow(SSdbOper *pOper);
 
 void    *sdbGetRow(void *handle, void *key);
-void    *sdbFetchRow(void *handle, void *pNode, void **ppRow);
+void    *sdbFetchRow(void *handle, void *pIter, void **ppRow);
+void     sdbFreeIter(void *pIter);
 void     sdbIncRef(void *thandle, void *pRow);
 void     sdbDecRef(void *thandle, void *pRow);
 int64_t  sdbGetNumOfRows(void *handle);
diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h
index 9c4aa4a2a5..84220c9cf8 100644
--- a/src/mnode/inc/mgmtTable.h
+++ b/src/mnode/inc/mgmtTable.h
@@ -27,8 +27,8 @@ void    mgmtCleanUpTables();
 void *  mgmtGetTable(char *tableId);
 void    mgmtIncTableRef(void *pTable);
 void    mgmtDecTableRef(void *pTable);
-void *  mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable);
-void *  mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable);
+void *  mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
+void *  mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
 void    mgmtDropAllChildTables(SDbObj *pDropDb);
 void    mgmtDropAllSuperTables(SDbObj *pDropDb);
 
diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h
index 0a21b1f704..2edd71f3e7 100644
--- a/src/mnode/inc/mgmtUser.h
+++ b/src/mnode/inc/mgmtUser.h
@@ -24,7 +24,7 @@ extern "C" {
 int32_t   mgmtInitUsers();
 void      mgmtCleanUpUsers();
 SUserObj *mgmtGetUser(char *name);
-void *    mgmtGetNextUser(void *pNode, SUserObj **pUser);
+void *    mgmtGetNextUser(void *pIter, SUserObj **pUser);
 void      mgmtIncUserRef(SUserObj *pUser);
 void      mgmtDecUserRef(SUserObj *pUser);
 SUserObj *mgmtGetUserFromConn(void *pConn);
diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h
index 21a2c9b896..7acf7112a4 100644
--- a/src/mnode/inc/mgmtVgroup.h
+++ b/src/mnode/inc/mgmtVgroup.h
@@ -35,7 +35,7 @@ void    mgmtDecVgroupRef(SVgObj *pVgroup);
 void    mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
 void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 
-void *  mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
+void *  mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);
 void    mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
 
diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c
index 1cacec96ee..27beff8944 100644
--- a/src/mnode/src/mgmtAcct.c
+++ b/src/mnode/src/mgmtAcct.c
@@ -126,8 +126,8 @@ void *mgmtGetAcct(char *name) {
   return sdbGetRow(tsAcctSdb, name);
 }
 
-void *mgmtGetNextAcct(void *pNode, SAcctObj **pAcct) {
-  return sdbFetchRow(tsAcctSdb, pNode, (void **)pAcct); 
+void *mgmtGetNextAcct(void *pIter, SAcctObj **pAcct) {
+  return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); 
 }
 
 void mgmtIncAcctRef(SAcctObj *pAcct) {
diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c
index ff5b4008bb..f8410a207e 100644
--- a/src/mnode/src/mgmtBalance.c
+++ b/src/mnode/src/mgmtBalance.c
@@ -22,6 +22,7 @@
 #include "mgmtInt.h"
 #include "mgmtMnode.h"
 #include "mgmtDnode.h"
+#include "mgmtSdb.h"
 #include "mgmtVgroup.h"
 
 #ifndef _SYNC
@@ -33,13 +34,13 @@ void    balanceUpdateMgmt() {}
 void    balanceReset() {}
 
 int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-  void *     pNode = NULL;
+  void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
   float      vnodeUsage = 1000.0;
 
   while (1) {
-    pNode = mgmtGetNextDnode(pNode, &pDnode);
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
 
     if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) {
@@ -55,6 +56,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
     mgmtDecDnodeRef(pDnode);
   }
 
+  sdbFreeIter(pIter);
+
   if (pSelDnode == NULL) {
     mError("failed to alloc vnode to vgroup");
     return TSDB_CODE_NO_ENOUGH_DNODES;
diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c
index 2f17df92fa..3f43d4bf89 100644
--- a/src/mnode/src/mgmtDb.c
+++ b/src/mnode/src/mgmtDb.c
@@ -156,8 +156,8 @@ int32_t mgmtInitDbs() {
   return 0;
 }
 
-void *mgmtGetNextDb(void *pNode, SDbObj **pDb) {
-  return sdbFetchRow(tsDbSdb, pNode, (void **)pDb);
+void *mgmtGetNextDb(void *pIter, SDbObj **pDb) {
+  return sdbFetchRow(tsDbSdb, pIter, (void **)pDb);
 }
 
 SDbObj *mgmtGetDb(char *db) {
@@ -583,7 +583,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
   if (pUser == NULL) return 0;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb);
+    pShow->pIter = mgmtGetNextDb(pShow->pIter, &pDb);
     if (pDb == NULL) break;
 
     cols = 0;
@@ -865,14 +865,15 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     }
   }
 
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup = NULL;
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;   
     mgmtSendCreateVgroupMsg(pVgroup, NULL);
     mgmtDecVgroupRef(pVgroup);
   }
+  sdbFreeIter(pIter);
 
   if (oldReplica != pDb->cfg.replications) {
     balanceNotify();
@@ -983,12 +984,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
 void  mgmtDropAllDbs(SAcctObj *pAcct)  {
   int32_t numOfDbs = 0;
   SDbObj *pDb = NULL;
-  void *  pNode = NULL;
+  void *  pIter = NULL;
 
   mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user);
 
   while (1) {
-    pNode = mgmtGetNextDb(pNode, &pDb);
+    pIter = mgmtGetNextDb(pIter, &pDb);
     if (pDb == NULL) break;
 
     if (pDb->pAcct == pAcct) {
@@ -1005,5 +1006,7 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
     mgmtDecDbRef(pDb);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs);
 }
diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c
index d60dae32ae..463f5dde47 100644
--- a/src/mnode/src/mgmtDnode.c
+++ b/src/mnode/src/mgmtDnode.c
@@ -170,8 +170,8 @@ void mgmtCleanupDnodes() {
   sdbCloseTable(tsDnodeSdb);
 }
 
-void *mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { 
-  return sdbFetchRow(tsDnodeSdb, pNode, (void **)pDnode); 
+void *mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode) { 
+  return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); 
 }
 
 int32_t mgmtGetDnodesNum() {
@@ -184,17 +184,20 @@ void *mgmtGetDnode(int32_t dnodeId) {
 
 void *mgmtGetDnodeByEp(char *ep) {
   SDnodeObj *pDnode = NULL;
-  void *     pNode = NULL;
+  void *     pIter = NULL;
 
   while (1) {
-    pNode = mgmtGetNextDnode(pNode, &pDnode);
+    pIter = mgmtGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
+      sdbFreeIter(pIter);
       return pDnode;
     }
     mgmtDecDnodeRef(pDnode);
   }
 
+  sdbFreeIter(pIter);
+
   return NULL;
 }
 
@@ -530,7 +533,7 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
 
   pShow->numOfRows = mgmtGetDnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
 
   mgmtDecUserRef(pUser);
 
@@ -544,7 +547,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char      *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode);
+    pShow->pIter = mgmtGetNextDnode(pShow->pIter, &pDnode);
     if (pDnode == NULL) break;
 
     cols = 0;
@@ -636,7 +639,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->numOfRows = mgmtGetDnodesNum() * TSDB_MOD_MAX;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -648,7 +651,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
 
   while (numOfRows < rows) {
     SDnodeObj *pDnode = NULL;
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+    pShow->pIter = mgmtGetNextDnode(pShow->pIter, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
 
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
@@ -738,7 +741,7 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   }
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -821,7 +824,8 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   if (pShow->payloadLen > 0 ) {
     pDnode = mgmtGetDnodeByEp(pShow->payload);
   } else {
-    mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
+    void *pIter = mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
+    sdbFreeIter(pIter);
   }
 
   if (pDnode != NULL) {
@@ -830,7 +834,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   }
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = pDnode;
+  pShow->pIter = pDnode;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -844,12 +848,12 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
   if (0 == rows) return 0;
 
-  pDnode = (SDnodeObj *)(pShow->pNode);
+  pDnode = (SDnodeObj *)(pShow->pIter);
   if (pDnode != NULL) {
-    void *pNode = NULL;
+    void *pIter = NULL;
     SVgObj *pVgroup;
     while (1) {
-      pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+      pIter = mgmtGetNextVgroup(pIter, &pVgroup);
       if (pVgroup == NULL) break;
 
       for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
@@ -869,6 +873,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
       mgmtDecVgroupRef(pVgroup);
     }
+    sdbFreeIter(pIter);
   } else {
     numOfRows = 0;
   }
diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c
index a78e5574c0..d9ddd465f6 100644
--- a/src/mnode/src/mgmtMnode.c
+++ b/src/mnode/src/mgmtMnode.c
@@ -95,11 +95,12 @@ static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
 static int32_t mgmtMnodeActionRestored() {
   if (mgmtGetMnodesNum() == 1) {
     SMnodeObj *pMnode = NULL;
-    mgmtGetNextMnode(NULL, &pMnode);
+    void *pIter = mgmtGetNextMnode(NULL, &pMnode);
     if (pMnode != NULL) {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
       mgmtDecMnodeRef(pMnode);
     }
+    sdbFreeIter(pIter);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -157,8 +158,8 @@ void mgmtDecMnodeRef(SMnodeObj *pMnode) {
   sdbDecRef(tsMnodeSdb, pMnode);
 }
 
-void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { 
-  return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode); 
+void *mgmtGetNextMnode(void *pIter, SMnodeObj **pMnode) { 
+  return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); 
 }
 
 char *mgmtGetMnodeRoleStr(int32_t role) {
@@ -177,10 +178,10 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
 }
 
 void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
-    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
     strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
@@ -194,6 +195,7 @@ void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
     
     mgmtDecMnodeRef(pMnode);
   }
+  sdbFreeIter(pIter);
 }
 
 void mgmtGetMnodeInfos(void *param) {
@@ -201,10 +203,10 @@ void mgmtGetMnodeInfos(void *param) {
   mnodes->inUse = 0;
   
   int32_t index = 0;
-  void *pNode = NULL;
+  void *pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
-    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    pIter = mgmtGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
     mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
@@ -216,6 +218,7 @@ void mgmtGetMnodeInfos(void *param) {
     index++;
     mgmtDecMnodeRef(pMnode);
   }
+  sdbFreeIter(pIter);
 
   mnodes->nodeNum = index;
 }
@@ -317,7 +320,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
 
   pShow->numOfRows = mgmtGetMnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   mgmtDecUserRef(pUser);
 
   return 0;
@@ -330,7 +333,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char      *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode);
+    pShow->pIter = mgmtGetNextMnode(pShow->pIter, &pMnode);
     if (pMnode == NULL) break;
 
     cols = 0;
diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c
index 64ac8ed710..6667bff052 100644
--- a/src/mnode/src/mgmtProfile.c
+++ b/src/mnode/src/mgmtProfile.c
@@ -140,7 +140,7 @@ int32_t mgmtGetQueries(SShowObj *pShow, void *pConn) {
 //
 //  // sorting based on useconds
 //
-//  pShow->pNode = pQueryShow;
+//  pShow->pIter = pQueryShow;
 
   return 0;
 }
@@ -187,7 +187,7 @@ int32_t mgmtGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetQueries(pShow, pConn);
@@ -252,7 +252,7 @@ int32_t mgmtRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pCo
   char *pWrite;
   int32_t   cols = 0;
 
-  SQueryShow *pQueryShow = (SQueryShow *)pShow->pNode;
+  SQueryShow *pQueryShow = (SQueryShow *)pShow->pIter;
 
   if (rows > pQueryShow->numOfQueries - pQueryShow->index) rows = pQueryShow->numOfQueries - pQueryShow->index;
 
@@ -339,7 +339,7 @@ int32_t mgmtGetStreams(SShowObj *pShow, void *pConn) {
 //
 //  // sorting based on useconds
 //
-//  pShow->pNode = pStreamShow;
+//  pShow->pIter = pStreamShow;
 
   return 0;
 }
@@ -397,7 +397,7 @@ int32_t mgmtGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetStreams(pShow, pConn);
@@ -409,7 +409,7 @@ int32_t mgmtRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pCo
   char *pWrite;
   int32_t   cols = 0;
 
-  SStreamShow *pStreamShow = (SStreamShow *)pShow->pNode;
+  SStreamShow *pStreamShow = (SStreamShow *)pShow->pIter;
 
   if (rows > pStreamShow->numOfStreams - pStreamShow->index) rows = pStreamShow->numOfStreams - pStreamShow->index;
 
@@ -592,7 +592,7 @@ int mgmtGetConns(SShowObj *pShow, void *pConn) {
   //
   //  // sorting based on useconds
   //
-  //  pShow->pNode = pConnShow;
+  //  pShow->pIter = pConnShow;
 
   return 0;
 }
@@ -627,7 +627,7 @@ int32_t mgmtGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
 
   pShow->numOfRows = 1000000;
-  pShow->pNode = NULL;
+  pShow->pIter = NULL;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
   mgmtGetConns(pShow, pConn);
@@ -639,7 +639,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
   char      *pWrite;
   int32_t   cols = 0;
 
-  SConnShow *pConnShow = (SConnShow *)pShow->pNode;
+  SConnShow *pConnShow = (SConnShow *)pShow->pIter;
 
   if (rows > pConnShow->numOfConns - pConnShow->index) rows = pConnShow->numOfConns - pConnShow->index;
 
diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c
index 3116ad39f3..3a9ef390f9 100644
--- a/src/mnode/src/mgmtSdb.c
+++ b/src/mnode/src/mgmtSdb.c
@@ -239,10 +239,10 @@ void sdbUpdateSync() {
   }
 
   if (index == 0) {
-    void *pNode = NULL;
+    void *pIter = NULL;
     while (1) {
       SMnodeObj *pMnode = NULL;
-      pNode = mgmtGetNextMnode(pNode, &pMnode);
+      pIter = mgmtGetNextMnode(pIter, &pMnode);
       if (pMnode == NULL) break;
 
       syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
@@ -252,6 +252,7 @@ void sdbUpdateSync() {
 
       mgmtDecMnodeRef(pMnode);
     }
+    sdbFreeIter(pIter);
   }
 
   syncCfg.replica = index;
@@ -666,7 +667,10 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
   }
 
   SSdbRow *pMeta = taosHashIterGet(pIter);
-  if (pMeta == NULL) return NULL;
+  if (pMeta == NULL) {
+    taosHashDestroyIter(pIter);
+    return NULL;
+  }
 
   *ppRow = pMeta->row;
   sdbIncRef(handle, pMeta->row);
@@ -674,6 +678,12 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
   return pIter;
 }
 
+void sdbFreeIter(void *pIter) {
+  if (pIter != NULL) {
+    taosHashDestroyIter(pIter);
+  }
+}
+
 void *sdbOpenTable(SSdbTableDesc *pDesc) {
   SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
   
diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c
index c27dc66dd5..2e3ef7384a 100644
--- a/src/mnode/src/mgmtShell.c
+++ b/src/mnode/src/mgmtShell.c
@@ -48,6 +48,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
+static void mgmtFreeShowObj(void *data);
 
 void *tsMgmtTmr;
 static void *tsMgmtTranQhandle = NULL;
@@ -65,7 +66,7 @@ int32_t mgmtInitShell() {
   
   tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
   tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
-  tsQhandleCache = taosCacheInit(tsMgmtTmr, 10);
+  tsQhandleCache = taosCacheInitWithCb(tsMgmtTmr, 10, mgmtFreeShowObj);
 
   return 0;
 }
@@ -479,7 +480,7 @@ bool mgmtCheckQhandle(uint64_t qhandle) {
     mTrace("qhandle:%p is retrived", qhandle);
     return true;
   } else {
-    mTrace("qhandle:%p is already freed", qhandle);
+    mTrace("qhandle:%p is already released", qhandle);
     return false;
   }
 }
@@ -498,8 +499,14 @@ void* mgmtSaveQhandle(void *qhandle, int32_t size) {
   return NULL;
 }
 
+static void mgmtFreeShowObj(void *data) {
+  SShowObj *pShow = data;
+  //sdbFreeIter(pShow->pIter);
+  mTrace("qhandle:%p is destroyed", pShow);
+}
+
 void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
-  mTrace("qhandle:%p is freed", qhandle);
+  mTrace("qhandle:%p is released", qhandle);
   taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
 }
 
diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c
index affd7a1073..ae2c4c1f6c 100644
--- a/src/mnode/src/mgmtTable.c
+++ b/src/mnode/src/mgmtTable.c
@@ -247,25 +247,19 @@ static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) {
 }
 
 static int32_t mgmtChildTableActionRestored() {
-  void *pNode = NULL;
-  void *pLastNode = NULL;
+  void *pIter = NULL;
   SChildTableObj *pTable = NULL;
 
   while (1) {
-    pLastNode = pNode;
     mgmtDecTableRef(pTable);
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
     if (pTable == NULL) break;
 
     SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
     if (pDb == NULL) {
       mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
-      SSdbOper desc = {0};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
     mgmtDecDbRef(pDb);
@@ -274,12 +268,8 @@ static int32_t mgmtChildTableActionRestored() {
     if (pVgroup == NULL) {
       mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
     mgmtDecVgroupRef(pVgroup);
@@ -288,24 +278,16 @@ static int32_t mgmtChildTableActionRestored() {
       mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it",
              pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
 
     if (pVgroup->tableList == NULL) {
       mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId);
       pTable->vgId = 0;
-      SSdbOper desc = {0};
-      desc.type = SDB_OPER_LOCAL;
-      desc.pObj = pTable;
-      desc.table = tsChildTableSdb;
+      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
       sdbDeleteRow(&desc);
-      pNode = pLastNode;
       continue;
     }
 
@@ -314,18 +296,16 @@ static int32_t mgmtChildTableActionRestored() {
       if (pSuperTable == NULL) {
         mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid);
         pTable->vgId = 0;
-        SSdbOper desc = {0};
-        desc.type = SDB_OPER_LOCAL;
-        desc.pObj = pTable;
-        desc.table = tsChildTableSdb;
+        SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
         sdbDeleteRow(&desc);
-        pNode = pLastNode;
         continue;
       }
       mgmtDecTableRef(pSuperTable);
     }
   }
 
+  sdbFreeIter(pIter);
+
   return 0;
 }
 
@@ -560,10 +540,10 @@ static void *mgmtGetSuperTable(char *tableId) {
 
 static void *mgmtGetSuperTableByUid(uint64_t uid) {
   SSuperTableObj *pStable = NULL;
-  void *          pNode = NULL;
+  void *pIter = NULL;
 
   while (1) {
-    pNode = mgmtGetNextSuperTable(pNode, &pStable);
+    pIter = mgmtGetNextSuperTable(pIter, &pStable);
     if (pStable == NULL) break;
     if (pStable->uid == uid) {
       return pStable;
@@ -571,6 +551,8 @@ static void *mgmtGetSuperTableByUid(uint64_t uid) {
     mgmtDecTableRef(pStable);
   }
 
+  sdbFreeIter(pIter);
+
   return NULL;
 }
 
@@ -588,12 +570,12 @@ void *mgmtGetTable(char *tableId) {
   return NULL;
 }
 
-void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) {
-  return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable);
+void *mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable) {
+  return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
 }
 
-void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) {
-  return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable);
+void *mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable) {
+  return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
 }
 
 void mgmtIncTableRef(void *p1) {
@@ -1122,7 +1104,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
 
   while (numOfRows < rows) {    
     mgmtDecTableRef(pTable);
-    pShow->pNode = mgmtGetNextSuperTable(pShow->pNode, &pTable);
+    pShow->pIter = mgmtGetNextSuperTable(pShow->pIter, &pTable);
     if (pTable == NULL) break;
     if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
       continue;
@@ -1172,8 +1154,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
 }
 
 void mgmtDropAllSuperTables(SDbObj *pDropDb) {
-  void *pNode = NULL;
-  void *pLastNode = NULL;
+  void *  pIter= NULL;
   int32_t numOfTables = 0;
   int32_t dbNameLen = strlen(pDropDb->name);
   SSuperTableObj *pTable = NULL;
@@ -1181,8 +1162,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
   mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextSuperTable(pNode, &pTable);
+    pIter = mgmtGetNextSuperTable(pIter, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1192,13 +1172,14 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables ++;
     }
 
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -1753,8 +1734,7 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
 }
 
 void mgmtDropAllChildTables(SDbObj *pDropDb) {
-  void *pNode = NULL;
-  void *pLastNode = NULL;
+  void *  pIter = NULL;
   int32_t numOfTables = 0;
   int32_t dbNameLen = strlen(pDropDb->name);
   SChildTableObj *pTable = NULL;
@@ -1762,8 +1742,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
   mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1773,26 +1752,25 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables++;
     }
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
 static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
-  void *pNode = NULL;
-  void *pLastNode = NULL;
+  void *  pIter = NULL;
   int32_t numOfTables = 0;
   SChildTableObj *pTable = NULL;
 
   mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables);
 
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextChildTable(pNode, &pTable);
+    pIter = mgmtGetNextChildTable(pIter, &pTable);
     if (pTable == NULL) break;
 
     if (pTable->superTable == pStable) {
@@ -1802,13 +1780,14 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
         .pObj = pTable,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfTables++;
     }
 
     mgmtDecTableRef(pTable);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
 }
 
@@ -2079,7 +2058,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
 
   while (numOfRows < rows) {
     mgmtDecTableRef(pTable);
-    pShow->pNode = mgmtGetNextChildTable(pShow->pNode, &pTable);
+    pShow->pIter = mgmtGetNextChildTable(pShow->pIter, &pTable);
     if (pTable == NULL) break;
 
     // not belong to current db
diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c
index 350367e0ae..cc22ed2169 100644
--- a/src/mnode/src/mgmtUser.c
+++ b/src/mnode/src/mgmtUser.c
@@ -155,8 +155,8 @@ SUserObj *mgmtGetUser(char *name) {
   return (SUserObj *)sdbGetRow(tsUserSdb, name);
 }
 
-void *mgmtGetNextUser(void *pNode, SUserObj **pUser) { 
-  return sdbFetchRow(tsUserSdb, pNode, (void **)pUser); 
+void *mgmtGetNextUser(void *pIter, SUserObj **pUser) { 
+  return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); 
 }
 
 void mgmtIncUserRef(SUserObj *pUser) { 
@@ -300,7 +300,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   char     *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser);
+    pShow->pIter = mgmtGetNextUser(pShow->pIter, &pUser);
     if (pUser == NULL) break;
     
     cols = 0;
@@ -504,15 +504,13 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
 }
 
 void  mgmtDropAllUsers(SAcctObj *pAcct)  {
-  void *    pNode = NULL;
-  void *    pLastNode = NULL;
+  void *    pIter = NULL;
   int32_t   numOfUsers = 0;
   int32_t   acctNameLen = strlen(pAcct->user);
   SUserObj *pUser = NULL;
 
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextUser(pNode, &pUser);
+    pIter = mgmtGetNextUser(pIter, &pUser);
     if (pUser == NULL) break;
 
     if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
@@ -522,13 +520,14 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
         .pObj = pUser,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfUsers++;
     }
 
     mgmtDecUserRef(pUser);
   }
 
+  sdbFreeIter(pIter);
+
   mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
 }
 
diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c
index b87c6ad727..f8447a31fd 100644
--- a/src/mnode/src/mgmtVgroup.c
+++ b/src/mnode/src/mgmtVgroup.c
@@ -288,8 +288,8 @@ SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) {
   return pDb->pHead;
 }
 
-void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { 
-  return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); 
+void *mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup) { 
+  return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
 void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
@@ -429,10 +429,10 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
 
   if (NULL == pTable) {
     pShow->numOfRows = pDb->numOfVgroups;
-    pShow->pNode = pDb->pHead;
+    pShow->pIter = pDb->pHead;
   } else {
     pShow->numOfRows = 1;
-    pShow->pNode = pVgroup;
+    pShow->pIter = pVgroup;
   }
 
    mgmtDecDbRef(pDb);
@@ -457,9 +457,9 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
   }
 
   while (numOfRows < rows) {
-    pVgroup = (SVgObj *) pShow->pNode;
+    pVgroup = (SVgObj *) pShow->pIter;
     if (pVgroup == NULL) break;
-    pShow->pNode = (void *) pVgroup->next;
+    pShow->pIter = (void *) pVgroup->next;
 
     cols = 0;
 
@@ -749,14 +749,12 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
 }
 
 void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
-  void *  pNode = NULL;
-  void *  pLastNode = NULL;
+  void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
   int32_t numOfVgroups = 0;
 
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
@@ -766,24 +764,23 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
         .pObj = pVgroup,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfVgroups++;
       continue;
     }
     mgmtDecVgroupRef(pVgroup);
   }
+
+  sdbFreeIter(pIter);
 }
 
 void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
-  void *pNode = NULL;
-  void *pLastNode = NULL;
+  void *  pIter = NULL;
   int32_t numOfVgroups = 0;
   SVgObj *pVgroup = NULL;
 
   mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
   while (1) {
-    pLastNode = pNode;
-    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    pIter = mgmtGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->pDb == pDropDb) {
@@ -793,7 +790,6 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
         .pObj = pVgroup,
       };
       sdbDeleteRow(&oper);
-      pNode = pLastNode;
       numOfVgroups++;
 
       if (sendMsg) {
@@ -804,5 +800,7 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
     mgmtDecVgroupRef(pVgroup);
   }
 
+  sdbFreeIter(pIter);
+
   mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
 }
diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h
index 2e26f8f0e3..6dd707b763 100644
--- a/src/util/inc/tcache.h
+++ b/src/util/inc/tcache.h
@@ -67,6 +67,7 @@ typedef struct {
   void *       pTimer;
   SCacheStatis statistics;
   SHashObj *   pHashTable;
+  _hash_free_fn_t freeFp;
   int          numOfElemsInTrash;  // number of element in trash
   int16_t      deleting;           // set the deleting flag to stop refreshing ASAP.
   T_REF_DECLARE()
@@ -88,6 +89,7 @@ typedef struct {
  * @return
  */
 SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds);
+SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data));
 
 /**
  * add data into cache
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 02a5b93141..7d8cb1faa4 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -149,6 +149,7 @@ static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) {
   }
   
   pElem->pData->signature = 0;
+  if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data);
   free(pElem->pData);
   free(pElem);
 }
@@ -210,7 +211,8 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo
   int32_t size = pNode->size;
   taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize);
   
-  uTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size);
+  uTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size);  
+  if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data);
   free(pNode);
 }
 
@@ -380,7 +382,7 @@ static void taosCacheRefresh(void *handle, void *tmrId) {
     }
 }
 
-SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
+SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb)(void *data)) {
   if (tmrCtrl == NULL || refreshTime <= 0) {
     return NULL;
   }
@@ -399,8 +401,9 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
   }
   
   // set free cache node callback function for hash table
-  taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
+  // taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
   
+  pCacheObj->freeFp = freeCb;
   pCacheObj->refreshTime = refreshTime * 1000;
   pCacheObj->tmrCtrl = tmrCtrl;
   
@@ -419,6 +422,10 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
   return pCacheObj;
 }
 
+SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
+  return taosCacheInitWithCb(tmrCtrl, refreshTime, NULL);
+}
+
 void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) {
   SCacheDataNode *pNode;
   

From 5b459cf8fee2366121082964c0462783386351cf Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 14:54:55 +0800
Subject: [PATCH 22/33] scripts

---
 tests/script/general/parser/auto_create_tb.sim |  9 +++++----
 tests/script/unique/mnode/mgmt22.sim           | 15 ++-------------
 2 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/tests/script/general/parser/auto_create_tb.sim b/tests/script/general/parser/auto_create_tb.sim
index a29d917244..ad65ea723f 100644
--- a/tests/script/general/parser/auto_create_tb.sim
+++ b/tests/script/general/parser/auto_create_tb.sim
@@ -149,16 +149,17 @@ sql drop table tb1
 #### auto create multiple tables
 sql insert into tb1 using $stb tags(1) values ( $ts0 , 1, 1, 1, 1, 'bin1', 1, 1, 1, '涛思数据1') tb2 using $stb tags(2) values ( $ts0 , 2, 2, 2, 2, 'bin2', 2, 2, 2, '涛思数据2') tb3 using $stb tags(3) values ( $ts0 , 3, 3, 3, 3, 'bin3', 3, 3, 3, '涛思数据3')
 sql show tables
+print $rows $data00 $data10 $data20
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb1 then
+if $data00 != tb3 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb3 then
+if $data20 != tb1 then
   return -1
 endi
 
@@ -220,13 +221,13 @@ sql show tables
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb1 then
+if $data00 != tb3 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb3 then
+if $data20 != tb1 then
   return -1
 endi
 
diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim
index e4e1235e66..37e38fbf66 100644
--- a/tests/script/unique/mnode/mgmt22.sim
+++ b/tests/script/unique/mnode/mgmt22.sim
@@ -8,7 +8,7 @@ system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2
 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2
 
 print ============== step1
-system sh/exec_up.sh -n dnode1 -s start -t
+system sh/exec_up.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
@@ -20,7 +20,7 @@ if $data2_1 != master then
 endi
 
 print ============== step2
-system sh/exec_up.sh -n dnode2 -s start -t
+system sh/exec_up.sh -n dnode2 -s start
 sql create dnode $hostname2
 
 $x = 0
@@ -41,17 +41,6 @@ if $data2_2 != slave then
 	goto show2
 endi
 
-system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode2 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode3 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec_up.sh -n dnode8 -s stop  -x SIGINT
-
-return
-
 print ============== step3
 sql_error drop dnode $hostname1 -x error1
 print should not drop master

From 605e63200a241b680961a7317b897fecb6a3c7fa Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Wed, 13 May 2020 10:12:57 +0800
Subject: [PATCH 23/33] fix failed test cases

---
 src/client/src/tscServer.c                 |  7 +--
 src/client/src/tscSub.c                    |  8 +--
 src/query/src/queryExecutor.c              | 63 +++++++++++++---------
 tests/script/general/cache/new_metrics.sim |  2 +-
 4 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 32eee08541..4d9d107b01 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -39,7 +39,7 @@ int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql);
 void tscProcessActivityTimer(void *handle, void *tmrId);
 int tscKeepConn[TSDB_SQL_MAX] = {0};
 
-TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid);
+TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt);
 void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts);
 void tscSaveSubscriptionProgress(void* sub);
 
@@ -575,6 +575,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
 
 static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
+  TSKEY dfltKey = htobe64(pQueryMsg->window.skey);
 
   STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
@@ -596,7 +597,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
     pTableIdInfo->tid = htonl(pTableMeta->sid);
     pTableIdInfo->uid = htobe64(pTableMeta->uid);
-    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid));
+    pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
 
@@ -624,7 +625,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
       STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
       pTableIdInfo->tid = htonl(pItem->tid);
       pTableIdInfo->uid = htobe64(pItem->uid);
-      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid));
+      pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey));
       pMsg += sizeof(STableIdInfo);
     }
   }
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 9a3f3d290c..8e90660a24 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -56,16 +56,16 @@ static int tscCompareSubscriptionProgress(const void* a, const void* b) {
   return 0;
 }
 
-TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid) {
+TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   if (sub == NULL) {
-    return 0;
+    return dflt;
   }
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
   SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
   if (p == NULL) {
-    return INT64_MIN;
+    return dflt;
   }
   return p->key;
 }
@@ -228,7 +228,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   for( size_t i = 0; i < numOfTables; i++ ) {
     STidTags* tt = taosArrayGet( tables, i );
     SSubscriptionProgress p = { .uid = tt->uid };
-    p.key = tscGetSubscriptionProgress(pSub, tt->uid);
+    p.key = tscGetSubscriptionProgress(pSub, tt->uid, INT64_MIN);
     taosArrayPush(progress, &p);
   }
   taosArraySort(progress, tscCompareSubscriptionProgress);
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index 24f0f53ca5..f48404ab3a 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -4162,6 +4162,42 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
   return true;
 }
 
+
+static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
+  SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
+  SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
+  if (onlyQueryTags(pQuery)) {
+    return;
+  }
+
+  if (isSTableQuery && (!isIntervalQuery(pQuery)) && (!isFixedOutputQuery(pQuery))) {
+    return;
+  }
+
+  STsdbQueryCond cond = {
+    .twindow = pQuery->window,
+    .order   = pQuery->order.order,
+    .colList = pQuery->colList,
+    .numOfCols = pQuery->numOfCols,
+  };
+
+  if (!isSTableQuery
+    && (pQInfo->groupInfo.numOfTables == 1)
+    && (cond.order == TSDB_ORDER_ASC) 
+    && (!isIntervalQuery(pQuery))
+    && (!isGroupbyNormalCol(pQuery->pGroupbyExpr))
+    && (!isFixedOutputQuery(pQuery))
+  ) {
+    SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
+    SGroupItem* pItem = taosArrayGet(pa, 0);
+    cond.twindow = pItem->info->win;
+  }
+
+  pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
+}
+
+
 int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
 
@@ -4170,30 +4206,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool
 
   setScanLimitationByResultBuffer(pQuery);
   changeExecuteScanOrder(pQuery, false);
-
-  STsdbQueryCond cond = {
-      .twindow = pQuery->window,
-      .order   = pQuery->order.order,
-      .colList = pQuery->colList,
-      .numOfCols = pQuery->numOfCols,
-  };
-  
-  
-  // normal query setup the queryhandle here
-  if (!onlyQueryTags(pQuery)) {
-    if (!isSTableQuery && isFirstLastRowQuery(pQuery)) {  // in case of last_row query, invoke a different API.
-      pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
-    } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
-
-      if(pQInfo->groupInfo.numOfTables == 1) {
-        SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0);
-        SGroupItem* pItem = taosArrayGet(pa, 0);
-        cond.twindow = pItem->info->win;
-      }
-
-      pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
-    }
-  }
+  setupQueryHandle(tsdb, pQInfo, isSTableQuery);
   
   pQInfo->tsdb = tsdb;
   pQInfo->vgId = vgId;
@@ -5772,7 +5785,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
       if (pTableId != NULL ) {
         window.skey = pTableId->key;
       } else {
-        window.skey = INT64_MIN;
+        window.skey = pQueryMsg->window.skey;
       }
       item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window);
       item.info->groupIdx = i;
diff --git a/tests/script/general/cache/new_metrics.sim b/tests/script/general/cache/new_metrics.sim
index 64170ec67e..0e304d9acb 100644
--- a/tests/script/general/cache/new_metrics.sim
+++ b/tests/script/general/cache/new_metrics.sim
@@ -27,7 +27,7 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool)
 $i = 0
 while $i < 5
   $tb = $tbPrefix . $i
-  sql create table $tb using $mt tags( 0 )  
+  sql create table $tb using $mt tags( $i )  
   $x = 0
   while $x < $rowNum
     $val = $x * 60000

From e81204431e1d19870b3afd1f01557fb4b365d769 Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Wed, 13 May 2020 15:12:25 +0800
Subject: [PATCH 24/33] [td-225]

---
 src/client/src/tscSecondaryMerge.c       |  3 ++-
 src/query/src/queryExecutor.c            |  2 +-
 src/tsdb/src/tsdbRead.c                  | 18 ++++++++++--------
 tests/script/general/import/replica1.sim |  3 ++-
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c
index 71d50b56f9..a3a0441ff8 100644
--- a/src/client/src/tscSecondaryMerge.c
+++ b/src/client/src/tscSecondaryMerge.c
@@ -283,7 +283,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   // used to keep the latest input row
   pReducer->pTempBuffer = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage));
-
   pReducer->discardData = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage));
   pReducer->discard = false;
 
@@ -311,6 +310,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     return;
   }
 
+  size = tscSqlExprNumOfExprs(pQueryInfo);
+  
   pReducer->pTempBuffer->numOfElems = 0;
   pReducer->pResInfo = calloc(size, sizeof(SResultInfo));
 
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index 4b48d79316..670bda62ef 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -2399,7 +2399,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     // todo extract methods
     if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) {
       TSKEY           skey1, ekey1;
-      STimeWindow     w = {0};
+      STimeWindow     w = TSWINDOW_INITIALIZER;
       SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
       if (QUERY_IS_ASC_QUERY(pQuery)) {
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index b01407024d..7768ecf28e 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -54,12 +54,12 @@ typedef struct SQueryFilePos {
 typedef struct SDataBlockLoadInfo {
   SFileGroup* fileGroup;
   int32_t     slot;
-  int32_t     sid;
+  int32_t     tid;
   SArray*     pLoadedCols;
 } SDataBlockLoadInfo;
 
 typedef struct SLoadCompBlockInfo {
-  int32_t sid; /* table sid */
+  int32_t tid; /* table tid */
   int32_t fileId;
   int32_t fileListIndex;
 } SLoadCompBlockInfo;
@@ -127,12 +127,12 @@ static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle);
 
 static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
   pBlockLoadInfo->slot = -1;
-  pBlockLoadInfo->sid = -1;
+  pBlockLoadInfo->tid = -1;
   pBlockLoadInfo->fileGroup = NULL;
 }
 
 static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
-  pCompBlockLoadInfo->sid = -1;
+  pCompBlockLoadInfo->tid = -1;
   pCompBlockLoadInfo->fileId = -1;
   pCompBlockLoadInfo->fileListIndex = -1;
 }
@@ -423,7 +423,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
   bool    blockLoaded = false;
   SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
 
-  if (pCheckInfo->pDataCols == NULL) {
+  if (pCheckInfo->pDataCols == NULL) { // todo: why not the real data?
     pCheckInfo->pDataCols = tdNewDataCols(pRepo->tsdbMeta->maxRowBytes, pRepo->tsdbMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
   }
 
@@ -434,7 +434,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 
     pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup;
     pBlockLoadInfo->slot = pQueryHandle->cur.slot;
-    pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid;
+    pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid;
 
     blockLoaded = true;
   }
@@ -614,7 +614,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
     SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
     int32_t bytes = pCol->info.bytes;
 
-    for (int32_t j = 0; j < numOfCols; ++j) {
+    for (int32_t j = 0; j < numOfCols; ++j) {  //todo opt performance
       SDataCol* src = &pQueryHandle->rhelper.pDataCols[0]->cols[j];
 
       if (pCol->info.colId == src->colId) {
@@ -1197,7 +1197,9 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
     } else {
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
-      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->sid == pCheckInfo->pTableObj->tableId.tid) {
+      
+      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
+          pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {
         SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock;
diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim
index b2b7291623..1bd1419496 100644
--- a/tests/script/general/import/replica1.sim
+++ b/tests/script/general/import/replica1.sim
@@ -111,7 +111,8 @@ sql import into tb values(1520000034001, 34001)
 sql import into tb values(1520000050001, 50001)
 sql select * from tb;
 print $rows
-if $rows != 19 then 
+if $rows != 19 then
+  print expect 19, actual: $rows
   return -1
 endi
 

From f3b80c84e5c58e9a65c0cb0d5749990202087a45 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 15:22:49 +0800
Subject: [PATCH 25/33] [TD-149] add callback func to cache

---
 src/mnode/src/mgmtShell.c | 12 ++++++------
 src/util/src/tcache.c     |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c
index 2e3ef7384a..804572b9ff 100644
--- a/src/mnode/src/mgmtShell.c
+++ b/src/mnode/src/mgmtShell.c
@@ -477,10 +477,10 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) {
 bool mgmtCheckQhandle(uint64_t qhandle) {
   void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle);
   if (pSaved == (void *)qhandle) {
-    mTrace("qhandle:%p is retrived", qhandle);
+    mTrace("show:%p, is retrieved", qhandle);
     return true;
   } else {
-    mTrace("qhandle:%p is already released", qhandle);
+    mTrace("show:%p, is already released", qhandle);
     return false;
   }
 }
@@ -492,7 +492,7 @@ void* mgmtSaveQhandle(void *qhandle, int32_t size) {
     void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60);
     free(qhandle);
     
-    mTrace("qhandle:%p is saved", newQhandle);
+    mTrace("show:%p, is saved", newQhandle);
     return newQhandle;
   }
   
@@ -501,12 +501,12 @@ void* mgmtSaveQhandle(void *qhandle, int32_t size) {
 
 static void mgmtFreeShowObj(void *data) {
   SShowObj *pShow = data;
-  //sdbFreeIter(pShow->pIter);
-  mTrace("qhandle:%p is destroyed", pShow);
+  sdbFreeIter(pShow->pIter);
+  mTrace("show:%p, is destroyed", pShow);
 }
 
 void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
-  mTrace("qhandle:%p is released", qhandle);
+  mTrace("show:%p, is released, force:%s", qhandle, forceRemove ? "true" : "false");
   taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
 }
 
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 7d8cb1faa4..0c4978dc5c 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -401,7 +401,7 @@ SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb
   }
   
   // set free cache node callback function for hash table
-  // taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
+  taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
   
   pCacheObj->freeFp = freeCb;
   pCacheObj->refreshTime = refreshTime * 1000;

From 543415b8d5cb1e1ec22bbbc3916a22baef212d40 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Wed, 13 May 2020 16:27:23 +0800
Subject: [PATCH 26/33] support GCC 4.8

[TD-307]
---
 .travis.yml                      | 23 +++++++++++++++++++++++
 src/client/src/tscPrepare.c      |  2 +-
 src/client/src/tscSQLParser.c    |  4 ++--
 src/client/src/tscSubquery.c     |  2 +-
 src/dnode/src/dnodeMain.c        |  2 +-
 src/dnode/src/dnodeModule.c      |  2 +-
 src/os/linux/inc/os.h            | 11 +++++++++++
 src/os/linux/src/linuxPlatform.c |  2 +-
 src/tsdb/src/tsdbMain.c          |  4 ++--
 src/util/src/exception.c         |  5 +++--
 src/util/src/tconfig.c           |  2 +-
 src/util/src/tdes.c              |  2 +-
 src/util/src/terror.c            |  3 ++-
 13 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 979b7588e5..41937853d6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -203,6 +203,29 @@ matrix:
           ;;
         esac
 
+  - os: linux
+    dist: trusty
+    language: c
+    git:
+      - depth: 1
+
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+    env: 
+      - DESC="trusty/gcc-4.8 build"
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
+
+    script:
+      - cmake .. > /dev/null
+      - make > /dev/null
+
   - os: linux
     language: c
     compiler: clang
diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c
index 260649050b..2caa4ad436 100644
--- a/src/client/src/tscPrepare.c
+++ b/src/client/src/tscPrepare.c
@@ -194,7 +194,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
 
 static char* normalStmtBuildSql(STscStmt* stmt) {
   SNormalStmt* normal = &stmt->normal;
-  SStringBuilder sb = {0};
+  SStringBuilder sb; memset(&sb, 0, sizeof(sb));
 
   if (taosStringBuilderSetJmp(&sb) != 0) {
     taosStringBuilderDestroy(&sb);
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index ebb081bb3a..551b0a0457 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -3558,7 +3558,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac
     return TSDB_CODE_SUCCESS;
   }
 
-  SStringBuilder sb1 = {0};
+  SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1));
   taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
 
   char db[TSDB_TABLE_ID_LEN] = {0};
@@ -3813,7 +3813,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
   pQueryInfo->window.ekey = INT64_MAX;
 
   // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
-  SStringBuilder sb = {0};
+  SStringBuilder sb; memset(&sb, 0, sizeof(sb));
   SCondExpr      condExpr = {0};
 
   if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) {
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 02af71ee66..dea76968f6 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -488,7 +488,7 @@ void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) {
     if( prev == NULL || tt->vgId != prev->vgId ) {
       SVgroupsInfo* pvg = pTableMetaInfo->vgroupList;
 
-      SVgroupTableInfo info = { 0 };
+      SVgroupTableInfo info = {{ 0 }};
       for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) {
         if( tt->vgId == pvg->vgroups[m].vgId ) {
           info.vgInfo = pvg->vgroups[m];
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index dc5f7d192f..7949a62966 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -84,7 +84,7 @@ int32_t main(int32_t argc, char *argv[]) {
   }
 
   /* Set termination handler. */
-  struct sigaction act = {0};
+  struct sigaction act = {{0}};
   act.sa_flags = SA_SIGINFO;
   act.sa_sigaction = signal_handler;
   sigaction(SIGTERM, &act, NULL);
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 2e081de2a8..86193fcebc 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -33,7 +33,7 @@ typedef struct {
   void    (*stopFp)();
 } SModule;
 
-static SModule  tsModule[TSDB_MOD_MAX] = {0};
+static SModule  tsModule[TSDB_MOD_MAX] = {{0}};
 static uint32_t tsModuleStatus = 0;
 
 static void dnodeSetModuleStatus(int32_t module) {
diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h
index cab899753d..40fcf83431 100644
--- a/src/os/linux/inc/os.h
+++ b/src/os/linux/inc/os.h
@@ -253,6 +253,17 @@ void taosBlockSIGPIPE();
 #define BUILDIN_CLZ(val) __builtin_clz(val)
 #define BUILDIN_CTZ(val) __builtin_ctz(val)
 
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c
index 72da97287f..73cccd020a 100644
--- a/src/os/linux/src/linuxPlatform.c
+++ b/src/os/linux/src/linuxPlatform.c
@@ -157,7 +157,7 @@ void *taosProcessAlarmSignal(void *tharg) {
   void (*callback)(int) = tharg;
 
   static timer_t         timerId;
-  struct sigevent sevent = {0};
+  struct sigevent sevent = {{0}};
 
   #ifdef _ALPINE
     sevent.sigev_notify = SIGEV_THREAD;
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index f06230c16f..fb25e9dcd9 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -161,7 +161,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
   SFileGroup *pFGroup = NULL;
 
   SFileGroupIter iter;
-  SRWHelper      rhelper = {0};
+  SRWHelper      rhelper = {{0}};
 
   if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
   tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
@@ -948,7 +948,7 @@ static void *tsdbCommitData(void *arg) {
   STsdbCache *pCache = pRepo->tsdbCache;
   STsdbCfg *  pCfg = &(pRepo->config);
   SDataCols * pDataCols = NULL;
-  SRWHelper   whelper = {0};
+  SRWHelper   whelper = {{0}};
   if (pCache->imem == NULL) return NULL;
 
   tsdbPrint("vgId: %d, starting to commit....", pRepo->config.tsdbId);
diff --git a/src/util/src/exception.c b/src/util/src/exception.c
index 3d2949c093..e1f6516fe7 100644
--- a/src/util/src/exception.c
+++ b/src/util/src/exception.c
@@ -1,7 +1,8 @@
+#include "os.h"
 #include "exception.h"
 
 
-static _Thread_local SExceptionNode* expList;
+static threadlocal SExceptionNode* expList;
 
 void exceptionPushNode( SExceptionNode* node ) {
     node->prev = expList;
@@ -145,4 +146,4 @@ void cleanupExecuteTo( int32_t anchor, bool failed ) {
 
 void cleanupExecute( SExceptionNode* node, bool failed ) {
     doExecuteCleanup( node, 0, failed );
-}
\ No newline at end of file
+}
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index 61ae647bf4..32a3df844c 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -25,7 +25,7 @@
 #include "tsystem.h"
 #include "tutil.h"
 
-SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {0};
+SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}};
 int32_t    tsGlobalConfigNum = 0;
 
 static char *tsGlobalUnit[] = {
diff --git a/src/util/src/tdes.c b/src/util/src/tdes.c
index 6f72dec574..00474e4ae2 100644
--- a/src/util/src/tdes.c
+++ b/src/util/src/tdes.c
@@ -32,7 +32,7 @@ char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mo
   unsigned int  number_of_blocks = len / 8;
   unsigned char data_block[9] = {0};
   unsigned char processed_block[9] = {0};
-  key_set       key_sets[17] = {0};
+  key_set       key_sets[17]; memset(key_sets, 0, sizeof(key_sets));
   char*         dest = calloc(len + 1, 1);
   generate_sub_keys(key, key_sets);
 
diff --git a/src/util/src/terror.c b/src/util/src/terror.c
index 628f7a2d5c..c040a11362 100644
--- a/src/util/src/terror.c
+++ b/src/util/src/terror.c
@@ -27,10 +27,11 @@ typedef struct {
 } STaosError;
  
 
+#include "os.h"
 #include "taoserror.h"
 
 
-static _Thread_local int32_t tsErrno;
+static threadlocal int32_t tsErrno;
 int32_t* taosGetErrno() {
   return &tsErrno;
 }

From cfcbefba48ac61db673c4e0aca0a4dc54da80ef5 Mon Sep 17 00:00:00 2001
From: hjLiao <hjliao@taosdata.com>
Date: Wed, 13 May 2020 16:32:33 +0800
Subject: [PATCH 27/33] [td-225] fix bugs in multi-vnode projection query.

---
 src/client/src/tscAsync.c    |  27 ++++++-
 src/client/src/tscServer.c   |   3 +-
 src/client/src/tscSql.c      |   6 +-
 src/client/src/tscSubquery.c |   4 +-
 src/client/src/tscUtil.c     | 132 ++++++++++++++++++-----------------
 src/query/inc/qsqltype.h     |   2 +-
 6 files changed, 99 insertions(+), 75 deletions(-)

diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 9f207936df..8fba577909 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -145,7 +145,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
     return;
   }
   
-  // local reducer has handle this situation during super table non-projection query.
+  // local merge has handle this situation during super table non-projection query.
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) {
     pRes->numOfTotalInCurrentClause += pRes->numOfRows;
   }
@@ -222,9 +222,30 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi
   tscResetForNextRetrieve(pRes);
   
   // handle the sub queries of join query
-  if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+  if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     tscFetchDatablockFromSubquery(pSql);
-  } else {
+  } else if (pRes->completed && pCmd->command == TSDB_SQL_FETCH) {
+    if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
+      tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode);
+      return;
+    } else {
+      /*
+       * all available virtual node has been checked already, now we need to check
+       * for the next subclause queries
+       */
+      if (pCmd->clauseIndex < pCmd->numOfClause - 1) {
+        tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode);
+        return;
+      }
+    
+      /*
+       * 1. has reach the limitation
+       * 2. no remain virtual nodes to be retrieved anymore
+       */
+      (*pSql->fetchFp)(param, pSql, 0);
+    }
+    return;
+  } else { // current query is not completed, continue retrieve from node
     if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
       pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
     }
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index b958bbe15c..889433770a 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -1444,8 +1444,9 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
   }
 
   pRes->row = 0;
+  pRes->completed = (pRes->numOfRows == 0);
 
-  uint8_t code = pRes->code;
+  int32_t code = pRes->code;
   if (pRes->code == TSDB_CODE_SUCCESS) {
     (*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
   } else {
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 14f0fa07ca..87292f4fe6 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -463,11 +463,11 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   }
   
-  // current data are exhausted, fetch more data
-  if (pRes->row >= pRes->numOfRows && pRes->completed != true &&
+  // current data set are exhausted, fetch more data from node
+  if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql)) &&
       (pCmd->command == TSDB_SQL_RETRIEVE ||
        pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE ||
-       pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE ||
+       pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE ||
        pCmd->command == TSDB_SQL_FETCH ||
        pCmd->command == TSDB_SQL_SHOW ||
        pCmd->command == TSDB_SQL_SELECT ||
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 1e4a142a90..cf14525c49 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1203,7 +1203,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
     }
   }
   
-  pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_METRIC_JOIN_RETRIEVE;
+  pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_TABLE_JOIN_RETRIEVE;
   
   return TSDB_CODE_SUCCESS;
 }
@@ -1949,7 +1949,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
   
   assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
   
-  if(pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+  if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     if (pRes->completed) {
       tfree(pRes->tsrow);
     }
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 171d40b932..f0bc8d9755 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -386,7 +386,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
 
   int32_t cmd = pCmd->command;
   if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      cmd == TSDB_SQL_METRIC_JOIN_RETRIEVE) {
+      cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
     tscRemoveFromSqlList(pSql);
   }
   
@@ -1644,9 +1644,11 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro
 void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
   tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
   
-  int32_t index = pQueryInfo->numOfTables;
-  while (index >= 0) {
-    doRemoveTableMetaInfo(pQueryInfo, --index, removeFromCache);
+  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
+  
+    tscClearTableMetaInfo(pTableMetaInfo, removeFromCache);
+    free(pTableMetaInfo);
   }
   
   tfree(pQueryInfo->pTableMetaInfo);
@@ -1698,16 +1700,8 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
   taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
   tfree(pTableMetaInfo->vgroupList);
   
-  if (pTableMetaInfo->tagColList != NULL) {
-    size_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList);
-    for(int32_t i = 0; i < numOfTags; ++i) { // todo do NOT use the allocated object
-      SColumn* pCol = taosArrayGetP(pTableMetaInfo->tagColList, i);
-      tfree(pCol);
-    }
-    
-    taosArrayDestroy(pTableMetaInfo->tagColList);
-    pTableMetaInfo->tagColList = NULL;
-  }
+  tscColumnListDestroy(pTableMetaInfo->tagColList);
+  pTableMetaInfo->tagColList = NULL;
 }
 
 void tscResetForNextRetrieve(SSqlRes* pRes) {
@@ -1991,22 +1985,28 @@ char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
 
 /**
  *  If current vnode query does not return results anymore (pRes->numOfRows == 0), try the next vnode if exists,
- *  in case of multi-vnode super table projection query and the result does not reach the limitation.
+ *  while multi-vnode super table projection query and the result does not reach the limitation.
  */
 bool hasMoreVnodesToTry(SSqlObj* pSql) {
-//  SSqlCmd* pCmd = &pSql->cmd;
-//  SSqlRes* pRes = &pSql->res;
-
-//  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
-  
-//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-//  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) {
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  if (pCmd->command != TSDB_SQL_FETCH) {
     return false;
-//  }
+  }
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   
-//  int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
-//  return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
-//         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < totalVnode - 1);
+  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  assert(pRes->completed);
+  
+  // for normal table, do not try any more if result are exhausted
+  if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) {
+    return false;
+  }
+  
+  int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+  return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
+         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1);
 }
 
 void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
@@ -2022,12 +2022,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
   assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes));
 
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  int32_t totalVnode = 0;
-//  int32_t         totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
-
-  while (++pTableMetaInfo->vgroupIndex < totalVnode) {
+  
+  int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+  while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
     tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
-             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotalInCurrentClause);
+             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfTotalInCurrentClause);
 
     /*
      * update the limit and offset value for the query on the next vnode,
@@ -2043,10 +2042,10 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
     }
 
     pQueryInfo->limit.offset = pRes->offset;
-
     assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0));
-    tscTrace("%p new query to next vnode, vnode index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, pSql,
-             pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
+    
+    tscTrace("%p new query to next vgroup, index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64,
+        pSql, pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
 
     /*
      * For project query with super table join, the numOfSub is equalled to the number of all subqueries.
@@ -2060,43 +2059,46 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
     tscResetForNextRetrieve(pRes);
 
     // in case of async query, set the callback function
-    void* fp1 = pSql->fp;
+//    void* fp1 = pSql->fp;
     pSql->fp = fp;
 
-    if (fp1 != NULL) {
-      assert(fp != NULL);
+//    if (fp1 != NULL) {
+//      assert(fp != NULL);
+//    }
+
+    int32_t ret = tscProcessSql(pSql);
+    if (ret == TSDB_CODE_SUCCESS) {
+      return;
+    } else {// todo check for failure
     }
-
-    int32_t ret = tscProcessSql(pSql);  // todo check for failure
-
     // in case of async query, return now
-    if (fp != NULL) {
-      return;
-    }
-
-    if (ret != TSDB_CODE_SUCCESS) {
-      pSql->res.code = ret;
-      return;
-    }
-
-    // retrieve data
-    assert(pCmd->command == TSDB_SQL_SELECT);
-    pCmd->command = TSDB_SQL_FETCH;
-
-    if ((ret = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) {
-      pSql->res.code = ret;
-      return;
-    }
-
-    // if the result from current virtual node are empty, try next if exists. otherwise, return the results.
-    if (pRes->numOfRows > 0) {
-      break;
-    }
+//    if (fp != NULL) {
+//      return;
+//    }
+//
+//    if (ret != TSDB_CODE_SUCCESS) {
+//      pSql->res.code = ret;
+//      return;
+//    }
+//
+//    // retrieve data
+//    assert(pCmd->command == TSDB_SQL_SELECT);
+//    pCmd->command = TSDB_SQL_FETCH;
+//
+//    if ((ret = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) {
+//      pSql->res.code = ret;
+//      return;
+//    }
+//
+//    // if the result from current virtual node are empty, try next if exists. otherwise, return the results.
+//    if (pRes->numOfRows > 0) {
+//      break;
+//    }
   }
 
-  if (pRes->numOfRows == 0) {
-    tscTrace("%p all vnodes exhausted, prj query completed. total res:%d", pSql, totalVnode, pRes->numOfTotal);
-  }
+//  if (pRes->numOfRows == 0) {
+//    tscTrace("%p all vnodes exhausted, prj query completed. total res:%d", pSql, totalVnode, pRes->numOfTotal);
+//  }
 }
 
 void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h
index 08d30be925..4087be49ee 100644
--- a/src/query/inc/qsqltype.h
+++ b/src/query/inc/qsqltype.h
@@ -61,7 +61,7 @@ enum _sql_type {
   TSDB_SQL_LOCAL,  // SQL below for client local
   TSDB_SQL_DESCRIBE_TABLE,
   TSDB_SQL_RETRIEVE_LOCALMERGE,
-  TSDB_SQL_METRIC_JOIN_RETRIEVE,
+  TSDB_SQL_TABLE_JOIN_RETRIEVE,
 
   /*
    * build empty result instead of accessing dnode to fetch result

From b320961643bb76016b2a831fde1d46d767d5a110 Mon Sep 17 00:00:00 2001
From: changshuaiqiang <sqchang@taosdata.com>
Date: Wed, 13 May 2020 17:03:38 +0800
Subject: [PATCH 28/33] test: add cases

---
 tests/pytest/insert/multi.py            | 51 ++++++++++++++
 tests/pytest/table/__init__.py          |  0
 tests/pytest/table/alter_column.py      | 88 +++++++++++++++++++++++++
 tests/pytest/table/del_stable.py        | 14 +++-
 tests/script/general/user/authority.sim | 12 +---
 5 files changed, 154 insertions(+), 11 deletions(-)
 create mode 100644 tests/pytest/insert/multi.py
 create mode 100644 tests/pytest/table/__init__.py
 create mode 100644 tests/pytest/table/alter_column.py

diff --git a/tests/pytest/insert/multi.py b/tests/pytest/insert/multi.py
new file mode 100644
index 0000000000..7725ccd210
--- /dev/null
+++ b/tests/pytest/insert/multi.py
@@ -0,0 +1,51 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdLog.info("create table")
+
+        tdSql.execute(
+            "create table if not exists st(ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            "CREATE TABLE if not exists dev_001 using st tags('dev_01')")
+
+        print("==============step2")
+        tdLog.info("multiple inserts")
+        tdSql.execute(
+            "INSERT INTO dev_001 VALUES ('2020-05-13 10:00:00.000', 1),('2020-05-13 10:00:00.001', 1)")
+        tdSql.checkAffectedRows(2)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/table/__init__.py b/tests/pytest/table/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/pytest/table/alter_column.py b/tests/pytest/table/alter_column.py
new file mode 100644
index 0000000000..15bc9f7aa7
--- /dev/null
+++ b/tests/pytest/table/alter_column.py
@@ -0,0 +1,88 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, db_test.stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdLog.info("create database and table")
+        tdSql.execute("create database db_test")
+        tdSql.execute(
+            "create table if not exists db_test.st(ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            "CREATE TABLE if not exists db_test.dev_001 using db_test.st tags('dev_01')")
+
+        print("==============step2")
+        tdLog.info("alter table add column")
+        tdSql.execute(
+            "ALTER TABLE db_test.st add COLUMN tag_version nchar(20)")
+        tdSql.query("describe db_test.st")
+        tdSql.checkRows(4)
+
+        print("==============step3")
+        tdLog.info("alter table drop column")
+        tdSql.execute("ALTER TABLE db_test.st drop COLUMN tag_version")
+        tdSql.query("describe db_test.st")
+        tdSql.checkRows(3)
+
+        print("==============step4")
+        tdLog.info("drop table")
+        tdSql.execute("drop table db_test.st")
+        tdSql.execute(
+            "create table if not exists db_test.st(ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            "CREATE TABLE if not exists db_test.dev_001 using db_test.st tags('dev_01')")
+        tdSql.execute(
+            "INSERT INTO db_test.dev_001 VALUES ('2020-05-13 10:00:00.000', 1)")
+        tdSql.query("select * from db_test.dev_001")
+        tdSql.checkRows(1)
+
+        print("==============step5")
+        tdLog.info("alter table add column")
+        tdSql.execute(
+            "ALTER TABLE db_test.st add COLUMN tag_version nchar(20)")
+        tdSql.query("describe db_test.st")
+        tdSql.checkRows(4)
+
+        tdSql.execute(
+            "INSERT INTO db_test.dev_001 VALUES ('2020-05-13 10:00:00.010', 1, '1.2.1')")
+        tdSql.query("select * from db_test.st")
+        tdSql.checkRows(2)
+
+        print("==============step6")
+        tdLog.info("alter table drop column")
+        tdSql.execute("ALTER TABLE db_test.st drop COLUMN tag_version")
+        tdSql.query("describe db_test.st")
+        tdSql.checkRows(3)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/table/del_stable.py b/tests/pytest/table/del_stable.py
index c81d598638..de30e6bdfe 100644
--- a/tests/pytest/table/del_stable.py
+++ b/tests/pytest/table/del_stable.py
@@ -38,8 +38,18 @@ class TDTestCase:
             tdSql.execute("drop table db.st")
         except Exception as e:
             tdLog.exit(e)
-        tdSql.execute("select * from db.st")
-        tdSql.checkRows(1)
+
+        try:
+            tdSql.execute("select * from db.st")
+        except Exception as e:
+            if e.args[0] != 'invalid table name':
+                tdLog.exit(e)
+
+        try:
+            tdSql.execute("select * from db.tb")
+        except Exception as e:
+            if e.args[0] != 'invalid table name':
+                tdLog.exit(e)
 
     def stop(self):
         tdSql.close()
diff --git a/tests/script/general/user/authority.sim b/tests/script/general/user/authority.sim
index 06f0467626..7ae8c6f1a4 100644
--- a/tests/script/general/user/authority.sim
+++ b/tests/script/general/user/authority.sim
@@ -30,7 +30,7 @@ sleep 2000
 
 sql create database dwrite
 sql show databases
-if $rows != 1 then
+if $rows != 2 then
   return -1
 endi
 
@@ -48,25 +48,19 @@ print ============ step5
 sql close
 sql connect read
 sleep 2000
-
 sql drop database dread
-sql drop database dwrite -x step51
+sql drop database dwrite
 
-step51:
 
 sql close
 sql connect
 sql show databases
-if $rows != 1 then
+if $rows != 0 then
   return -1
 endi
 
 sql close
 sql connect
 sleep 2000
-sql drop database d1
-sql drop database d2
-sql drop database d3
-sql drop database d4
 
 system sh/exec_up.sh -n dnode1 -s stop  -x SIGINT
\ No newline at end of file

From a23b0b0cd8a9ff316acbb20b08c0741b605195ee Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 17:24:06 +0800
Subject: [PATCH 29/33] replace sdb func

---
 src/client/src/tscSub.c            |   2 -
 src/dnode/src/dnodeMgmt.c          |   2 +-
 src/inc/vnode.h                    |   1 +
 src/mnode/src/mgmtSdb.c            |   2 -
 src/plugins/http/src/httpHandle.c  |   1 -
 src/plugins/http/src/httpSession.c |  69 +++----
 src/plugins/http/src/httpSystem.c  |   1 -
 src/plugins/http/src/httpUtil.c    |   1 -
 src/plugins/http/src/tgHandle.c    |   1 -
 src/query/src/tvariant.c           |   2 +-
 src/util/inc/hashint.h             |  26 ---
 src/util/inc/hashstr.h             |  26 ---
 src/util/inc/ihash.h               |  47 -----
 src/util/inc/shash.h               |  51 -----
 src/util/src/hashint.c             | 201 -------------------
 src/util/src/hashstr.c             | 214 --------------------
 src/util/src/ihash.c               | 290 ---------------------------
 src/util/src/shash.c               | 310 -----------------------------
 src/vnode/src/vnodeMain.c          |  55 ++---
 19 files changed, 69 insertions(+), 1233 deletions(-)
 delete mode 100644 src/util/inc/hashint.h
 delete mode 100644 src/util/inc/hashstr.h
 delete mode 100644 src/util/inc/ihash.h
 delete mode 100644 src/util/inc/shash.h
 delete mode 100644 src/util/src/hashint.c
 delete mode 100644 src/util/src/hashstr.c
 delete mode 100644 src/util/src/ihash.c
 delete mode 100644 src/util/src/shash.c

diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 856b678391..d003fa0a9d 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -14,8 +14,6 @@
  */
 
 #include "os.h"
-
-#include "shash.h"
 #include "taos.h"
 #include "trpc.h"
 #include "tsclient.h"
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 36a7c98807..f34acdeae3 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "ihash.h"
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "ttime.h"
@@ -174,6 +173,7 @@ static int32_t dnodeOpenVnodes() {
   int32_t numOfVnodes;
   int32_t status;
 
+  vnodeInit();
   status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
 
   if (status != TSDB_CODE_SUCCESS) {
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index f4fb8060fe..7ca17cafcc 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -35,6 +35,7 @@ typedef struct {
   void *qhandle; //used by query and retrieve msg
 } SRspRet;
 
+void    vnodeInit();
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeDrop(int32_t vgId);
 int32_t vnodeOpen(int32_t vgId, char *rootDir);
diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c
index 3a9ef390f9..0520bf8493 100644
--- a/src/mnode/src/mgmtSdb.c
+++ b/src/mnode/src/mgmtSdb.c
@@ -24,8 +24,6 @@
 #include "twal.h"
 #include "tsync.h"
 #include "tglobal.h"
-#include "hashint.h"
-#include "hashstr.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtInt.h"
diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c
index 4eaff3660a..8323ef7f4e 100644
--- a/src/plugins/http/src/httpHandle.c
+++ b/src/plugins/http/src/httpHandle.c
@@ -19,7 +19,6 @@
 #include "tglobal.h"
 #include "tsocket.h"
 #include "ttimer.h"
-#include "shash.h"
 #include "http.h"
 #include "httpLog.h"
 #include "httpCode.h"
diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c
index 4b095aeb77..e80d6f26b7 100644
--- a/src/plugins/http/src/httpSession.c
+++ b/src/plugins/http/src/httpSession.c
@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "shash.h"
+#include "hash.h"
 #include "taos.h"
 #include "ttime.h"
 #include "ttimer.h"
@@ -25,7 +25,6 @@
 #include "httpHandle.h"
 #include "httpResp.h"
 
-
 void httpAccessSession(HttpContext *pContext) {
   HttpServer *server = pContext->pThread->pServer;
   pthread_mutex_lock(&server->serverMutex);
@@ -50,8 +49,11 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
   session.taos = taos;
   session.expire = (int)taosGetTimestampSec() + server->sessionExpire;
   session.access = 1;
-  snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-  pContext->session = (HttpSession *)taosAddStrHash(server->pSessionHash, session.id, (char *)(&session));
+  int sessionIdLen = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+
+  taosHashPut(server->pSessionHash, session.id, sessionIdLen, (char *)(&session), sizeof(HttpSession));
+  pContext->session = taosHashGet(server->pSessionHash, session.id, sessionIdLen);
+
   if (pContext->session == NULL) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
               httpMsg[HTTP_SESSION_FULL]);
@@ -71,9 +73,9 @@ void httpFetchSessionImp(HttpContext *pContext) {
   pthread_mutex_lock(&server->serverMutex);
 
   char sessionId[HTTP_SESSION_ID_LEN];
-  snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+  int sessonIdLen = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
 
-  pContext->session = (HttpSession *)taosGetStrHashData(server->pSessionHash, sessionId);
+  pContext->session = taosHashGet(server->pSessionHash, sessionId, sessonIdLen);
   if (pContext->session != NULL && pContext->session == pContext->session->signature) {
     pContext->session->access++;
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d",
@@ -120,8 +122,7 @@ void httpRestoreSession(HttpContext *pContext) {
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpResetSession(char *session) {
-  HttpSession *pSession = (HttpSession *)session;
+void httpResetSession(HttpSession *pSession) {
   httpTrace("close session:%p:%p", pSession, pSession->taos);
   if (pSession->taos != NULL) {
     taos_close(pSession->taos);
@@ -131,15 +132,20 @@ void httpResetSession(char *session) {
 }
 
 void httpRemoveAllSessions(HttpServer *pServer) {
-  if (pServer->pSessionHash != NULL) {
-    taosCleanUpStrHashWithFp(pServer->pSessionHash, httpResetSession);
-    pServer->pSessionHash = NULL;
+  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
+
+  while (taosHashIterNext(pIter)) {
+    HttpSession *pSession = taosHashIterGet(pIter);
+    if (pSession == NULL) continue;
+    httpResetSession(pSession);
   }
+
+  taosHashDestroyIter(pIter);
 }
 
 bool httpInitAllSessions(HttpServer *pServer) {
   if (pServer->pSessionHash == NULL) {
-    pServer->pSessionHash = taosInitStrHash(100, sizeof(HttpSession), taosHashStringStep1);
+    pServer->pSessionHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true);
   }
   if (pServer->pSessionHash == NULL) {
     httpError("http init session pool failed");
@@ -152,46 +158,41 @@ bool httpInitAllSessions(HttpServer *pServer) {
   return true;
 }
 
-int httpSessionExpired(char *session) {
-  HttpSession *pSession = (HttpSession *)session;
-  time_t       cur = taosGetTimestampSec();
+bool httpSessionExpired(HttpSession *pSession) {
+  time_t cur = taosGetTimestampSec();
 
   if (pSession->taos != NULL) {
     if (pSession->expire > cur) {
-      return 0;  // un-expired, so return false
+      return false;  // un-expired, so return false
     }
     if (pSession->access > 0) {
       httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos,
                 pSession->access);
-      return 0;  // still used, so return false
+      return false;  // still used, so return false
     }
     httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d",
               pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire);
   }
 
-  return 1;
+  return true;
 }
 
-void httpRemoveExpireSessions(HttpServer *pServer) {
-  int expiredNum = 0;
-  do {
+void httpRemoveExpireSessions(HttpServer *pServer) {  
+  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
+
+  while (taosHashIterNext(pIter)) {
+    HttpSession *pSession = taosHashIterGet(pIter);
+    if (pSession == NULL) continue;
+
     pthread_mutex_lock(&pServer->serverMutex);
-
-    HttpSession *pSession = (HttpSession *)taosVisitStrHashWithFp(pServer->pSessionHash, httpSessionExpired);
-    if (pSession == NULL) {
-      pthread_mutex_unlock(&pServer->serverMutex);
-      break;
+    if (httpSessionExpired(pSession)) {
+      httpResetSession(pSession);
+      taosHashRemove(pServer->pSessionHash, pSession->id, strlen(pSession->id));
     }
-
-    httpResetSession((char *)pSession);
-    taosDeleteStrHashNode(pServer->pSessionHash, pSession->id, pSession);
-
     pthread_mutex_unlock(&pServer->serverMutex);
+  }
 
-    if (++expiredNum > 10) {
-      break;
-    }
-  } while (true);
+  taosHashDestroyIter(pIter);
 }
 
 void httpProcessSessionExpire(void *handle, void *tmrId) {
diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c
index 8b95077632..e7a5344be5 100644
--- a/src/plugins/http/src/httpSystem.c
+++ b/src/plugins/http/src/httpSystem.c
@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "shash.h"
 #include "taos.h"
 #include "tglobal.h"
 #include "tsocket.h"
diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c
index fc5f771c69..77fc399272 100644
--- a/src/plugins/http/src/httpUtil.c
+++ b/src/plugins/http/src/httpUtil.c
@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tmd5.h"
-#include "shash.h"
 #include "taos.h"
 #include "http.h"
 #include "httpLog.h"
diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c
index c854e0adc3..1a55c325d5 100644
--- a/src/plugins/http/src/tgHandle.c
+++ b/src/plugins/http/src/tgHandle.c
@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tglobal.h"
-#include "shash.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tgHandle.h"
diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c
index 51d3286722..6f8d579936 100644
--- a/src/query/src/tvariant.c
+++ b/src/query/src/tvariant.c
@@ -17,7 +17,7 @@
 #include "hash.h"
 #include "hashfunc.h"
 #include "os.h"
-#include "shash.h"
+#include "hash.h"
 #include "taos.h"
 #include "taosdef.h"
 #include "tstoken.h"
diff --git a/src/util/inc/hashint.h b/src/util/inc/hashint.h
deleted file mode 100644
index 052689f337..0000000000
--- a/src/util/inc/hashint.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _sdb_int_hash_header_
-#define _sdb_int_hash_header_
-
-void *sdbOpenIntHash(int maxSessions, int dataSize);
-void sdbCloseIntHash(void *handle);
-void *sdbAddIntHash(void *handle, void *key, void *pData);
-void sdbDeleteIntHash(void *handle, void *key);
-void *sdbGetIntHashData(void *handle, void *key);
-void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta);
-
-#endif
diff --git a/src/util/inc/hashstr.h b/src/util/inc/hashstr.h
deleted file mode 100644
index 66f6e88857..0000000000
--- a/src/util/inc/hashstr.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _sdb_str_hash_header_
-#define _sdb_str_hash_header_
-
-void *sdbOpenStrHash(int maxSessions, int dataSize);
-void sdbCloseStrHash(void *handle);
-void *sdbAddStrHash(void *handle, void *key, void *pData);
-void sdbDeleteStrHash(void *handle, void *key);
-void *sdbGetStrHashData(void *handle, void *key);
-void *sdbFetchStrHashData(void *handle, void *ptr, void **ppMeta);
-
-#endif
diff --git a/src/util/inc/ihash.h b/src/util/inc/ihash.h
deleted file mode 100644
index f283abe737..0000000000
--- a/src/util/inc/ihash.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_IHASH_H
-#define TDENGINE_IHASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void *, uint64_t));
-
-void taosCleanUpIntHash(void *handle);
-
-char *taosGetIntHashData(void *handle, uint64_t key);
-
-void taosDeleteIntHash(void *handle, uint64_t key);
-
-char *taosAddIntHash(void *handle, uint64_t key, char *pData);
-
-int32_t taosHashInt(void *handle, uint64_t key);
-
-void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *));
-
-void taosVisitIntHashWithFp(void *handle, void (*fp)(char *, void *), void *param);
-
-int32_t taosGetIntHashSize(void *handle);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_IHASH_H
diff --git a/src/util/inc/shash.h b/src/util/inc/shash.h
deleted file mode 100644
index 2651a6b349..0000000000
--- a/src/util/inc/shash.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TSHASH_H
-#define TDENGINE_TSHASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-void *taosInitStrHash(uint32_t maxSessions, uint32_t dataSize, uint32_t (*fp)(void *, char *));
-
-void taosCleanUpStrHash(void *handle);
-
-void *taosGetStrHashData(void *handle, char *string);
-
-void taosDeleteStrHash(void *handle, char *string);
-
-void taosDeleteStrHashNode(void *handle, char *string, void *pDeleteNode);
-
-void *taosAddStrHash(void *handle, char *string, char *pData);
-
-void *taosAddStrHashWithSize(void *handle, char *string, char *pData, int dataSize);
-
-uint32_t taosHashString(void *handle, char *string);
-
-uint32_t taosHashStringStep1(void *handle, char *string);
-
-char *taosVisitStrHashWithFp(void *handle, int (*fp)(char *));
-
-void taosCleanUpStrHashWithFp(void *handle, void (*fp)(char *));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TSHASH_H
diff --git a/src/util/src/hashint.c b/src/util/src/hashint.c
deleted file mode 100644
index 87a6da4fa8..0000000000
--- a/src/util/src/hashint.c
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <pthread.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "tmempool.h"
-#include "taosdef.h"
-
-typedef struct _long_hash_t {
-  uint32_t             key;
-  int                  hash;
-  struct _long_hash_t *prev;
-  struct _long_hash_t *next;
-  char                 data[];
-} SLongHash;
-
-typedef struct {
-  SLongHash **longHashList;
-  mpool_h     longHashMemPool;
-  int         maxSessions;
-  int         dataSize;
-} SHashObj;
-
-int sdbHashLong(void *handle, uint32_t ip) {
-  SHashObj *pObj = (SHashObj *)handle;
-  int       hash = 0;
-
-  hash = ip >> 16;
-  hash += (ip & 0xFFFF);
-
-  hash = hash % pObj->maxSessions;
-
-  return hash;
-}
-
-void *sdbAddIntHash(void *handle, void *pKey, void *data) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-  uint32_t   key = *((uint32_t *)pKey);
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = sdbHashLong(pObj, key);
-  pNode = (SLongHash *)taosMemPoolMalloc(pObj->longHashMemPool);
-  pNode->key = key;
-  memcpy(pNode->data, data, pObj->dataSize);
-  pNode->prev = 0;
-  pNode->next = pObj->longHashList[hash];
-  pNode->hash = hash;
-
-  if (pObj->longHashList[hash] != 0) (pObj->longHashList[hash])->prev = pNode;
-  pObj->longHashList[hash] = pNode;
-
-  return pObj;
-}
-
-void sdbDeleteIntHash(void *handle, void *pKey) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-  uint32_t   key = *((uint32_t *)pKey);
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  hash = sdbHashLong(pObj, key);
-
-  pNode = pObj->longHashList[hash];
-  while (pNode) {
-    if (pNode->key == key) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->longHashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    taosMemPoolFree(pObj->longHashMemPool, (char *)pNode);
-  }
-}
-
-void *sdbGetIntHashData(void *handle, void *pKey) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-  uint32_t   key = *((uint32_t *)pKey);
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = sdbHashLong(pObj, key);
-  pNode = pObj->longHashList[hash];
-
-  while (pNode) {
-    if (pNode->key == key) {
-      break;
-    }
-    pNode = pNode->next;
-  }
-
-  if (pNode) return pNode->data;
-
-  return NULL;
-}
-
-void *sdbOpenIntHash(int maxSessions, int dataSize) {
-  SLongHash **longHashList;
-  mpool_h     longHashMemPool;
-  SHashObj *  pObj;
-
-  longHashMemPool = taosMemPoolInit(maxSessions, sizeof(SLongHash) + dataSize);
-  if (longHashMemPool == 0) return NULL;
-
-  longHashList = calloc(sizeof(SLongHash *), maxSessions);
-  if (longHashList == 0) {
-    taosMemPoolCleanUp(longHashMemPool);
-    return NULL;
-  }
-
-  pObj = malloc(sizeof(SHashObj));
-  if (pObj == NULL) {
-    taosMemPoolCleanUp(longHashMemPool);
-    free(longHashList);
-    return NULL;
-  }
-
-  pObj->maxSessions = maxSessions;
-  pObj->longHashMemPool = longHashMemPool;
-  pObj->longHashList = longHashList;
-  pObj->dataSize = dataSize;
-
-  return pObj;
-}
-
-void sdbCloseIntHash(void *handle) {
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  if (pObj->longHashMemPool) taosMemPoolCleanUp(pObj->longHashMemPool);
-
-  if (pObj->longHashList) free(pObj->longHashList);
-
-  memset(pObj, 0, sizeof(SHashObj));
-  free(pObj);
-}
-
-void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta) {
-  SHashObj * pObj = (SHashObj *)handle;
-  SLongHash *pNode = (SLongHash *)ptr;
-  int        hash = 0;
-
-  *ppMeta = NULL;
-  if (pObj == NULL || pObj->maxSessions <= 0) return NULL;
-  if (pObj->longHashList == NULL) return NULL;
-
-  if (pNode) {
-    hash = pNode->hash + 1;
-    pNode = pNode->next;
-  }
-
-  if (pNode == NULL) {
-    for (int i = hash; i < pObj->maxSessions; ++i) {
-      pNode = pObj->longHashList[i];
-      if (pNode) break;
-    }
-  }
-
-  if (pNode) *ppMeta = pNode->data;
-
-  return pNode;
-}
diff --git a/src/util/src/hashstr.c b/src/util/src/hashstr.c
deleted file mode 100644
index d87e605176..0000000000
--- a/src/util/src/hashstr.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "taosdef.h"
-
-#define MAX_STR_LEN 40
-
-typedef struct _str_node_t {
-  char                string[TSDB_TABLE_ID_LEN];
-  int                 hash;
-  struct _str_node_t *prev;
-  struct _str_node_t *next;
-  char                data[];
-} SHashNode;
-
-typedef struct {
-  SHashNode **hashList;
-  int         maxSessions;
-  int         dataSize;
-} SHashObj;
-
-int sdbHashString(void *handle, char *string) {
-  SHashObj *   pObj = (SHashObj *)handle;
-  unsigned int hash = 0, hashv;
-  char *       c;
-  int          len = strlen(string);
-
-  c = string;
-
-  while (len >= 4) {
-    hash += *((int *)c);
-    c += 4;
-    len -= 4;
-  }
-
-  while (len > 0) {
-    hash += *c;
-    c++;
-    len--;
-  }
-
-  hashv = hash / pObj->maxSessions;
-  hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions;
-
-  return hash;
-}
-
-void *sdbAddStrHash(void *handle, void *key, void *pData) {
-  int        hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-  char *     string = (char *)key;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = sdbHashString(pObj, string);
-
-  int size = sizeof(SHashNode) + pObj->dataSize;
-  pNode = (SHashNode *)malloc(size);
-  memset(pNode, 0, size);
-  strcpy(pNode->string, string);
-  memcpy(pNode->data, pData, pObj->dataSize);
-  pNode->prev = 0;
-  pNode->next = pObj->hashList[hash];
-  pNode->hash = hash;
-
-  if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode;
-  pObj->hashList[hash] = pNode;
-
-  return pNode->data;
-}
-
-void sdbDeleteStrHash(void *handle, void *key) {
-  int        hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-  char *     string = (char *)key;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  hash = sdbHashString(pObj, string);
-  pNode = pObj->hashList[hash];
-  while (pNode) {
-    if (strcmp(pNode->string, string) == 0) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->hashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    memset(pNode, 0, sizeof(SHashNode));
-    free(pNode);
-  }
-}
-
-void *sdbGetStrHashData(void *handle, void *key) {
-  int        hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-  char *     string = (char *)key;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = sdbHashString(pObj, string);
-  pNode = pObj->hashList[hash];
-
-  while (pNode) {
-    if (strcmp(pNode->string, string) == 0) {
-      break;
-    }
-    pNode = pNode->next;
-  }
-
-  if (pNode) return pNode->data;
-
-  return NULL;
-}
-
-void *sdbOpenStrHash(int maxSessions, int dataSize) {
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)malloc(sizeof(SHashObj));
-  if (pObj == NULL) {
-    return NULL;
-  }
-
-  memset(pObj, 0, sizeof(SHashObj));
-  pObj->maxSessions = maxSessions;
-  pObj->dataSize = dataSize;
-
-  pObj->hashList = (SHashNode **)malloc(sizeof(SHashNode *) * maxSessions);
-  if (pObj->hashList == NULL) {
-    free(pObj);
-    return NULL;
-  }
-  memset(pObj->hashList, 0, sizeof(SHashNode *) * maxSessions);
-
-  return (void *)pObj;
-}
-
-void sdbCloseStrHash(void *handle) {
-  SHashObj  *pObj;
-  SHashNode *pNode, *pNext;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return;
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        free(pNode);
-        pNode = pNext;
-      }
-    }
-
-    free(pObj->hashList);
-  }
-
-  memset(pObj, 0, sizeof(SHashObj));
-  free(pObj);
-}
-
-void *sdbFetchStrHashData(void *handle, void *ptr, void **ppMeta) {
-  SHashObj  *pObj = (SHashObj *)handle;
-  SHashNode *pNode = (SHashNode *)ptr;
-  int        hash = 0;
-
-  *ppMeta = NULL;
-  if (pObj == NULL || pObj->maxSessions <= 0) return NULL;
-  if (pObj->hashList == NULL) return NULL;
-
-  if (pNode) {
-    hash = pNode->hash + 1;
-    pNode = pNode->next;
-  }
-
-  if (pNode == NULL) {
-    for (int i = hash; i < pObj->maxSessions; ++i) {
-      pNode = pObj->hashList[i];
-      if (pNode) break;
-    }
-  }
-
-  if (pNode) *ppMeta = pNode->data;
-
-  return pNode;
-}
diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c
deleted file mode 100644
index 2cfadad964..0000000000
--- a/src/util/src/ihash.c
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-
-typedef struct _str_node_t {
-  uint64_t            key;
-  struct _str_node_t *prev;
-  struct _str_node_t *next;
-  char                data[];
-} IHashNode;
-
-typedef struct {
-  IHashNode **hashList;
-  int64_t    *lockedBy;
-  int32_t     maxSessions;
-  int32_t     dataSize;
-  int32_t   (*hashFp)(void *, uint64_t key);
-} IHashObj;
-
-int32_t taosHashInt(void *handle, uint64_t key) {
-  IHashObj *pObj = (IHashObj *)handle;
-  int32_t   hash = key % pObj->maxSessions;
-  return hash;
-}
-
-static void taosLockIntHash(IHashObj *pObj, int hash);
-static void taosUnlockIntHash(IHashObj *pObj, int hash);
-
-char *taosAddIntHash(void *handle, uint64_t key, char *pData) {
-  int32_t    hash;
-  IHashNode *pNode;
-  IHashObj * pObj;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = (*pObj->hashFp)(pObj, key);
-
-  pNode = (IHashNode *)malloc(sizeof(IHashNode) + (size_t)pObj->dataSize);
-  if (pNode == NULL)
-    return NULL;
-  
-  taosLockIntHash(pObj, hash);
-
-  pNode->key = key;
-  if (pData != NULL) {
-    memcpy(pNode->data, pData, (size_t)pObj->dataSize);
-  }
-  pNode->prev = 0;
-  pNode->next = pObj->hashList[hash];
-
-  if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode;
-  pObj->hashList[hash] = pNode;
-
-  taosUnlockIntHash(pObj, hash);
-
-  return (char *)pNode->data;
-}
-
-void taosDeleteIntHash(void *handle, uint64_t key) {
-  int32_t    hash;
-  IHashNode *pNode;
-  IHashObj * pObj;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  hash = (*(pObj->hashFp))(pObj, key);
-
-  taosLockIntHash(pObj, hash);
-
-  pNode = pObj->hashList[hash];
-  while (pNode) {
-    if (pNode->key == key) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->hashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    free(pNode);
-  }
-
-  taosUnlockIntHash(pObj, hash);
-}
-
-char *taosGetIntHashData(void *handle, uint64_t key) {
-  int32_t    hash;
-  IHashNode *pNode;
-  IHashObj * pObj;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-
-  hash = (*pObj->hashFp)(pObj, key);
-
-  taosLockIntHash(pObj, hash);
-
-  pNode = pObj->hashList[hash];
-
-  while (pNode) {
-    if (pNode->key == key) {
-      break;
-    }
-
-    pNode = pNode->next;
-  }
-
-  taosUnlockIntHash(pObj, hash);
-
-  if (pNode) return pNode->data;
-
-  return NULL;
-}
-
-void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void *, uint64_t)) {
-  IHashObj *pObj;
-
-  pObj = (IHashObj *)malloc(sizeof(IHashObj));
-  if (pObj == NULL) {
-    return NULL;
-  }
-
-  memset(pObj, 0, sizeof(IHashObj));
-  pObj->maxSessions = maxSessions;
-  pObj->dataSize = dataSize;
-  pObj->hashFp = fp;
-
-  pObj->hashList = (IHashNode **)malloc(sizeof(IHashNode *) * (size_t)maxSessions);
-  if (pObj->hashList == NULL) {
-    free(pObj);
-    return NULL;
-  }
-  memset(pObj->hashList, 0, sizeof(IHashNode *) * (size_t)maxSessions);
-
-  pObj->lockedBy = (int64_t *)calloc(sizeof(int64_t), maxSessions);
-  if (pObj->lockedBy == NULL) {
-    free(pObj);
-    free(pObj->hashList);
-    pObj = NULL;
-  }
-
-  return pObj;
-}
-
-void taosCleanUpIntHash(void *handle) {
-  IHashObj * pObj;
-  IHashNode *pNode, *pNext;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return;
-
-  if (pObj->hashList) {
-    for (int32_t i = 0; i < pObj->maxSessions; ++i) {
-      taosLockIntHash(pObj, i);
-
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        free(pNode);
-        pNode = pNext;
-      }
-
-      taosUnlockIntHash(pObj, i);
-    }
-
-    free(pObj->hashList);
-  }
-
-  free(pObj->lockedBy);
-  free(pObj);
-}
-
-void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) {
-  IHashObj * pObj;
-  IHashNode *pNode, *pNext;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return;
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      taosLockIntHash(pObj, i);
-
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        if (fp != NULL) (*fp)(pNode->data);
-        free(pNode);
-        pNode = pNext;
-      }
-
-      taosUnlockIntHash(pObj, i);
-    }
-
-    free(pObj->hashList);
-  }
-
-  memset(pObj, 0, sizeof(IHashObj));
-  free(pObj);
-}
-
-void taosVisitIntHashWithFp(void *handle, int (*fp)(char *, void *), void *param) {
-  IHashObj * pObj;
-  IHashNode *pNode, *pNext;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return;
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      taosLockIntHash(pObj, i);
-
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        (*fp)(pNode->data, param);
-        pNode = pNext;
-      }
-
-      taosUnlockIntHash(pObj, i);
-    }
-  }
-}
-
-int32_t taosGetIntHashSize(void *handle) {
-  IHashObj * pObj;
-  IHashNode *pNode, *pNext;
-  int32_t    num = 0;
-
-  pObj = (IHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return 0;
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      taosLockIntHash(pObj, i);
-
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        num++;
-        pNode = pNext;
-      }
-
-      taosUnlockIntHash(pObj, i);
-    }
-  }
-
-  return num;
-}
-
-static void taosLockIntHash(IHashObj *pObj, int hash) {
-  int64_t tid = taosGetPthreadId();
-  int     i = 0;
-  while (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), 0, tid) != 0) {
-    if (++i % 1000 == 0) {
-      sched_yield();
-    }
-  }
-}
-
-static void taosUnlockIntHash(IHashObj *pObj, int hash) {
-  int64_t tid = taosGetPthreadId();
-  if (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), tid, 0) != tid) {
-    assert(false);
-  }
-}
-
diff --git a/src/util/src/shash.c b/src/util/src/shash.c
deleted file mode 100644
index 807ba58951..0000000000
--- a/src/util/src/shash.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "shash.h"
-#include "tulog.h"
-
-typedef struct _str_node_t {
-  char *              string;
-  struct _str_node_t *prev;
-  struct _str_node_t *next;
-  char                data[];
-} SHashNode;
-
-typedef struct {
-  SHashNode **hashList;
-  uint32_t    maxSessions;
-  uint32_t    dataSize;
-  uint32_t  (*hashFp)(void *, char *string);
-  pthread_mutex_t mutex;
-} SHashObj;
-
-uint32_t taosHashString(void *handle, char *string) {
-  SHashObj *pObj = (SHashObj *)handle;
-  uint32_t  hash = 0, hashv;
-  char *    c;
-
-  c = string;
-  while (*c) {
-    hash += *((int *)c);
-    c += 4;
-  }
-
-  hashv = hash / pObj->maxSessions;
-  hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions;
-
-  return hash;
-}
-
-uint32_t taosHashStringStep1(void *handle, char *string) {
-  SHashObj *pObj = (SHashObj *)handle;
-  uint32_t  hash = 0, hashv;
-  char *    c;
-
-  c = string;
-  while (*c) {
-    hash += *c;
-    c++;
-  }
-
-  hashv = hash / pObj->maxSessions;
-  hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions;
-
-  return hash;
-}
-
-void *taosAddStrHashWithSize(void *handle, char *string, char *pData, int dataSize) {
-  uint32_t   hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-  if (string == NULL || string[0] == 0) return NULL;
-
-  hash = (*pObj->hashFp)(pObj, string);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = (SHashNode *)malloc(sizeof(SHashNode) + (size_t)dataSize + strlen(string) + 1);
-  memcpy(pNode->data, pData, (size_t)dataSize);
-  pNode->prev = 0;
-  pNode->next = pObj->hashList[hash];
-  pNode->string = pNode->data + dataSize;
-  strcpy(pNode->string, string);
-
-  if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode;
-  pObj->hashList[hash] = pNode;
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  uTrace("hash:%d:%s is added", hash, string);
-
-  return pNode->data;
-}
-
-void *taosAddStrHash(void *handle, char *string, char *pData) {
-  if (string == NULL || string[0] == 0) return NULL;
-
-  SHashObj *pObj = (SHashObj *)handle;
-  return taosAddStrHashWithSize(handle, string, pData, pObj->dataSize);
-}
-
-void taosDeleteStrHashNode(void *handle, char *string, void *pDeleteNode) {
-  uint32_t   hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-  bool       find = false;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-  if (string == NULL || string[0] == 0) return;
-
-  hash = (*(pObj->hashFp))(pObj, string);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = pObj->hashList[hash];
-
-  while (pNode) {
-    if (strcmp(pNode->string, string) != 0) continue;
-    if (pNode->data == pDeleteNode) {
-      find = true;
-      break;
-    }
-
-    pNode = pNode->next;
-  }
-
-  if (find) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->hashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    uTrace("hash:%d:%s:%p is removed", hash, string, pNode);
-
-    free(pNode);
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-}
-
-void taosDeleteStrHash(void *handle, char *string) {
-  uint32_t   hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-  if (string == NULL || string[0] == 0) return;
-
-  hash = (*(pObj->hashFp))(pObj, string);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = pObj->hashList[hash];
-  while (pNode) {
-    if (strcmp(pNode->string, string) == 0) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->hashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    uTrace("hash:%d:%s:%p is removed", hash, string, pNode);
-
-    free(pNode);
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-}
-
-void *taosGetStrHashData(void *handle, char *string) {
-  uint32_t   hash;
-  SHashNode *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
-  if (string == NULL || string[0] == 0) return NULL;
-
-  hash = (*pObj->hashFp)(pObj, string);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = pObj->hashList[hash];
-
-  while (pNode) {
-    if (strcmp(pNode->string, string) == 0) {
-      uTrace("hash:%d:%s is retrieved", hash, string);
-      break;
-    }
-
-    pNode = pNode->next;
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  if (pNode) return pNode->data;
-
-  return NULL;
-}
-
-void *taosInitStrHash(uint32_t maxSessions, uint32_t dataSize, uint32_t (*fp)(void *, char *)) {
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)malloc(sizeof(SHashObj));
-  if (pObj == NULL) {
-    return NULL;
-  }
-
-  memset(pObj, 0, sizeof(SHashObj));
-  pObj->maxSessions = maxSessions;
-  pObj->dataSize = dataSize;
-  pObj->hashFp = fp;
-
-  pObj->hashList = (SHashNode **)malloc(sizeof(SHashNode *) * (size_t)maxSessions);
-  if (pObj->hashList == NULL) {
-    free(pObj);
-    return NULL;
-  }
-  memset(pObj->hashList, 0, sizeof(SHashNode *) * (size_t)maxSessions);
-
-  pthread_mutex_init(&pObj->mutex, NULL);
-
-  return pObj;
-}
-
-void taosCleanUpStrHashWithFp(void *handle, void (*fp)(char *)) {
-  SHashObj * pObj;
-  SHashNode *pNode, *pNext;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return;
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        if (fp != NULL) fp(pNode->data);
-        free(pNode);
-        pNode = pNext;
-      }
-    }
-
-    free(pObj->hashList);
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  pthread_mutex_destroy(&pObj->mutex);
-
-  memset(pObj, 0, sizeof(SHashObj));
-  free(pObj);
-}
-
-void taosCleanUpStrHash(void *handle) { taosCleanUpStrHashWithFp(handle, NULL); }
-
-char *taosVisitStrHashWithFp(void *handle, int (*fp)(char *)) {
-  SHashObj * pObj;
-  SHashNode *pNode, *pNext;
-  char *     pData = NULL;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions <= 0) return NULL;
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  if (pObj->hashList) {
-    for (int i = 0; i < pObj->maxSessions; ++i) {
-      pNode = pObj->hashList[i];
-      while (pNode) {
-        pNext = pNode->next;
-        int flag = fp(pNode->data);
-        if (flag) {
-          pData = pNode->data;
-          goto VisitEnd;
-        }
-
-        pNode = pNext;
-      }
-    }
-  }
-
-VisitEnd:
-
-  pthread_mutex_unlock(&pObj->mutex);
-  return pData;
-}
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 8d35be9640..20ec430916 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "ihash.h"
+#include "hash.h"
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tutil.h"
@@ -32,7 +32,6 @@
 static int32_t  tsOpennedVnodes;
 static void    *tsDnodeVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
-static void     vnodeBuildVloadMsg(char *pNode, void * param);
 static int      vnodeWalCallback(void *arg);
 static int32_t  vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
 static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
@@ -55,11 +54,11 @@ int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
 void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
 #endif
 
-static void vnodeInit() {
+void vnodeInit() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
-  tsDnodeVnodesHash = taosInitIntHash(TSDB_MAX_VNODES, sizeof(SVnodeObj *), taosHashInt);
+  tsDnodeVnodesHash = taosHashInit(TSDB_MAX_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
   if (tsDnodeVnodesHash == NULL) {
     vError("failed to init vnode list");
   }
@@ -69,7 +68,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
   int32_t code;
   pthread_once(&vnodeModuleInit, vnodeInit);
 
-  SVnodeObj *pTemp = (SVnodeObj *)taosGetIntHashData(tsDnodeVnodesHash, pVnodeCfg->cfg.vgId);
+  SVnodeObj *pTemp = (SVnodeObj *)taosHashGet(tsDnodeVnodesHash, (const char *)&pVnodeCfg->cfg.vgId, sizeof(int32_t));
   if (pTemp != NULL) {
     vPrint("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp);
     return TSDB_CODE_SUCCESS;
@@ -121,7 +120,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
 }
 
 int32_t vnodeDrop(int32_t vgId) {
-  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t));
   if (ppVnode == NULL || *ppVnode == NULL) {
     vTrace("vgId:%d, failed to drop, vgId not exist", vgId);
     return TSDB_CODE_INVALID_VGROUP_ID;
@@ -148,7 +147,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
   code = vnodeReadCfg(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
     vError("vgId:%d, failed to read cfg file", pVnode->vgId);
-    taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
+    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
     return code;
   }
 
@@ -183,12 +182,12 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pVnode->version  = 0;  
   pVnode->tsdbCfg.tsdbId = pVnode->vgId;
   pVnode->rootDir = strdup(rootDir);
-  taosAddIntHash(tsDnodeVnodesHash, pVnode->vgId, (char *)(&pVnode));
-  
+  taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
+
   int32_t code = vnodeReadCfg(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
     vError("vgId:%d, failed to read cfg file", pVnode->vgId);
-    taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
+    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
     return code;
   }
 
@@ -213,7 +212,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pVnode->tsdb = tsdbOpenRepo(temp, &appH);
   if (pVnode->tsdb == NULL) {
     vError("vgId:%d, failed to open tsdb at %s(%s)", pVnode->vgId, temp, tstrerror(terrno));
-    taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
+    taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
     return terrno;
   }
 
@@ -249,7 +248,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 }
 
 int32_t vnodeClose(int32_t vgId) {
-  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t));
   if (ppVnode == NULL || *ppVnode == NULL) return 0;
 
   SVnodeObj *pVnode = *ppVnode;
@@ -293,14 +292,14 @@ void vnodeRelease(void *pVnodeRaw) {
   vTrace("vgId:%d, vnode is released, vnodes:%d", vgId, count);
 
   if (count <= 0) {
-    taosCleanUpIntHash(tsDnodeVnodesHash);
+    taosHashCleanup(tsDnodeVnodesHash);
     vnodeModuleInit = PTHREAD_ONCE_INIT;
     tsDnodeVnodesHash = NULL;
   }
 }
 
 void *vnodeGetVnode(int32_t vgId) {
-  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t));
   if (ppVnode == NULL || *ppVnode == NULL) {
     terrno = TSDB_CODE_INVALID_VGROUP_ID;
     vPrint("vgId:%d, not exist", vgId);
@@ -334,16 +333,8 @@ void *vnodeGetWal(void *pVnode) {
   return ((SVnodeObj *)pVnode)->wal; 
 }
 
-void vnodeBuildStatusMsg(void *param) {
-  SDMStatusMsg *pStatus = param;
-  taosVisitIntHashWithFp(tsDnodeVnodesHash, vnodeBuildVloadMsg, pStatus);
-}
-
-static void vnodeBuildVloadMsg(char *pNode, void * param) {
-  SVnodeObj *pVnode = *(SVnodeObj **) pNode;
+static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
   if (pVnode->status == TAOS_VN_STATUS_DELETING) return;
-
-  SDMStatusMsg *pStatus = param;
   if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
 
   SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
@@ -357,8 +348,24 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) {
   pLoad->replica = pVnode->syncCfg.replica;
 }
 
+void vnodeBuildStatusMsg(void *param) {
+  SDMStatusMsg *pStatus = param;
+  SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+
+  while (taosHashIterNext(pIter)) {
+    SVnodeObj **pVnode = taosHashIterGet(pIter);
+    if (pVnode == NULL) continue;
+    if (*pVnode == NULL) continue;
+
+    vnodeBuildVloadMsg(*pVnode, pStatus);
+    pStatus++;
+  }
+
+  taosHashDestroyIter(pIter);
+}
+
 static void vnodeCleanUp(SVnodeObj *pVnode) {
-  taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
+  taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
 
   if (pVnode->sync) {
     syncStop(pVnode->sync);

From 5f86564a1c9fe9aaec2e65167ded22e18ccd7bd0 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 17:46:15 +0800
Subject: [PATCH 30/33] [149] fix bug while replace sdb

---
 src/dnode/src/dnodeMgmt.c |   1 -
 src/inc/vnode.h           |   1 -
 src/rpc/src/rpcUdp.c      |   1 -
 src/util/inc/thash.h      |  41 --------
 src/util/src/thash.c      | 192 --------------------------------------
 src/vnode/src/vnodeMain.c |   4 +-
 6 files changed, 3 insertions(+), 237 deletions(-)
 delete mode 100644 src/util/inc/thash.h
 delete mode 100644 src/util/src/thash.c

diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index f34acdeae3..81d426b496 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -173,7 +173,6 @@ static int32_t dnodeOpenVnodes() {
   int32_t numOfVnodes;
   int32_t status;
 
-  vnodeInit();
   status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
 
   if (status != TSDB_CODE_SUCCESS) {
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index 7ca17cafcc..f4fb8060fe 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -35,7 +35,6 @@ typedef struct {
   void *qhandle; //used by query and retrieve msg
 } SRspRet;
 
-void    vnodeInit();
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeDrop(int32_t vgId);
 int32_t vnodeOpen(int32_t vgId, char *rootDir);
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 8d5721a437..c551f6b1db 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -18,7 +18,6 @@
 #include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "thash.h"
 #include "rpcLog.h"
 #include "rpcHaship.h"
 #include "rpcUdp.h"
diff --git a/src/util/inc/thash.h b/src/util/inc/thash.h
deleted file mode 100644
index 592db62fe5..0000000000
--- a/src/util/inc/thash.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_THASH_H
-#define TDENGINE_THASH_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void *taosOpenHash(int maxSessions, int (*fp)(void *, uint64_t));
-
-void taosCloseHash(void *handle);
-
-int taosAddHash(void *handle, uint64_t, uint32_t id);
-
-void taosDeleteHash(void *handle, uint64_t);
-
-int32_t taosGetIdFromHash(void *handle, uint64_t);
-
-int taosHashLong(void *, uint64_t ip);
-
-uint64_t taosHashUInt64(uint64_t handle);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_THASH_H
diff --git a/src/util/src/thash.c b/src/util/src/thash.c
deleted file mode 100644
index e3c6fe26b4..0000000000
--- a/src/util/src/thash.c
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "tmempool.h"
-
-typedef struct _long_hash_t {
-  unsigned int         id;
-  struct _long_hash_t *prev;
-  struct _long_hash_t *next;
-  uint64_t             cont;
-} SLongHash;
-
-typedef struct {
-  SLongHash **longHashList;
-  mpool_h     longHashMemPool;
-  int (*hashFp)(void *, uint64_t);
-  int             maxSessions;
-  pthread_mutex_t mutex;
-} SHashObj;
-
-uint64_t taosHashUInt64(uint64_t handle) {
-  uint64_t hash = handle >> 16;
-  hash += handle & 0xFFFF;
-  return hash;
-}
-
-int taosHashLong(void *handle, uint64_t ip) {
-  SHashObj *pObj = (SHashObj *)handle;
-  int       hash = 0;
-
-  hash = (int)(ip >> 16);
-  hash += (int)(ip & 0xFFFF);
-
-  hash = hash % pObj->maxSessions;
-
-  return hash;
-}
-
-int taosAddHash(void *handle, uint64_t cont, unsigned int id) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return -1;
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  hash = (*pObj->hashFp)(pObj, cont);
-
-  pNode = (SLongHash *)taosMemPoolMalloc(pObj->longHashMemPool);
-  pNode->cont = cont;
-  pNode->id = id;
-  pNode->prev = 0;
-  pNode->next = pObj->longHashList[hash];
-
-  if (pObj->longHashList[hash] != 0) (pObj->longHashList[hash])->prev = pNode;
-  pObj->longHashList[hash] = pNode;
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  return 0;
-}
-
-void taosDeleteHash(void *handle, uint64_t cont) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  hash = (*pObj->hashFp)(pObj, cont);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = pObj->longHashList[hash];
-  while (pNode) {
-    if (pNode->cont == cont) break;
-
-    pNode = pNode->next;
-  }
-
-  if (pNode) {
-    if (pNode->prev) {
-      pNode->prev->next = pNode->next;
-    } else {
-      pObj->longHashList[hash] = pNode->next;
-    }
-
-    if (pNode->next) {
-      pNode->next->prev = pNode->prev;
-    }
-
-    taosMemPoolFree(pObj->longHashMemPool, (char *)pNode);
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-}
-
-int32_t taosGetIdFromHash(void *handle, uint64_t cont) {
-  int        hash;
-  SLongHash *pNode;
-  SHashObj * pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return -1;
-
-  hash = (*pObj->hashFp)(pObj, cont);
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  pNode = pObj->longHashList[hash];
-
-  while (pNode) {
-    if (pNode->cont == cont) {
-      break;
-    }
-
-    pNode = pNode->next;
-  }
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  if (pNode) return (int32_t)pNode->id;
-
-  return -1;
-}
-
-void *taosOpenHash(int maxSessions, int (*fp)(void *, uint64_t)) {
-  SLongHash **longHashList;
-  mpool_h     longHashMemPool;
-  SHashObj *  pObj;
-
-  longHashMemPool = taosMemPoolInit(maxSessions, sizeof(SLongHash));
-  if (longHashMemPool == 0) return NULL;
-
-  longHashList = calloc(sizeof(SLongHash *), (size_t)maxSessions);
-  if (longHashList == 0) {
-    taosMemPoolCleanUp(longHashMemPool);
-    return NULL;
-  }
-
-  pObj = malloc(sizeof(SHashObj));
-  if (pObj == NULL) {
-    taosMemPoolCleanUp(longHashMemPool);
-    free(longHashList);
-    return NULL;
-  }
-
-  pObj->maxSessions = maxSessions;
-  pObj->longHashMemPool = longHashMemPool;
-  pObj->longHashList = longHashList;
-  pObj->hashFp = fp;
-
-  pthread_mutex_init(&pObj->mutex, NULL);
-
-  return pObj;
-}
-
-void taosCloseHash(void *handle) {
-  SHashObj *pObj;
-
-  pObj = (SHashObj *)handle;
-  if (pObj == NULL || pObj->maxSessions == 0) return;
-
-  pthread_mutex_lock(&pObj->mutex);
-
-  if (pObj->longHashMemPool) taosMemPoolCleanUp(pObj->longHashMemPool);
-
-  if (pObj->longHashList) free(pObj->longHashList);
-
-  pthread_mutex_unlock(&pObj->mutex);
-
-  pthread_mutex_destroy(&pObj->mutex);
-
-  memset(pObj, 0, sizeof(SHashObj));
-  free(pObj);
-}
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 20ec430916..5b36fcba16 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -54,7 +54,7 @@ int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
 void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
 #endif
 
-void vnodeInit() {
+static void vnodeInit() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
@@ -299,6 +299,8 @@ void vnodeRelease(void *pVnodeRaw) {
 }
 
 void *vnodeGetVnode(int32_t vgId) {
+  if (tsDnodeVnodesHash == NULL) return NULL;
+
   SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t));
   if (ppVnode == NULL || *ppVnode == NULL) {
     terrno = TSDB_CODE_INVALID_VGROUP_ID;

From 580216c9538075cc6aaf8208e315a9a147ed7ffc Mon Sep 17 00:00:00 2001
From: slguan <slguan@taosdata.com>
Date: Wed, 13 May 2020 17:56:11 +0800
Subject: [PATCH 31/33] [TD-149] fix compile error after merge develop

---
 src/vnode/src/vnodeMain.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index e70e29a9a8..838b9d290f 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -32,7 +32,6 @@
 static int32_t  tsOpennedVnodes;
 static void    *tsDnodeVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
-static int      vnodeWalCallback(void *arg);
 static int32_t  vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
 static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
 static int32_t  vnodeSaveVersion(SVnodeObj *pVnode);

From cb75eed72979669a3d8e42adb53fc484ab79d264 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 18:31:20 +0800
Subject: [PATCH 32/33] scripts

---
 tests/script/jenkins/basic.txt | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 6b1902f590..44d7f1e8ea 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -15,7 +15,7 @@ cd ../../../debug; make
 ./test.sh -f general/cache/restart_metrics.sim
 ./test.sh -f general/cache/restart_table.sim
 
-#hongze ./test.sh -f general/column/commit.sim
+./test.sh -f general/column/commit.sim
 ./test.sh -f general/column/metrics.sim
 ./test.sh -f general/column/table.sim
 
@@ -32,7 +32,7 @@ cd ../../../debug; make
 ./test.sh -f general/compute/first.sim
 # liao ./test.sh -f general/compute/interval.sim
 ./test.sh -f general/compute/last.sim
-# liao ./test.sh -f general/compute/leastsquare.sim
+./test.sh -f general/compute/leastsquare.sim
 ./test.sh -f general/compute/max.sim
 ./test.sh -f general/compute/min.sim
 ./test.sh -f general/compute/null.sim
@@ -59,10 +59,10 @@ cd ../../../debug; make
 ./test.sh -f general/db/tables.sim
 
 ./test.sh -f general/field/2.sim
-#liao ./test.sh -f general/field/3.sim
-#liao? ./test.sh -f general/field/4.sim
-#liao? ./test.sh -f general/field/5.sim
-#liao? ./test.sh -f general/field/6.sim
+./test.sh -f general/field/3.sim
+./test.sh -f general/field/4.sim
+./test.sh -f general/field/5.sim
+./test.sh -f general/field/6.sim
 ./test.sh -f general/field/bigint.sim
 ./test.sh -f general/field/binary.sim
 ./test.sh -f general/field/bool.sim
@@ -98,7 +98,7 @@ cd ../../../debug; make
 #unsupport ./test.sh -f general/parser/alter1.sim
 #unsupport ./test.sh -f general/parser/alter_stable.sim
 ./test.sh -f general/parser/auto_create_tb.sim
-#slguan ./test.sh -f general/parser/auto_create_tb_drop_tb.sim
+./test.sh -f general/parser/auto_create_tb_drop_tb.sim
 ./test.sh -f general/parser/col_arithmetic_operation.sim
 ./test.sh -f general/parser/columnValue.sim
 ./test.sh -f general/parser/commit.sim
@@ -148,7 +148,7 @@ cd ../../../debug; make
 
 ./test.sh -f general/stable/disk.sim
 ./test.sh -f general/stable/metrics.sim
-#liao? ./test.sh -f general/stable/values.sim
+./test.sh -f general/stable/values.sim
 ./test.sh -f general/stable/vnode3.sim
 
 ./test.sh -f general/table/autocreate.sim
@@ -179,10 +179,10 @@ cd ../../../debug; make
 ./test.sh -f general/table/tinyint.sim
 ./test.sh -f general/table/vgroup.sim
 
-#liao ./test.sh -f general/tag/3.sim
-#liao? ./test.sh -f general/tag/4.sim
-#liao? ./test.sh -f general/tag/5.sim
-#liao? ./test.sh -f general/tag/6.sim
+./test.sh -f general/tag/3.sim
+./test.sh -f general/tag/4.sim
+./test.sh -f general/tag/5.sim
+./test.sh -f general/tag/6.sim
 #unsupport ./test.sh -f general/tag/add.sim
 ./test.sh -f general/tag/bigint.sim
 ./test.sh -f general/tag/binary_binary.sim
@@ -195,10 +195,10 @@ cd ../../../debug; make
 #unsupport ./test.sh -f general/tag/commit.sim
 ./test.sh -f general/tag/create.sim
 #unsupport ./test.sh -f general/tag/delete.sim
-#./test.sh -f general/tag/double.sim
+./test.sh -f general/tag/double.sim
 ./test.sh -f general/tag/filter.sim
-#liao? ./test.sh -f general/tag/float.sim
-#liao? ./test.sh -f general/tag/int_binary.sim
+./test.sh -f general/tag/float.sim
+./test.sh -f general/tag/int_binary.sim
 ./test.sh -f general/tag/int_float.sim
 ./test.sh -f general/tag/int.sim
 #unsupport ./test.sh -f general/tag/set.sim

From 174f0354f018c02942d52380734bbbb4c41032c9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 13 May 2020 18:36:42 +0800
Subject: [PATCH 33/33] scripts

---
 tests/script/jenkins/basic.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 44d7f1e8ea..db7a3d6757 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -15,6 +15,8 @@ cd ../../../debug; make
 ./test.sh -f general/cache/restart_metrics.sim
 ./test.sh -f general/cache/restart_table.sim
 
+./test.sh -f general/connection/connections.sim
+
 ./test.sh -f general/column/commit.sim
 ./test.sh -f general/column/metrics.sim
 ./test.sh -f general/column/table.sim
@@ -205,6 +207,7 @@ cd ../../../debug; make
 ./test.sh -f general/tag/smallint.sim
 ./test.sh -f general/tag/tinyint.sim
 
+./test.sh -f general/user/authority.sim
 ./test.sh -f general/user/basic1.sim
 ./test.sh -f general/user/monitor.sim
 ./test.sh -f general/user/pass_alter.sim