[TD-1980] release mem/imem table reference once the data in cache are all scanned.

src/query/src/qExecutor.c

@@ -6926,7 +6926,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
 
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
-    doFreeQueryHandle(pQInfo);
+//    doFreeQueryHandle(pQInfo);
     *continueExec = false;
     (*pRsp)->completed = 1;  // notify no more result to client
   } else {

src/tsdb/src/tsdbMemTable.c

@@ -197,7 +197,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
     tsdbUnRefMemTable(pRepo, pIMem);
   }
 
-  tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
+  tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
 }
 
 void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {

src/tsdb/src/tsdbRead.c

@@ -120,8 +120,6 @@ typedef struct STsdbQueryHandle {
   SDataCols     *pDataCols;        // in order to hold current file data block
   int32_t        allocSize;        // allocated data block size
   SMemRef       *pMemRef;
-//  SMemTable     *mem;              // mem-table
-//  SMemTable     *imem;             // imem-table, acquired from snapshot
   SArray        *defaultLoadColumn;// default load column
   SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
   SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@@ -194,9 +192,12 @@ static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
 }
 
 static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
-  assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
+  assert(pQueryHandle != NULL);
-
   SMemRef* pMemRef = pQueryHandle->pMemRef;
+  if (pMemRef == NULL) { // it has been freed
+    return;
+  }
+
   if (--pMemRef->ref == 0) {
     tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
     pMemRef->mem = NULL;
@@ -205,6 +206,7 @@ static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
 
   pQueryHandle->pMemRef = NULL;
 }
+
 static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
   size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
   assert(sizeOfGroup >= 1 && pMeta != NULL);
@@ -1821,6 +1823,8 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     pQueryHandle->activeIndex += 1;
   }
 
+  // no data in memtable or imemtable, decrease the memory reference.
+  tsdbMayUnTakeMemSnapshot(pQueryHandle);
   return false;
 }
 
@@ -1947,27 +1951,20 @@ static void destroyHelper(void* param) {
   free(param);
 }
 
-// handle data in cache situation
+static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) {
-bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
+  assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL);
-  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
-
-  int64_t stime = taosGetTimestampUs();
-  int64_t elapsedTime = stime;
-
-  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  assert(numOfTables > 0);
 
   SDataBlockInfo blockInfo = {{0}, 0};
-  if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
+
   pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
   pQueryHandle->order = TSDB_ORDER_DESC;
 
-    if (!tsdbNextDataBlock(pHandle)) {
+  if (!tsdbNextDataBlock((void*) pQueryHandle)) {
     return false;
   }
 
-    tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
+  tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo);
-    /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
+  /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn);
   if (terrno != TSDB_CODE_SUCCESS) {
     return false;
   }
@@ -2005,7 +2002,6 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
     }
 
     STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
-
     taosTFree(cond.colList);
 
     pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
@@ -2059,6 +2055,27 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   return true;
 }
 
+// handle data in cache situation
+bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
+  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
+
+  int64_t stime = taosGetTimestampUs();
+  int64_t elapsedTime = stime;
+
+  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+  assert(numOfTables > 0);
+
+  if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
+    SMemRef* pMemRef = pQueryHandle->pMemRef;
+    tsdbMayTakeMemSnapshot(pQueryHandle);
+    bool ret = getNeighborRows(pQueryHandle);
+    tsdbMayUnTakeMemSnapshot(pQueryHandle);
+
+    // restore the pMemRef
+    pQueryHandle->pMemRef = pMemRef;
+    return ret;
+  }
+
   if (pQueryHandle->checkFiles) {
     // check if the query range overlaps with the file data block
     bool exists = true;