From a6ba7e678fc8d2d14bea9444a65121a8de9c971f Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Thu, 31 Aug 2023 13:56:12 +0800
Subject: [PATCH] fix: search memory in ttlMgrFindExpired

---
 source/dnode/vnode/src/meta/metaCommit.c |  2 ++
 source/dnode/vnode/src/meta/metaTable.c  | 15 +++++--------
 source/dnode/vnode/src/meta/metaTtl.c    | 28 ++++++++++++++++++++----
 source/dnode/vnode/src/vnd/vnodeSvr.c    |  4 ++--
 4 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c
index d262567953..59416fd284 100644
--- a/source/dnode/vnode/src/meta/metaCommit.c
+++ b/source/dnode/vnode/src/meta/metaCommit.c
@@ -52,7 +52,9 @@ int  metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv
 int  metaPrepareAsyncCommit(SMeta *pMeta) {
    // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
   int code = 0;
+  metaWLock(pMeta);
   code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
+  metaULock(pMeta);
   code = tdbCommit(pMeta->pEnv, pMeta->txn);
 
   return code;
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 9a298a4bb7..18e1aa303b 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -931,21 +931,16 @@ end:
 }
 
 int metaTtlFindExpired(SMeta *pMeta, int64_t timePointMs, SArray *tbUids, int32_t ttlDropMaxCount) {
-  metaWLock(pMeta);
-  int ret = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
-  if (ret != 0) {
-    metaError("ttl failed to flush, ret:%d", ret);
-    goto _err;
-  }
+  metaRLock(pMeta);
+
+  int ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids, ttlDropMaxCount);
+
+  metaULock(pMeta);
 
-  ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids, ttlDropMaxCount);
   if (ret != 0) {
     metaError("ttl failed to find expired table, ret:%d", ret);
-    goto _err;
   }
 
-_err:
-  metaULock(pMeta);
   return ret;
 }
 
diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c
index f920296b4a..58ecf54512 100644
--- a/source/dnode/vnode/src/meta/metaTtl.c
+++ b/source/dnode/vnode/src/meta/metaTtl.c
@@ -299,7 +299,7 @@ int ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) {
   ret = 0;
 
 _out:
-  metaDebug("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix,
+  metaTrace("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix,
             updCtx->uid, updCtx->changeTimeMs, updCtx->ttlDays);
 
   return ret;
@@ -323,7 +323,7 @@ int ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) {
   ret = 0;
 
 _out:
-  metaDebug("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid);
+  metaTrace("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid);
 
   return ret;
 }
@@ -363,17 +363,37 @@ int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtime
   ret = 0;
 
 _out:
-  metaDebug("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid,
+  metaTrace("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid,
             pUpdCtimeCtx->changeTimeMs);
 
   return ret;
 }
 
 int ttlMgrFindExpired(STtlManger *pTtlMgr, int64_t timePointMs, SArray *pTbUids, int32_t ttlDropMaxCount) {
+  int ret = -1;
+
   STtlIdxKeyV1   ttlKey = {.deleteTimeMs = timePointMs, .uid = INT64_MAX};
   STtlExpiredCtx expiredCtx = {
       .ttlDropMaxCount = ttlDropMaxCount, .count = 0, .expiredKey = ttlKey, .pTbUids = pTbUids};
-  return tdbTbTraversal(pTtlMgr->pTtlIdx, &expiredCtx, ttlMgrFindExpiredOneEntry);
+  ret = tdbTbTraversal(pTtlMgr->pTtlIdx, &expiredCtx, ttlMgrFindExpiredOneEntry);
+  if (ret) {
+    goto _out;
+  }
+
+  size_t vIdx = 0;
+  for (size_t i = 0; i < pTbUids->size; i++) {
+    tb_uid_t *pUid = taosArrayGet(pTbUids, i);
+    if (taosHashGet(pTtlMgr->pDirtyUids, pUid, sizeof(tb_uid_t)) == NULL) {
+      // not in dirty && expired in tdb => must be expired
+      taosArraySet(pTbUids, vIdx, pUid);
+      vIdx++;
+    }
+  }
+
+  taosArrayPopTailBatch(pTbUids, pTbUids->size - vIdx);
+
+_out:
+  return ret;
 }
 
 static bool ttlMgrNeedFlush(STtlManger *pTtlMgr) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 0b7f969ed7..22d5d40877 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -160,7 +160,7 @@ static int32_t vnodePreProcessDropTtlMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   }
 
   {  // find expired uids
-    tbUids = taosArrayInit(8, sizeof(int64_t));
+    tbUids = taosArrayInit(8, sizeof(tb_uid_t));
     if (tbUids == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       TSDB_CHECK_CODE(code, lino, _exit);
@@ -945,7 +945,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
     int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
 
     char detail[1000] = {0};
-    sprintf(detail, "btime:%" PRId64 ", flags:%d, ttl:%d, type:%d", 
+    sprintf(detail, "btime:%" PRId64 ", flags:%d, ttl:%d, type:%d",
             pCreateReq->btime, pCreateReq->flags, pCreateReq->ttl, pCreateReq->type);
 
     SName name = {0};