From 47410b16d23f53574fcfac67bbee0b62a9ee7f35 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 2 May 2022 03:02:34 +0000
Subject: [PATCH 01/24] refact TDB

---
 source/dnode/vnode/src/meta/metaTDBImpl.c | 16 +++----
 source/dnode/vnode/src/meta/metaTable.c   | 12 ++---
 source/dnode/vnode/src/tsdb/tsdbTDBImpl.c |  2 +-
 source/libs/tdb/inc/tdb.h                 |  4 +-
 source/libs/tdb/src/db/tdbBtree.c         | 57 +++++++++++++++++++++++
 source/libs/tdb/src/db/tdbDb.c            |  9 +++-
 source/libs/tdb/src/inc/tdbInt.h          |  2 +
 source/libs/tdb/test/tdbTest.cpp          |  4 +-
 8 files changed, 87 insertions(+), 19 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index bba707076c..cb556e8630 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   pVal = pBuf = buf;
   metaEncodeTbInfo(&pBuf, pTbCfg);
   vLen = POINTER_DISTANCE(pBuf, buf);
-  ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     pVal = pBuf = buf;
     metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
-    ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
+    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
     if (ret < 0) {
       return -1;
     }
@@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   kLen = nameLen + 1 + sizeof(uid);
   pVal = NULL;
   vLen = 0;
-  ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(ctbIdxKey);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   int32_t kLen = sizeof(pSmaCfg->indexUid);
   int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
 
-  ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn);
   if (ret < 0) {
     taosMemoryFreeClear(pBuf);
     return -1;
@@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   val = NULL;
   vLen = 0;
 
-  ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn);
   if (ret < 0) {
     taosMemoryFreeClear(pBuf);
     return -1;
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index cb0b3dc81c..41ef231dcf 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -218,7 +218,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   tCoderClear(&coder);
 
   // write to table.db
-  if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
     goto _err;
   }
 
@@ -231,11 +231,11 @@ _err:
 }
 
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
+  return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+  return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
 }
 
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -258,12 +258,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
   ttlKey.uid = pME->uid;
 
-  return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
+  return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
-  return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
+  return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -304,7 +304,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
   tEncodeSSchemaWrapper(&coder, pSW);
 
-  if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
     rcode = -1;
     goto _exit;
   }
diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c
index 74878e817f..8a553e94fb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c
+++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c
@@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) {
 int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
   int32_t ret;
 
-  ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+  ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
   if (ret < 0) {
     tsdbError("Failed to create insert sma data into db, ret = %d", ret);
     return -1;
diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h
index 49da6fd8ed..dae085c672 100644
--- a/source/libs/tdb/inc/tdb.h
+++ b/source/libs/tdb/inc/tdb.h
@@ -40,7 +40,9 @@ int tdbCommit(TENV *pEnv, TXN *pTxn);
 int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb);
 int tdbDbClose(TDB *pDb);
 int tdbDbDrop(TDB *pDb);
-int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
+int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn);
+int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 2665dd1cf6..d711b47838 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -202,6 +202,36 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in
   return 0;
 }
 
+int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
+  SBTC btc;
+  int  c;
+  int  ret;
+
+  tdbBtcOpen(&btc, pBt, pTxn);
+
+  // move the cursor
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+
+  if (btc.idx < 0 || c != 0) {
+    tdbBtcClose(&btc);
+    return -1;
+  }
+
+  // delete the key
+  if (tdbBtcDelete(&btc) < 0) {
+    tdbBtcClose(&btc);
+    return -1;
+  }
+
+  tdbBtcClose(&btc);
+  return 0;
+}
+
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) {
   return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen);
 }
@@ -1363,6 +1393,33 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int
   return 0;
 }
 
+int tdbBtcDelete(SBTC *pBtc) {
+#if 0
+  // check transaction
+  if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) {
+    return -1;
+  }
+
+  int idx = pBtc->idx;
+  int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+
+  tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
+
+  tdbPageDropCell(pBtc->pPage, idx);
+
+  if (idx == nCells - 1) {
+    // drop cells above
+    for (;;) {
+      /* code */
+    }
+
+    // do balance
+  }
+#endif
+
+  return 0;
+}
+
 int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
   int         ret;
   int         nCells;
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index aadb9238b0..f9ef2e7901 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -75,10 +75,17 @@ int tdbDbDrop(TDB *pDb) {
   return 0;
 }
 
-int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
   return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
 }
 
+int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); }
+
+int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
+  // TODO
+  return 0;
+}
+
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
   return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen);
 }
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 4c19dee03e..ec0ad1fa55 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -128,6 +128,7 @@ struct SBTC {
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
 int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
+int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
@@ -141,6 +142,7 @@ int tdbBtcMoveToNext(SBTC *pBtc);
 int tdbBtcMoveToPrev(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
+int tdbBtcDelete(SBTC *pBtc);
 
 // tdbPager.c ====================================
 
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 1f6f28261f..9b4bc46ea9 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -152,7 +152,7 @@ TEST(tdb_test, simple_test) {
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key%d", iData);
       sprintf(val, "value%d", iData);
-      ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn);
+      ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
       GTEST_ASSERT_EQ(ret, 0);
 
       // if pool is full, commit the transaction and start a new one
@@ -269,7 +269,7 @@ TEST(tdb_test, simple_test2) {
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key%d", iData);
       sprintf(val, "value%d", iData);
-      ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn);
+      ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
       GTEST_ASSERT_EQ(ret, 0);
     }
 

From 52d157718b588df2f50b0db3d13c4799b4091108 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 2 May 2022 07:55:30 +0000
Subject: [PATCH 02/24] more tdb delte

---
 source/libs/tdb/src/db/tdbBtree.c | 75 +++++++++++++++++++++++++------
 source/libs/tdb/src/db/tdbPage.c  |  5 +++
 source/libs/tdb/src/inc/tdbInt.h  |  2 +
 3 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index d711b47838..1d697e0035 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1394,32 +1394,79 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int
 }
 
 int tdbBtcDelete(SBTC *pBtc) {
-#if 0
-  // check transaction
-  if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) {
+  int         idx = pBtc->idx;
+  int         nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+  SPager     *pPager = pBtc->pBt->pPager;
+  const void *pKey;
+  i8          iPage;
+  SPage      *pPage;
+  SPgno       pgno;
+  SCell      *pCell;
+  int         szCell;
+  int         nKey;
+  int         ret;
+
+  ASSERT(idx >= 0 && idx < nCells);
+
+  // drop the cell on the leaf
+  ret = tdbPagerWrite(pPager, pBtc->pPage);
+  if (ret < 0) {
+    ASSERT(0);
     return -1;
   }
 
-  int idx = pBtc->idx;
-  int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
-
-  tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
-
   tdbPageDropCell(pBtc->pPage, idx);
 
+  // update interior page or do balance
   if (idx == nCells - 1) {
-    // drop cells above
-    for (;;) {
-      /* code */
+    if (idx) {
+      pBtc->idx--;
+      tdbBtcGet(pBtc, &pKey, &nKey, NULL, NULL);
+
+      // loop to update the interial page
+      pgno = TDB_PAGE_PGNO(pBtc->pPage);
+      for (iPage = pBtc->iPage - 1; iPage >= 0; iPage--) {
+        pPage = pBtc->pgStack[iPage];
+        idx = pBtc->idxStack[iPage];
+        nCells = TDB_PAGE_TOTAL_CELLS(pPage);
+
+        if (idx < nCells) {
+          ret = tdbPagerWrite(pPager, pPage);
+          if (ret < 0) {
+            ASSERT(0);
+            return -1;
+          }
+
+          // update the cell with new key
+          pCell = tdbOsMalloc(nKey + 9);
+          tdbBtreeEncodeCell(pPage, pKey, nKey, &pgno, sizeof(pgno), pCell, &szCell);
+
+          ret = tdbPageUpdateCell(pPage, idx, pCell, szCell);
+          if (ret < 0) {
+            tdbOsFree(pCell);
+            ASSERT(0);
+            return -1;
+          }
+          tdbOsFree(pCell);
+          break;
+        } else {
+          pgno = TDB_PAGE_PGNO(pPage);
+        }
+      }
+    } else {
+      // delete the leaf page and do balance (TODO)
     }
-
-    // do balance
   }
-#endif
 
   return 0;
 }
 
+int tdbBtcUpsert(SBTC *pBtc) {
+  ASSERT(0);
+  // TODO
+  return 0;
+}
+
 int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
   int         ret;
   int         nCells;
diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c
index f0fdd82944..92272fb438 100644
--- a/source/libs/tdb/src/db/tdbPage.c
+++ b/source/libs/tdb/src/db/tdbPage.c
@@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
   return 0;
 }
 
+int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
+  tdbPageDropCell(pPage, idx);
+  return tdbPageInsertCell(pPage, idx, pCell, szCell, 0);
+}
+
 int tdbPageDropCell(SPage *pPage, int idx) {
   int    lidx;
   SCell *pCell;
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index ec0ad1fa55..d1a1109381 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -143,6 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
 int tdbBtcDelete(SBTC *pBtc);
+int tdbBtcUpsert(SBTC *pBtc);
 
 // tdbPager.c ====================================
 
@@ -280,6 +281,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
 void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *));
 int  tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
 int  tdbPageDropCell(SPage *pPage, int idx);
+int  tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell);
 void tdbPageCopy(SPage *pFromPage, SPage *pToPage);
 int  tdbPageCapacity(int pageSize, int amHdrSize);
 

From 9d0d5295f40517fd474eea94765301e8baf187e7 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 2 May 2022 10:13:45 +0000
Subject: [PATCH 03/24] more TDB delte

---
 source/libs/tdb/src/db/tdbBtree.c | 4 +++-
 source/libs/tdb/src/inc/tdbInt.h  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 1d697e0035..887ab10ec0 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1455,13 +1455,15 @@ int tdbBtcDelete(SBTC *pBtc) {
       }
     } else {
       // delete the leaf page and do balance (TODO)
+      ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
+      ASSERT(0);
     }
   }
 
   return 0;
 }
 
-int tdbBtcUpsert(SBTC *pBtc) {
+int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
   ASSERT(0);
   // TODO
   return 0;
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index d1a1109381..e18952505c 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -143,7 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
 int tdbBtcDelete(SBTC *pBtc);
-int tdbBtcUpsert(SBTC *pBtc);
+int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen);
 
 // tdbPager.c ====================================
 

From 05f06e04c1403a2e8889cfc8795ddb5a1507d2d2 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 2 May 2022 15:17:43 +0000
Subject: [PATCH 04/24] refact sl

---
 include/util/tskiplist2.h        | 159 +++++++++++++++++++++++++++++++
 source/libs/tdb/test/tdbTest.cpp |   4 +-
 source/util/src/tskiplist2.c     | 139 +++++++++++++++++++++++++++
 3 files changed, 301 insertions(+), 1 deletion(-)
 create mode 100644 include/util/tskiplist2.h
 create mode 100644 source/util/src/tskiplist2.c

diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h
new file mode 100644
index 0000000000..ce44ff96eb
--- /dev/null
+++ b/include/util/tskiplist2.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_UTIL_SKIPLIST2_H_
+#define _TD_UTIL_SKIPLIST2_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SL_MAX_LEVEL 15
+
+typedef struct SSkipList2 SSkipList2;
+typedef struct SSLCursor  SSLCursor;
+typedef struct SSLCfg     SSLCfg;
+
+typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
+
+// SSkipList2
+int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
+int32_t slClose(SSkipList2 *pSl);
+
+// SSLCursor
+int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
+int32_t slcClose(SSLCursor *pSlc);
+int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey);
+int32_t slcMoveToNext(SSLCursor *pSlc);
+int32_t slcMoveToPrev(SSLCursor *pSlc);
+int32_t slcMoveToFirst(SSLCursor *pSlc);
+int32_t slcMoveToLast(SSLCursor *pSlc);
+int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData);
+int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData);
+int32_t slcDrop(SSLCursor *pSlc);
+
+// struct
+struct SSLCfg {
+  int8_t    maxLevel;
+  int32_t   nKey;
+  int32_t   nData;
+  tslCmprFn cmprFn;
+  void     *pPool;
+  void *(*xMalloc)(void *, int32_t size);
+  void (*xFree)(void *, void *);
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SKIPLIST2_H_*/
+
+#if 0
+#ifndef _TD_UTIL_SKILIST_H
+#define _TD_UTIL_SKILIST_H
+
+#include "os.h"
+#include "taos.h"
+#include "tarray.h"
+#include "tfunctional.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MAX_SKIP_LIST_LEVEL          15
+#define SKIP_LIST_RECORD_PERFORMANCE 0
+
+// For key property setting
+#define SL_ALLOW_DUP_KEY             (uint8_t)0x0  // Allow duplicate key exists (for tag index usage)
+#define SL_DISCARD_DUP_KEY           (uint8_t)0x1  // Discard duplicate key (for data update=0 case)
+#define SL_UPDATE_DUP_KEY            (uint8_t)0x2  // Update duplicate key by remove/insert (for data update!=0 case)
+
+// For thread safety setting
+#define SL_THREAD_SAFE               (uint8_t)0x4
+
+typedef char *SSkipListKey;
+typedef char *(*__sl_key_fn_t)(const void *);
+
+typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc);
+typedef void *(*iter_next_fn_t)(void *iter);
+
+typedef struct SSkipListNode {
+  uint8_t               level;
+  void                 *pData;
+  struct SSkipListNode *forwards[];
+} SSkipListNode;
+
+#define SL_GET_NODE_DATA(n)                (n)->pData
+#define SL_NODE_GET_FORWARD_POINTER(n, l)  (n)->forwards[(l)]
+#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
+
+typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus;
+
+typedef struct SSkipList {
+  uint32_t           seed;
+  __compar_fn_t      comparFn;
+  __sl_key_fn_t      keyFn;
+  TdThreadRwlock    *lock;
+  uint16_t           len;
+  uint8_t            maxLevel;
+  uint8_t            flags;
+  uint8_t            type;  // static info above
+  uint8_t            level;
+  uint32_t           size;
+  SSkipListNode     *pHead;  // point to the first element
+  SSkipListNode     *pTail;  // point to the last element
+  tGenericSavedFunc *insertHandleFn;
+} SSkipList;
+
+typedef struct SSkipListIterator {
+  SSkipList     *pSkipList;
+  SSkipListNode *cur;
+  int32_t        step;   // the number of nodes that have been checked already
+  int32_t        order;  // order of the iterator
+  SSkipListNode *next;   // next points to the true qualified node in skiplist
+} SSkipListIterator;
+
+#define SL_IS_THREAD_SAFE(s)  (((s)->flags) & SL_THREAD_SAFE)
+#define SL_DUP_MODE(s)        (((s)->flags) & ((((uint8_t)1) << 2) - 1))
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
+#define SL_GET_MIN_KEY(s)     SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
+#define SL_GET_MAX_KEY(s)     SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
+#define SL_SIZE(s)            (s)->size
+
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
+                           __sl_key_fn_t fn);
+void       tSkipListDestroy(SSkipList *pSkipList);
+SSkipListNode     *tSkipListPut(SSkipList *pSkipList, void *pData);
+void               tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate);
+SArray            *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
+void               tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
+SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
+bool               tSkipListIterNext(SSkipListIterator *iter);
+SSkipListNode     *tSkipListIterGet(SSkipListIterator *iter);
+void              *tSkipListDestroyIter(SSkipListIterator *iter);
+uint32_t           tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
+void               tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SKILIST_H*/
+
+#endif
\ No newline at end of file
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 9b4bc46ea9..4f20efd904 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -316,4 +316,6 @@ TEST(tdb_test, simple_test2) {
   // Close Env
   ret = tdbEnvClose(pEnv);
   GTEST_ASSERT_EQ(ret, 0);
-}
\ No newline at end of file
+}
+
+TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); }
\ No newline at end of file
diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c
new file mode 100644
index 0000000000..6657896131
--- /dev/null
+++ b/source/util/src/tskiplist2.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tskiplist2.h"
+
+typedef struct SSLNode SSLNode;
+struct SSLNode {
+  int8_t   level;
+  SSLNode *forwards[];
+};
+
+struct SSkipList2 {
+  int8_t        level;
+  uint32_t      seed;
+  int32_t       size;
+  const SSLCfg *pCfg;
+};
+
+struct SSLCursor {
+  SSkipList2 *pSl;
+  SSLNode    *forwards[SL_MAX_LEVEL];
+};
+
+static void   *slMalloc(void *pPool, int32_t size);
+static void    slFree(void *pPool, void *p);
+static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData);
+
+const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL,
+                             .nKey = -1,
+                             .nData = -1,
+                             .cmprFn = slCmprFn,
+                             .pPool = NULL,
+                             .xMalloc = slMalloc,
+                             .xFree = slFree};
+
+int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) {
+  SSkipList2 *pSl = NULL;
+
+  *ppSl = NULL;
+  if (pCfg == NULL) pCfg = &slDefaultCfg;
+
+  // check cfg (TODO)
+
+  // create handle
+  pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl));
+  // (TODO)
+  *ppSl = pSl;
+  return 0;
+}
+
+int32_t slClose(SSkipList2 *pSl) {
+  // TODO
+  return 0;
+}
+
+int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcClose(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) {
+  // TODO
+  return 0;
+}
+
+int32_t slcMoveToNext(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcMoveToPrev(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcMoveToFirst(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcMoveToLast(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) {
+  // TODO
+  return 0;
+}
+
+int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) {
+  // TODO
+  return 0;
+}
+
+int32_t slcDrop(SSLCursor *pSlc) {
+  // TODO
+  return 0;
+}
+
+static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); }
+
+static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); }
+
+static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) {
+  ASSERT(nKey1 >= 0 && nKey2 >= 0);
+
+  int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1;
+  int32_t c;
+
+  c = memcmp(pKey1, pKey2, nKey);
+  if (c == 0) {
+    if (nKey1 > nKey2) {
+      c = 1;
+    } else if (nKey1 < nKey2) {
+      c = -1;
+    }
+  }
+
+  return c;
+}
\ No newline at end of file

From f5c58a578288730547f049c9f9eb72ae3afb1b8b Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 3 May 2022 08:09:58 +0000
Subject: [PATCH 05/24] more refact

---
 include/util/tskiplist2.h        | 98 +-------------------------------
 source/libs/tdb/test/tdbTest.cpp | 65 ++++++++++++++++++++-
 2 files changed, 65 insertions(+), 98 deletions(-)

diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h
index ce44ff96eb..95fbee5a23 100644
--- a/include/util/tskiplist2.h
+++ b/include/util/tskiplist2.h
@@ -60,100 +60,4 @@ struct SSLCfg {
 }
 #endif
 
-#endif /*_TD_UTIL_SKIPLIST2_H_*/
-
-#if 0
-#ifndef _TD_UTIL_SKILIST_H
-#define _TD_UTIL_SKILIST_H
-
-#include "os.h"
-#include "taos.h"
-#include "tarray.h"
-#include "tfunctional.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define MAX_SKIP_LIST_LEVEL          15
-#define SKIP_LIST_RECORD_PERFORMANCE 0
-
-// For key property setting
-#define SL_ALLOW_DUP_KEY             (uint8_t)0x0  // Allow duplicate key exists (for tag index usage)
-#define SL_DISCARD_DUP_KEY           (uint8_t)0x1  // Discard duplicate key (for data update=0 case)
-#define SL_UPDATE_DUP_KEY            (uint8_t)0x2  // Update duplicate key by remove/insert (for data update!=0 case)
-
-// For thread safety setting
-#define SL_THREAD_SAFE               (uint8_t)0x4
-
-typedef char *SSkipListKey;
-typedef char *(*__sl_key_fn_t)(const void *);
-
-typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc);
-typedef void *(*iter_next_fn_t)(void *iter);
-
-typedef struct SSkipListNode {
-  uint8_t               level;
-  void                 *pData;
-  struct SSkipListNode *forwards[];
-} SSkipListNode;
-
-#define SL_GET_NODE_DATA(n)                (n)->pData
-#define SL_NODE_GET_FORWARD_POINTER(n, l)  (n)->forwards[(l)]
-#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
-
-typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus;
-
-typedef struct SSkipList {
-  uint32_t           seed;
-  __compar_fn_t      comparFn;
-  __sl_key_fn_t      keyFn;
-  TdThreadRwlock    *lock;
-  uint16_t           len;
-  uint8_t            maxLevel;
-  uint8_t            flags;
-  uint8_t            type;  // static info above
-  uint8_t            level;
-  uint32_t           size;
-  SSkipListNode     *pHead;  // point to the first element
-  SSkipListNode     *pTail;  // point to the last element
-  tGenericSavedFunc *insertHandleFn;
-} SSkipList;
-
-typedef struct SSkipListIterator {
-  SSkipList     *pSkipList;
-  SSkipListNode *cur;
-  int32_t        step;   // the number of nodes that have been checked already
-  int32_t        order;  // order of the iterator
-  SSkipListNode *next;   // next points to the true qualified node in skiplist
-} SSkipListIterator;
-
-#define SL_IS_THREAD_SAFE(s)  (((s)->flags) & SL_THREAD_SAFE)
-#define SL_DUP_MODE(s)        (((s)->flags) & ((((uint8_t)1) << 2) - 1))
-#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
-#define SL_GET_MIN_KEY(s)     SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
-#define SL_GET_MAX_KEY(s)     SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
-#define SL_SIZE(s)            (s)->size
-
-SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
-                           __sl_key_fn_t fn);
-void       tSkipListDestroy(SSkipList *pSkipList);
-SSkipListNode     *tSkipListPut(SSkipList *pSkipList, void *pData);
-void               tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate);
-SArray            *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
-void               tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
-SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
-bool               tSkipListIterNext(SSkipListIterator *iter);
-SSkipListNode     *tSkipListIterGet(SSkipListIterator *iter);
-void              *tSkipListDestroyIter(SSkipListIterator *iter);
-uint32_t           tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
-void               tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_SKILIST_H*/
-
-#endif
\ No newline at end of file
+#endif /*_TD_UTIL_SKIPLIST2_H_*/
\ No newline at end of file
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 4f20efd904..78e5bc7832 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -318,4 +318,67 @@ TEST(tdb_test, simple_test2) {
   GTEST_ASSERT_EQ(ret, 0);
 }
 
-TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); }
\ No newline at end of file
+TEST(tdb_test, simple_delete1) {
+  int       ret;
+  TDB      *pDb;
+  char      key[128];
+  char      data[128];
+  TXN       txn;
+  TENV     *pEnv;
+  SPoolMem *pPool;
+  void     *pKey = NULL;
+  void     *pData = NULL;
+  int       nKey;
+  int       nData;
+  int       nKV = 254;
+
+  taosRemoveDir("tdb");
+
+  pPool = openPool();
+
+  // open env
+  ret = tdbEnvOpen("tdb", 4096, 256, &pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open database
+  ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  tdbBegin(pEnv, &txn);
+
+  // loop to insert batch data
+  for (int iData = 0; iData < nKV; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d", iData);
+    ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  // query the data
+  for (int iData = 0; iData < nKV; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d", iData);
+
+    ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData);
+    GTEST_ASSERT_EQ(ret, 0);
+    GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0);
+  }
+
+  // loop to delete some data
+  for (int iData = nKV - 1; iData >= 0; iData--) {
+    sprintf(key, "key%d", iData);
+
+    ret = tdbDbDelete(pDb, key, strlen(key), &txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  // query the data
+
+  tdbCommit(pEnv, &txn);
+
+  closePool(pPool);
+
+  tdbDbClose(pDb);
+  tdbEnvClose(pEnv);
+}
\ No newline at end of file

From bf355e705a3ab4af0ff64cd821c03cf28ee686ab Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 3 May 2022 09:59:58 +0000
Subject: [PATCH 06/24] fix a TDB balance bug

---
 source/libs/tdb/src/db/tdbBtree.c | 8 ++++----
 source/libs/tdb/test/tdbTest.cpp  | 6 +++++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 887ab10ec0..176a76cf28 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -582,14 +582,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
       SCell *pCell;
       int    szLCell, szRCell;
 
+      // balance page (iNew) and (iNew-1)
       for (;;) {
         pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx);
 
-        if (childNotLeaf) {
-          szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell);
+        szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell);
+        if (!childNotLeaf) {
+          szRCell = szLCell;
         } else {
-          szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell);
-
           int    iPage = infoNews[iNew - 1].iPage;
           int    oIdx = infoNews[iNew - 1].oIdx + 1;
           SPage *pPage;
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 78e5bc7832..46c13be3c2 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -202,6 +202,8 @@ TEST(tdb_test, simple_test) {
       ret = tdbDbcOpen(pDb, &pDBC, NULL);
       GTEST_ASSERT_EQ(ret, 0);
 
+      tdbDbcMoveToFirst(pDBC);
+
       for (;;) {
         ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
         if (ret < 0) break;
@@ -283,6 +285,8 @@ TEST(tdb_test, simple_test2) {
       ret = tdbDbcOpen(pDb, &pDBC, NULL);
       GTEST_ASSERT_EQ(ret, 0);
 
+      tdbDbcMoveToFirst(pDBC);
+
       for (;;) {
         ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
         if (ret < 0) break;
@@ -341,7 +345,7 @@ TEST(tdb_test, simple_delete1) {
   GTEST_ASSERT_EQ(ret, 0);
 
   // open database
-  ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb);
+  ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb);
   GTEST_ASSERT_EQ(ret, 0);
 
   tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);

From d3487c88c25852d63c6f99a8cf65c34692fdb407 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 3 May 2022 11:24:52 +0000
Subject: [PATCH 07/24] TDB impl delete

---
 source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++--
 source/libs/tdb/test/tdbTest.cpp  | 44 ++++++++++++++++++++++++++-----
 2 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 176a76cf28..85cbae316d 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -766,6 +766,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
     }
   }
 
+  if (TDB_BTREE_PAGE_IS_ROOT(pParent) && TDB_PAGE_TOTAL_CELLS(pParent) == 0) {
+    i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]);
+    // copy content to the parent page
+    tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0);
+    tdbPageCopy(pNews[0], pParent);
+  }
+
   for (int i = 0; i < 3; i++) {
     if (pDivCell[i]) {
       tdbOsFree(pDivCell[i]);
@@ -1454,9 +1461,14 @@ int tdbBtcDelete(SBTC *pBtc) {
         }
       }
     } else {
-      // delete the leaf page and do balance (TODO)
+      // delete the leaf page and do balance
       ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
-      ASSERT(0);
+
+      ret = tdbBtreeBalance(pBtc);
+      if (ret < 0) {
+        ASSERT(0);
+        return -1;
+      }
     }
   }
 
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 46c13be3c2..994e53cd9d 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -120,7 +120,7 @@ TEST(tdb_test, simple_test) {
   TENV         *pEnv;
   TDB          *pDb;
   tdb_cmpr_fn_t compFunc;
-  int           nData = 10000000;
+  int           nData = 1000000;
   TXN           txn;
 
   taosRemoveDir("tdb");
@@ -291,9 +291,9 @@ TEST(tdb_test, simple_test2) {
         ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
         if (ret < 0) break;
 
-        std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
-        std::cout.write((char *)pVal, vLen) /* << " " << vLen */;
-        std::cout << std::endl;
+        // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
+        // std::cout.write((char *)pVal, vLen) /* << " " << vLen */;
+        // std::cout << std::endl;
 
         count++;
       }
@@ -333,15 +333,16 @@ TEST(tdb_test, simple_delete1) {
   void     *pKey = NULL;
   void     *pData = NULL;
   int       nKey;
+  TDBC     *pDbc;
   int       nData;
-  int       nKV = 254;
+  int       nKV = 69;
 
   taosRemoveDir("tdb");
 
   pPool = openPool();
 
   // open env
-  ret = tdbEnvOpen("tdb", 4096, 256, &pEnv);
+  ret = tdbEnvOpen("tdb", 1024, 256, &pEnv);
   GTEST_ASSERT_EQ(ret, 0);
 
   // open database
@@ -370,7 +371,7 @@ TEST(tdb_test, simple_delete1) {
   }
 
   // loop to delete some data
-  for (int iData = nKV - 1; iData >= 0; iData--) {
+  for (int iData = nKV - 1; iData > 30; iData--) {
     sprintf(key, "key%d", iData);
 
     ret = tdbDbDelete(pDb, key, strlen(key), &txn);
@@ -378,6 +379,35 @@ TEST(tdb_test, simple_delete1) {
   }
 
   // query the data
+  for (int iData = 0; iData < nKV; iData++) {
+    sprintf(key, "key%d", iData);
+
+    ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData);
+    if (iData <= 30) {
+      GTEST_ASSERT_EQ(ret, 0);
+    } else {
+      GTEST_ASSERT_EQ(ret, -1);
+    }
+  }
+
+  // loop to iterate the data
+  tdbDbcOpen(pDb, &pDbc, NULL);
+
+  ret = tdbDbcMoveToFirst(pDbc);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  pKey = NULL;
+  pData = NULL;
+  for (;;) {
+    ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData);
+    if (ret < 0) break;
+
+    std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " ";
+    std::cout.write((char *)pData, nData) /* << " " << vLen */;
+    std::cout << std::endl;
+  }
+
+  tdbDbcClose(pDbc);
 
   tdbCommit(pEnv, &txn);
 

From cfada1cc8ae2ee8e03baf6cf0b99ffa384336578 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 3 May 2022 14:08:30 +0000
Subject: [PATCH 08/24] more sl refact

---
 include/util/tskiplist2.h    |  6 ++++
 source/util/src/tskiplist2.c | 57 +++++++++++++++++++++++++++++-------
 2 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h
index 95fbee5a23..f3366dd574 100644
--- a/include/util/tskiplist2.h
+++ b/include/util/tskiplist2.h
@@ -32,6 +32,7 @@ typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2
 // SSkipList2
 int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
 int32_t slClose(SSkipList2 *pSl);
+int32_t slClear(SSkipList2 *pSl);
 
 // SSLCursor
 int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
@@ -56,6 +57,11 @@ struct SSLCfg {
   void (*xFree)(void *, void *);
 };
 
+struct SSLCursor {
+  SSkipList2 *pSl;
+  SSLNode   **forwards[SL_MAX_LEVEL];
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c
index 6657896131..e7391da285 100644
--- a/source/util/src/tskiplist2.c
+++ b/source/util/src/tskiplist2.c
@@ -27,11 +27,7 @@ struct SSkipList2 {
   uint32_t      seed;
   int32_t       size;
   const SSLCfg *pCfg;
-};
-
-struct SSLCursor {
-  SSkipList2 *pSl;
-  SSLNode    *forwards[SL_MAX_LEVEL];
+  SSLNode      *pHead[];
 };
 
 static void   *slMalloc(void *pPool, int32_t size);
@@ -48,25 +44,66 @@ const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL,
 
 int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) {
   SSkipList2 *pSl = NULL;
+  int32_t     size;
 
   *ppSl = NULL;
   if (pCfg == NULL) pCfg = &slDefaultCfg;
 
-  // check cfg (TODO)
+  // check config (TODO)
+
+  // malloc handle
+  size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2;
+  pSl = pCfg->xMalloc(pCfg->pPool, size);
+  if (pSl == NULL) {
+    return -1;
+  }
+
+  pSl->level = 0;
+  pSl->seed = taosRand();
+  pSl->size = 0;
+  pSl->pCfg = pCfg;
+
+  // init an empty skiplist
+  for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) {
+    pSl->pHead[i] = NULL;
+  }
 
-  // create handle
-  pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl));
-  // (TODO)
   *ppSl = pSl;
   return 0;
 }
 
 int32_t slClose(SSkipList2 *pSl) {
-  // TODO
+  if (pSl) {
+    slClear(pSl);
+    if (pSl->pCfg->xFree) {
+      pSl->pCfg->xFree(pSl->pCfg->pPool, pSl);
+    }
+  }
+
+  return 0;
+}
+
+int32_t slClear(SSkipList2 *pSl) {
+  // loop to clear sl
+  for (;;) {
+    // (TODO)
+  }
+
+  // init sl (TODO)
+
   return 0;
 }
 
 int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) {
+  pSlc->pSl = pSl;
+
+  for (int i = 0; i < SL_MAX_LEVEL; i++) {
+    if (i < pSl->pCfg->maxLevel) {
+    } else {
+      pSlc->forwards[i] = NULL;
+    }
+  }
+
   // TODO
   return 0;
 }

From ac0fb0237ec3ff4f0fc882322c617599e9655187 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 4 May 2022 03:57:16 +0000
Subject: [PATCH 09/24] drop table code

---
 include/util/tskiplist2.h               |  1 +
 source/dnode/vnode/src/inc/meta.h       |  2 -
 source/dnode/vnode/src/inc/vnodeInt.h   | 18 +++---
 source/dnode/vnode/src/meta/metaTable.c | 77 ++++++++++++++++++++++---
 source/dnode/vnode/src/vnd/vnodeSvr.c   | 39 +++++++++++--
 source/libs/tdb/inc/tdb.h               |  2 +-
 source/libs/tdb/src/db/tdbBtree.c       |  2 +-
 source/libs/tdb/src/db/tdbDb.c          |  6 +-
 source/util/src/tskiplist2.c            |  1 -
 9 files changed, 117 insertions(+), 31 deletions(-)

diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h
index f3366dd574..83e6ad0868 100644
--- a/include/util/tskiplist2.h
+++ b/include/util/tskiplist2.h
@@ -26,6 +26,7 @@ extern "C" {
 typedef struct SSkipList2 SSkipList2;
 typedef struct SSLCursor  SSLCursor;
 typedef struct SSLCfg     SSLCfg;
+typedef struct SSLNode    SSLNode;
 
 typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
 
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index ca7e823d6b..5970fdbe92 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -103,8 +103,6 @@ typedef struct {
 
 #if 1
 
-// int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
-int          metaDropTable(SMeta* pMeta, tb_uid_t uid);
 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
 void         metaCloseSmaCursor(SMSmaCursor* pSmaCur);
 int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 1c7f6034ee..5538a63b4b 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -46,13 +46,13 @@
 extern "C" {
 #endif
 
-typedef struct SVnodeInfo   SVnodeInfo;
-typedef struct SMeta        SMeta;
-typedef struct STsdb        STsdb;
-typedef struct STQ          STQ;
-typedef struct SVState      SVState;
-typedef struct SVBufPool    SVBufPool;
-typedef struct SQWorker SQHandle;
+typedef struct SVnodeInfo SVnodeInfo;
+typedef struct SMeta      SMeta;
+typedef struct STsdb      STsdb;
+typedef struct STQ        STQ;
+typedef struct SVState    SVState;
+typedef struct SVBufPool  SVBufPool;
+typedef struct SQWorker   SQHandle;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -77,6 +77,7 @@ int             metaCommit(SMeta* pMeta);
 int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
+int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
@@ -100,7 +101,7 @@ int32_t      tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
 int32_t      tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
-                              uint64_t taskId);
+                             uint64_t taskId);
 tsdbReaderT  tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                                  void* pMemRef);
 int32_t      tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
@@ -189,7 +190,6 @@ struct STbUidStore {
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
-
 static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
   SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);
   return (pRetention->freq > 0 && pRetention->keep > 0);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 41ef231dcf..c09bafd859 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -98,7 +98,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
   //   ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1);
   //   tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/);
   //   tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen);
-  //   tdbDbcDrop(pDbc1);
+  //   tdbDbcDelete(pDbc1);
   //   // drop from pTagIdx
   //   // drop from pTtlIdx
   // }
@@ -166,18 +166,79 @@ _err:
   return -1;
 }
 
-int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
-#if 0
-  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
-    // TODO: handle error
+int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
+  TDBC       *pTbDbc = NULL;
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pNameIdxc = NULL;
+  const void *pData;
+  int         nData;
+  tb_uid_t    uid;
+  int64_t     tver;
+  SMetaEntry  me = {0};
+  SCoder      coder = {0};
+  int         c, ret;
+
+  // search & delete the name idx
+  tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
+  if (ret < 0 || c) {
+    tdbDbcClose(pNameIdxc);
+    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
     return -1;
   }
 
-  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
-    // TODO
+  ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
     return -1;
   }
-#endif
+
+  uid = *(tb_uid_t *)pData;
+
+  tdbDbcDelete(pNameIdxc);
+  tdbDbcClose(pNameIdxc);
+
+  // search & delete uid idx
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  tver = *(int64_t *)pData;
+  tdbDbcDelete(pUidIdxc);
+  tdbDbcClose(pUidIdxc);
+
+  // search and get meta entry
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER);
+  // ret = metaDecodeEntry(&coder, &me);
+  // if (ret < 0) {
+  //   ASSERT(0);
+  //   return -1;
+  // }
+
+  tCoderClear(&coder);
+  tdbDbcClose(pTbDbc);
 
   return 0;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 91a470b118..878dd5fca4 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -445,14 +445,45 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM
 }
 
 static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  SVDropTbReq req = {0};
-  SVDropTbReq rsp = {0};
+  SVDropTbBatchReq req = {0};
+  SVDropTbBatchRsp rsp = {0};
+  SCoder           coder = {0};
+  int              ret;
+
+  pRsp->msgType = TDMT_VND_CREATE_STB_RSP;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+  pRsp->code = TSDB_CODE_SUCCESS;
 
   // decode req
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
+  ret = tDecodeSVDropTbBatchReq(&coder, &req);
+  if (ret < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    pRsp->code = terrno;
+    goto _exit;
+  }
 
   // process req
+  rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs);
+  for (int iReq = 0; iReq < req.nReqs; iReq++) {
+    SVDropTbReq *pDropTbReq = req.pReqs + iReq;
+    SVDropTbRsp  dropTbRsp = {0};
 
-  // return rsp
+    /* code */
+    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq);
+    if (ret < 0) {
+      dropTbRsp.code = TSDB_CODE_SUCCESS;
+    } else {
+      dropTbRsp.code = terrno;
+    }
+
+    taosArrayPush(rsp.pArray, &dropTbRsp);
+  }
+
+_exit:
+  tCoderClear(&coder);
+  // encode rsp (TODO)
   return 0;
 }
 
@@ -482,7 +513,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
 }
 
 int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
-  if(!pReq) {
+  if (!pReq) {
     terrno = TSDB_CODE_INVALID_PTR;
     return TSDB_CODE_FAILED;
   }
diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h
index dae085c672..cab242d5b3 100644
--- a/source/libs/tdb/inc/tdb.h
+++ b/source/libs/tdb/inc/tdb.h
@@ -58,7 +58,7 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in
 
 int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen);
 int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen);
-int tdbDbcDrop(TDBC *pDbc);
+int tdbDbcDelete(TDBC *pDbc);
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 
 // TXN
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 85cbae316d..ed7aed552a 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1394,7 +1394,7 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int
 
   if (ppVal) {
     *ppVal = (void *)pBtc->coder.pVal;
-    *kLen = pBtc->coder.vLen;
+    *vLen = pBtc->coder.vLen;
   }
 
   return 0;
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index f9ef2e7901..3358aefec0 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -136,11 +136,7 @@ int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int v
   return 0;
 }
 
-int tdbDbcDrop(TDBC *pDbc) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
+int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); }
 
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c
index e7391da285..77f5ed5051 100644
--- a/source/util/src/tskiplist2.c
+++ b/source/util/src/tskiplist2.c
@@ -16,7 +16,6 @@
 
 #include "tskiplist2.h"
 
-typedef struct SSLNode SSLNode;
 struct SSLNode {
   int8_t   level;
   SSLNode *forwards[];

From 5ed7b44b165ded8d3b391fd748e64bd2f4b2c55a Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 4 May 2022 12:35:05 +0800
Subject: [PATCH 10/24] enh(index): support index filter

---
 include/libs/index/index.h               | 12 +++++++++++-
 source/libs/executor/src/indexoperator.c | 14 ++++++++------
 source/libs/index/inc/indexCache.h       |  1 +
 source/libs/index/inc/indexInt.h         |  1 -
 source/libs/index/src/index.c            |  1 -
 source/libs/index/src/indexCache.c       | 23 ++++++++++++++++++++++-
 source/libs/index/src/indexTfile.c       | 23 ++++++++++++++++++++++-
 7 files changed, 64 insertions(+), 11 deletions(-)

diff --git a/include/libs/index/index.h b/include/libs/index/index.h
index bf260d4899..c94d75338a 100644
--- a/include/libs/index/index.h
+++ b/include/libs/index/index.h
@@ -47,7 +47,17 @@ typedef enum {
 } SIndexOperOnColumn;
 
 typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType;
-typedef enum { QUERY_TERM = 0, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType;
+typedef enum {
+  QUERY_TERM = 0,
+  QUERY_PREFIX,
+  QUERY_SUFFIX,
+  QUERY_REGEX,
+  QUERY_LESS_THAN,
+  QUERY_LESS_EQUAL,
+  QUERY_GREATER_THAN,
+  QUERY_GREATER_EQUAL,
+  QUERY_RANGE
+} EIndexQueryType;
 
 /*
  * create multi query
diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c
index 0f075c5de2..3b62bdd664 100644
--- a/source/libs/executor/src/indexoperator.c
+++ b/source/libs/executor/src/indexoperator.c
@@ -63,9 +63,14 @@ typedef struct SIFParam {
 } SIFParam;
 
 static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
-  if (src == OP_TYPE_GREATER_THAN || src == OP_TYPE_GREATER_EQUAL || src == OP_TYPE_LOWER_THAN ||
-      src == OP_TYPE_LOWER_EQUAL) {
-    *dst = QUERY_RANGE;
+  if (src == OP_TYPE_GREATER_THAN) {
+    *dst = QUERY_GREATER_THAN;
+  } else if (src == OP_TYPE_GREATER_EQUAL) {
+    *dst = QUERY_GREATER_EQUAL;
+  } else if (src == OP_TYPE_LOWER_THAN) {
+    *dst = QUERY_LESS_THAN;
+  } else if (src == OP_TYPE_LOWER_EQUAL) {
+    *dst = QUERY_LESS_EQUAL;
   } else if (src == OP_TYPE_EQUAL) {
     *dst = QUERY_TERM;
   } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) {
@@ -249,9 +254,6 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
   SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName,
                                    strlen(left->colName), right->condValue, strlen(right->condValue));
-  if (operType == OP_TYPE_LOWER_EQUAL || operType == OP_TYPE_GREATER_EQUAL || operType == OP_TYPE_GREATER_THAN ||
-      operType == OP_TYPE_LOWER_THAN) {
-  }
   if (tm == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h
index 3ea986ad48..44355db7a7 100644
--- a/source/libs/index/inc/indexCache.h
+++ b/source/libs/index/inc/indexCache.h
@@ -56,6 +56,7 @@ typedef struct CacheTerm {
   int8_t   colType;
 
   SIndexOperOnColumn operaType;
+  int8_t             qType;  // query type
 } CacheTerm;
 //
 
diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h
index 62ddf85985..04817c4df3 100644
--- a/source/libs/index/inc/indexInt.h
+++ b/source/libs/index/inc/indexInt.h
@@ -86,7 +86,6 @@ typedef struct SIndexTerm {
   int32_t            nColName;
   char*              colVal;
   int32_t            nColVal;
-  int8_t             qType;  // just use for range
 } SIndexTerm;
 
 typedef struct SIndexTermQuery {
diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index ee77aa92e9..83b5025ad0 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -262,7 +262,6 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryT
   tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1);
   memcpy(tm->colVal, colVal, nColVal);
   tm->nColVal = nColVal;
-  tm->qType = queryType;
 
   return tm;
 }
diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c
index 78368981b3..ea3cbf5658 100644
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@@ -38,10 +38,15 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S
 static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 
 static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
-    cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchRange};
+    cacheSearchTerm,      cacheSearchPrefix,      cacheSearchSuffix,       cacheSearchRegex, cacheSearchLessThan,
+    cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange};
 
 static void doMergeWork(SSchedMsg* msg);
 static bool indexCacheIteratorNext(Iterate* itera);
@@ -88,6 +93,22 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr,
   // impl later
   return 0;
 }
+static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
 static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
   // impl later
   return 0;
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 7072baf574..327a06084c 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -64,10 +64,15 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 
 static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
-    tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchRange};
+    tfSearchTerm,      tfSearchPrefix,      tfSearchSuffix,       tfSearchRegex, tfSearchLessThan,
+    tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange};
 
 TFileCache* tfileCacheCreate(const char* path) {
   TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache));
@@ -299,6 +304,22 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   fstSliceDestroy(&key);
   return 0;
 }
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  // impl later
+  return 0;
+}
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  // impl later
+  return 0;
+}
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  // impl later
+  return 0;
+}
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  // impl later
+  return 0;
+}
 static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
   bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   int      ret = 0;

From d5e578cf163b2b6477642744542314cb3bc71c9c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 4 May 2022 06:16:29 +0000
Subject: [PATCH 11/24] implement drop table

---
 source/dnode/vnode/src/meta/metaTable.c | 55 ++++++++++++++++++++++---
 1 file changed, 49 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index c09bafd859..7ddac9337d 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -176,6 +176,9 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
   int64_t     tver;
   SMetaEntry  me = {0};
   SCoder      coder = {0};
+  int8_t      type;
+  int64_t     ctime;
+  tb_uid_t    suid;
   int         c, ret;
 
   // search & delete the name idx
@@ -230,16 +233,56 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
     return -1;
   }
 
-  // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER);
-  // ret = metaDecodeEntry(&coder, &me);
-  // if (ret < 0) {
-  //   ASSERT(0);
-  //   return -1;
-  // }
+  // decode entry
+  void *pDataCopy = taosMemoryMalloc(nData);  // remove the copy (todo)
+  memcpy(pDataCopy, pData, nData);
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER);
+  ret = metaDecodeEntry(&coder, &me);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
+  type = me.type;
+  if (type == TSDB_CHILD_TABLE) {
+    ctime = me.ctbEntry.ctime;
+    suid = me.ctbEntry.suid;
+  } else if (type == TSDB_NORMAL_TABLE) {
+    ctime = me.ntbEntry.ctime;
+    suid = 0;
+  } else {
+    ASSERT(0);
+  }
+
+  taosMemoryFree(pDataCopy);
   tCoderClear(&coder);
   tdbDbcClose(pTbDbc);
 
+  if (type == TSDB_CHILD_TABLE) {
+    // remove the pCtbIdx
+    TDBC *pCtbIdxc = NULL;
+    tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
+
+    ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c);
+    if (ret < 0 || c != 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    tdbDbcDelete(pCtbIdxc);
+    tdbDbcClose(pCtbIdxc);
+
+    // remove tags from pTagIdx (todo)
+  } else if (type == TSDB_NORMAL_TABLE) {
+    // remove from pSkmDb
+  } else {
+    ASSERT(0);
+  }
+
+  // remove from ttl (todo)
+  if (ctime > 0) {
+  }
+
   return 0;
 }
 

From 9d2df3531670cb37266aa73fb71ae9a7cf5ff105 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 4 May 2022 07:07:34 +0000
Subject: [PATCH 12/24] drop super table

---
 source/dnode/vnode/src/meta/metaTable.c | 75 +++++++++++++++----------
 source/dnode/vnode/src/vnd/vnodeSvr.c   |  8 +--
 2 files changed, 50 insertions(+), 33 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 7ddac9337d..397e074061 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -72,44 +72,61 @@ _err:
 }
 
 int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
-  SMetaReader mr = {0};
+  TDBC       *pNameIdxc = NULL;
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pCtbIdxc = NULL;
+  SCtbIdxKey *pCtbIdxKey;
+  const void *pKey = NULL;
+  int         nKey;
+  const void *pData = NULL;
+  int         nData;
+  int         c, ret;
 
-  // validate req
-  metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) {
-    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
+  // prepare uid idx cursor
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
+  if (ret < 0 || c != 0) {
+    terrno = TSDB_CODE_VND_TB_NOT_EXIST;
+    tdbDbcClose(pUidIdxc);
     goto _err;
   }
 
-  // do drop
-  // drop from pTbDb
-  // drop from pSkmDb
-  // drop from pUidIdx
-  // drop from pNameIdx
-  // {
-  //   TDBC *pDbc1 = NULL;
-  //   void *pKey = NULL;
-  //   void *pVal = NULL;
-  //   int   kLen = 0;
-  //   int   vLen = 0;
-  //   int   ret = 0;
+  // prepare name idx cursor
+  tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+  }
 
-  //   // drop from pCtbIdx
-  //   ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1);
-  //   tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/);
-  //   tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen);
-  //   tdbDbcDelete(pDbc1);
-  //   // drop from pTagIdx
-  //   // drop from pTtlIdx
-  // }
+  tdbDbcDelete(pUidIdxc);
+  tdbDbcDelete(pNameIdxc);
+  tdbDbcClose(pUidIdxc);
+  tdbDbcClose(pNameIdxc);
 
-  // clear and return
-  metaReaderClear(&mr);
-  metaError("vgId:%d  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
+  // loop to drop each child table
+  tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
+  if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) {
+    tdbDbcClose(pCtbIdxc);
+    goto _exit;
+  }
+
+  for (;;) {
+    tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL);
+    pCtbIdxKey = (SCtbIdxKey *)pKey;
+
+    if (pCtbIdxKey->suid > pReq->suid) break;
+
+    // drop the child table (TODO)
+
+    if (tdbDbcMoveToNext(pCtbIdxc) < 0) break;
+  }
+
+_exit:
+  metaDebug("vgId:%d  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
   return 0;
 
 _err:
-  metaReaderClear(&mr);
   metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name,
             pReq->suid, tstrerror(terrno));
   return -1;
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 878dd5fca4..7d02840600 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i
   }
 
   // process request
-  // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) {
-  //   rcode = terrno;
-  //   goto _exit;
-  // }
+  if (metaDropSTable(pVnode->pMeta, version, &req) < 0) {
+    rcode = terrno;
+    goto _exit;
+  }
 
   // return rsp
 _exit:

From c734e2d9a19ed6033ff02fd0a93b9d830e6a261e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 4 May 2022 07:15:50 +0000
Subject: [PATCH 13/24] make case pass

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 7d02840600..878dd5fca4 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i
   }
 
   // process request
-  if (metaDropSTable(pVnode->pMeta, version, &req) < 0) {
-    rcode = terrno;
-    goto _exit;
-  }
+  // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) {
+  //   rcode = terrno;
+  //   goto _exit;
+  // }
 
   // return rsp
 _exit:

From 5b2b4da11bf0c9cd5604a456b35ec1d83b7275a8 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 4 May 2022 16:49:24 +0800
Subject: [PATCH 14/24] refactor: fix: reset confict trans types

---
 include/util/taoserror.h               |  3 ++-
 source/dnode/mnode/impl/src/mndTrans.c | 31 ++++++++++++++------------
 source/util/src/terror.c               |  3 ++-
 3 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index b8baf99552..abc752955d 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -264,7 +264,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TRANS_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03D0)
 #define TSDB_CODE_MND_TRANS_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03D1)
 #define TSDB_CODE_MND_TRANS_INVALID_STAGE       TAOS_DEF_ERROR_CODE(0, 0x03D2)
-#define TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL    TAOS_DEF_ERROR_CODE(0, 0x03D4)
+#define TSDB_CODE_MND_TRANS_CONFLICT            TAOS_DEF_ERROR_CODE(0, 0x03D3)
+#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR        TAOS_DEF_ERROR_CODE(0, 0x03D4)
 
 // mnode-mq
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E0)
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index ce2a0efd2f..1c238d4082 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -710,12 +710,12 @@ static bool mndIsStbTrans(STrans *pTrans) {
   return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END;
 }
 
-static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) {
+static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) {
   STrans *pTrans = NULL;
   void   *pIter = NULL;
-  bool    canParallel = true;
+  bool    conflict = false;
 
-  if (mndIsBasicTrans(pNewTrans)) return canParallel;
+  if (mndIsBasicTrans(pNewTrans)) return conflict;
 
   while (1) {
     pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@@ -724,7 +724,7 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) {
     if (mndIsGlobalTrans(pNewTrans)) {
       if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) {
         mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname);
-        canParallel = false;
+        conflict = true;
       } else {
       }
     }
@@ -732,11 +732,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) {
     else if (mndIsDbTrans(pNewTrans)) {
       if (mndIsGlobalTrans(pTrans)) {
         mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id);
-        canParallel = false;
+        conflict = true;
       } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) {
         if (pNewTrans->dbUid == pTrans->dbUid) {
           mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname);
-          canParallel = false;
+          conflict = true;
         }
       } else {
       }
@@ -745,11 +745,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) {
     else if (mndIsStbTrans(pNewTrans)) {
       if (mndIsGlobalTrans(pTrans)) {
         mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id);
-        canParallel = false;
+        conflict = true;
       } else if (mndIsDbTrans(pTrans)) {
         if (pNewTrans->dbUid == pTrans->dbUid) {
           mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname);
-          canParallel = false;
+          conflict = true;
         }
       } else {
       }
@@ -760,12 +760,12 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) {
 
   sdbCancelFetch(pMnode->pSdb, pIter);
   sdbRelease(pMnode->pSdb, pTrans);
-  return canParallel;
+  return conflict;
 }
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
-  if (!mndCheckTransCanParallel(pMnode, pTrans)) {
-    terrno = TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL;
+  if (mndCheckTransConflict(pMnode, pTrans)) {
+    terrno = TSDB_CODE_MND_TRANS_CONFLICT;
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     return -1;
   }
@@ -819,7 +819,8 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
 }
 
 static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
-  bool sendRsp = false;
+  bool    sendRsp = false;
+  int32_t code = pTrans->code;
 
   if (pTrans->stage == TRN_STAGE_FINISHED) {
     sendRsp = true;
@@ -829,12 +830,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
     if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION ||
         pTrans->stage == TRN_STAGE_ROLLBACK) {
       sendRsp = true;
+      if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
     }
   }
 
   if (pTrans->policy == TRN_POLICY_RETRY) {
     if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) {
       sendRsp = true;
+      if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
     }
   }
 
@@ -845,13 +848,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
     }
     taosMemoryFree(pTrans->rpcRsp);
 
-    mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage,
+    mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage,
            pTrans->rpcAHandle);
     SRpcMsg rspMsg = {
         .handle = pTrans->rpcHandle,
         .ahandle = pTrans->rpcAHandle,
         .refId = pTrans->rpcRefId,
-        .code = pTrans->code,
+        .code = code,
         .pCont = rpcCont,
         .contLen = pTrans->rpcRspLen,
     };
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index bd787d50b0..00fe8bd0e9 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -271,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE,    "Invalid func retrieve
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST,      "Transaction already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST,          "Transaction not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE,      "Invalid stage to kill")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL,   "Conflicting transaction not completed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT,           "Conflict transaction not completed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR,       "Unknown transaction error")
 
 // mnode-mq
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST,      "Topic already exists")

From 71021334fb6a928f38a0e926010e6ce51c7388b4 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 4 May 2022 20:37:37 +0800
Subject: [PATCH 15/24] refactor: adjust transaction code

---
 source/dnode/mnode/impl/src/mndTrans.c        | 12 +++---
 source/dnode/mnode/impl/src/mndUser.c         |  3 --
 source/dnode/mnode/impl/test/trans/trans2.cpp | 39 ++++++++++++++-----
 3 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 1c238d4082..e80a16b16e 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -829,15 +829,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
   if (pTrans->policy == TRN_POLICY_ROLLBACK) {
     if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION ||
         pTrans->stage == TRN_STAGE_ROLLBACK) {
-      sendRsp = true;
       if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
+      sendRsp = true;
     }
-  }
-
-  if (pTrans->policy == TRN_POLICY_RETRY) {
+  } else {
     if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) {
-      sendRsp = true;
       if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
+      sendRsp = true;
     }
   }
 
@@ -1102,8 +1100,8 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
   } else {
     pTrans->code = terrno;
     if (pTrans->policy == TRN_POLICY_ROLLBACK) {
-      pTrans->stage = TRN_STAGE_REDO_ACTION;
-      mError("trans:%d, stage from commit to redoAction since %s, failedTimes:%d", pTrans->id, terrstr(),
+      pTrans->stage = TRN_STAGE_UNDO_ACTION;
+      mError("trans:%d, stage from commit to undoAction since %s, failedTimes:%d", pTrans->id, terrstr(),
              pTrans->failedTimes);
       continueExec = true;
     } else {
diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index 1800d24cf2..5e15bdeb43 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -275,9 +275,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
   }
   sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
 
-  char *param = strdup("====> test code to be deleted later <=====");
-  mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
-
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index 252cd105a0..aaf2c649e4 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -19,7 +19,7 @@ void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {
 
 class MndTestTrans2 : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() {
+  static void InitLog() {
     dDebugFlag = 143;
     vDebugFlag = 0;
     mDebugFlag = 207;
@@ -42,9 +42,9 @@ class MndTestTrans2 : public ::testing::Test {
     if (taosInitLog("taosdlog", 1) != 0) {
       printf("failed to init log file\n");
     }
+  }
 
-    walInit();
-
+  static void InitMnode() {
     static SMsgCb msgCb = {0};
     msgCb.reportStartupFp = reportStartup;
     msgCb.pWrapper = (SMgmtWrapper *)(&msgCb);  // hack
@@ -63,6 +63,12 @@ class MndTestTrans2 : public ::testing::Test {
     pMnode = mndOpen(mnodepath, &opt);
   }
 
+  static void SetUpTestSuite() {
+    InitLog();
+    walInit();
+    InitMnode();
+  }
+
   static void TearDownTestSuite() {
     mndClose(pMnode);
     walCleanUp();
@@ -76,11 +82,11 @@ class MndTestTrans2 : public ::testing::Test {
   void SetUp() override {}
   void TearDown() override {}
 
-  void CreateUser(const char *user) {
+  int32_t CreateUser(const char *acct, const char *user) {
     SUserObj userObj = {0};
     taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
     tstrncpy(userObj.user, user, TSDB_USER_LEN);
-    tstrncpy(userObj.acct, "root", TSDB_USER_LEN);
+    tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
     userObj.createdTime = taosGetTimestampMs();
     userObj.updateTime = userObj.createdTime;
     userObj.superUser = 1;
@@ -94,19 +100,32 @@ class MndTestTrans2 : public ::testing::Test {
     char *param = strdup("====> test param <=====");
     mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
 
-    mndTransPrepare(pMnode, pTrans);
+    int32_t code = mndTransPrepare(pMnode, pTrans);
     mndTransDrop(pTrans);
+
+    return code;
   }
 };
 
 SMnode *MndTestTrans2::pMnode;
 
 TEST_F(MndTestTrans2, 01_CbFunc) {
+  const char *acct = "root";
+  const char *acct_invalid = "root1";
+  const char *user1 = "test1";
+  const char *user2 = "test2";
+  SUserObj   *pUser1 = NULL;
+  SUserObj   *pUser2 = NULL;
+
   ASSERT_NE(pMnode, nullptr);
 
-  const char *user1 = "test1";
-  CreateUser(user1);
-
-  SUserObj *pUser1 = mndAcquireUser(pMnode, user1);
+  // create user success
+  EXPECT_EQ(CreateUser(acct, user1), 0);
+  pUser1 = mndAcquireUser(pMnode, user1);
   ASSERT_NE(pUser1, nullptr);
+
+  // failed to create user and rollback
+  EXPECT_EQ(CreateUser(acct_invalid, user2), 0);
+  pUser2 = mndAcquireUser(pMnode, user2);
+  ASSERT_EQ(pUser2, nullptr);
 }

From fb7da7e566bedc0af30758785c29c30a7a4b2999 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 4 May 2022 21:02:27 +0800
Subject: [PATCH 16/24] enh(index): support index filter

---
 source/libs/index/inc/indexCache.h |  1 -
 source/libs/index/inc/indexFst.h   |  7 ++--
 source/libs/index/inc/indexInt.h   |  1 +
 source/libs/index/src/indexCache.c | 60 ++++++++++++++++++++++++++----
 source/libs/index/src/indexTfile.c | 43 +++++++++++++++++----
 5 files changed, 91 insertions(+), 21 deletions(-)

diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h
index 44355db7a7..3ea986ad48 100644
--- a/source/libs/index/inc/indexCache.h
+++ b/source/libs/index/inc/indexCache.h
@@ -56,7 +56,6 @@ typedef struct CacheTerm {
   int8_t   colType;
 
   SIndexOperOnColumn operaType;
-  int8_t             qType;  // query type
 } CacheTerm;
 //
 
diff --git a/source/libs/index/inc/indexFst.h b/source/libs/index/inc/indexFst.h
index 39ad5ffa8c..0a360c1c72 100644
--- a/source/libs/index/inc/indexFst.h
+++ b/source/libs/index/inc/indexFst.h
@@ -52,7 +52,6 @@ typedef struct FstRange {
   uint64_t end;
 } FstRange;
 
-typedef enum { GE, GT, LE, LT } RangeType;
 typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
 typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
 
@@ -174,9 +173,9 @@ Output   fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date,
 uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null);
 
 #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
-#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
-#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
-#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal)
+#define FST_STATE_ONE_TRNAS(node)      (node->state.state == OneTrans)
+#define FST_STATE_ANY_TRANS(node)      (node->state.state == AnyTrans)
+#define FST_STATE_EMPTY_FINAL(node)    (node->state.state == EmptyFinal)
 
 typedef struct FstLastTransition {
   uint8_t inp;
diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h
index 04817c4df3..5c7b8b9afe 100644
--- a/source/libs/index/inc/indexInt.h
+++ b/source/libs/index/inc/indexInt.h
@@ -34,6 +34,7 @@
 extern "C" {
 #endif
 
+typedef enum { LT, LE, GT, GE } RangeType;
 typedef enum { kTypeValue, kTypeDeletion } STermValueType;
 
 typedef struct SIndexStat {
diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c
index ea3cbf5658..13768ce682 100644
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@@ -43,6 +43,20 @@ static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult*
 static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
 static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+/*comm func of compare, used in (LE/LT/GE/GT compare)*/
+static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+                                      RangeType type);
+
+typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
+typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
+
+static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; }
+
+static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
+                                                                    tCompareGreaterThan, tCompareGreaterEqual};
 
 static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
     cacheSearchTerm,      cacheSearchPrefix,      cacheSearchSuffix,       cacheSearchRegex, cacheSearchLessThan,
@@ -93,21 +107,51 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr,
   // impl later
   return 0;
 }
+static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+                                      RangeType type) {
+  if (cache == NULL) {
+    return 0;
+  }
+  _cache_range_compare cmpFn = rangeCompare[type];
+
+  MemTable* mem = cache;
+  char*     key = indexCacheTermGet(ct);
+
+  SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
+  while (tSkipListIterNext(iter)) {
+    SSkipListNode* node = tSkipListIterGet(iter);
+    if (node == NULL) {
+      break;
+    }
+    CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+    TExeCond   cond = cmpFn(c->colVal, ct->colVal, ct->colType);
+    if (cond == MATCH) {
+      if (c->operaType == ADD_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        // taosArrayPush(result, &c->uid);
+        *s = kTypeValue;
+      } else if (c->operaType == DEL_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+      }
+    } else if (cond == CONTINUE) {
+    } else if (cond == BREAK) {
+      break;
+    }
+  }
+  tSkipListDestroyIter(iter);
+  return TSDB_CODE_SUCCESS;
+}
 static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
-  // impl later
-  return 0;
+  return cacheSearchCompareFunc(cache, ct, tr, s, LT);
 }
 static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
-  // impl later
-  return 0;
+  return cacheSearchCompareFunc(cache, ct, tr, s, LE);
 }
 static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
-  // impl later
-  return 0;
+  return cacheSearchCompareFunc(cache, ct, tr, s, GT);
 }
 static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
-  // impl later
-  return 0;
+  return cacheSearchCompareFunc(cache, ct, tr, s, GE);
 }
 static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
   // impl later
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 327a06084c..5aed2bd6b0 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -70,6 +70,8 @@ static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult
 static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
 
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
+
 static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
     tfSearchTerm,      tfSearchPrefix,      tfSearchSuffix,       tfSearchRegex, tfSearchLessThan,
     tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange};
@@ -304,21 +306,46 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   fstSliceDestroy(&key);
   return 0;
 }
+
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) {
+  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
+
+  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
+  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+
+  FstSlice h = fstSliceCreate((uint8_t*)p, sz);
+  fstStreamBuilderSetRange(sb, &h, type);
+  fstSliceDestroy(&h);
+
+  StreamWithState*       st = streamBuilderIntoStream(sb);
+  StreamWithStateResult* rt = NULL;
+  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    taosArrayPush(offsets, &(rt->out.out));
+    swsResultDestroy(rt);
+  }
+  streamWithStateDestroy(st);
+  fstStreamBuilderDestroy(sb);
+  return TSDB_CODE_SUCCESS;
+}
 static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
-  // impl later
-  return 0;
+  return tfSearchCompareFunc(reader, tem, tr, LT);
 }
 static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
-  // impl later
-  return 0;
+  return tfSearchCompareFunc(reader, tem, tr, LE);
 }
 static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
-  // impl later
-  return 0;
+  return tfSearchCompareFunc(reader, tem, tr, GT);
 }
 static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
-  // impl later
-  return 0;
+  return tfSearchCompareFunc(reader, tem, tr, GE);
 }
 static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
   bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);

From 2f42c2e7933f2231c15f2236fe17b86d70c49fca Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 4 May 2022 22:00:04 +0800
Subject: [PATCH 17/24] refactor: adjust transaction code

---
 include/common/tglobal.h                      |  4 ++
 source/common/src/tglobal.c                   |  4 ++
 source/common/src/tmsgcb.c                    | 62 ++++++++++++++++---
 source/dnode/mnode/impl/src/mndTrans.c        |  6 +-
 source/dnode/mnode/impl/src/mnode.c           | 11 ++--
 source/dnode/mnode/impl/test/trans/trans2.cpp |  6 ++
 6 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index fc3d575317..f253d31963 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -121,6 +121,10 @@ extern char     tsCompressor[];
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
+// internal 
+extern int32_t tsTransPullupMs;
+extern int32_t tsMaRebalanceMs;
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index bbf14b2fdc..4470f5b0a3 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500;                      // max range
 uint32_t tsCurRange = 100;                      // range
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
+// internal
+int32_t tsTransPullupMs = 6000;
+int32_t tsMaRebalanceMs = 2000;
+
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
   tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
   tsDiskCfg[index].level = level;
diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c
index c9cbb73884..42612cecb9 100644
--- a/source/common/src/tmsgcb.c
+++ b/source/common/src/tmsgcb.c
@@ -15,37 +15,83 @@
 
 #define _DEFAULT_SOURCE
 #include "tmsgcb.h"
+#include "taoserror.h"
 
 static SMsgCb tsDefaultMsgCb;
 
 void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; }
 
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) {
-  return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq);
+  PutToQueueFp fp = pMsgCb->queueFps[qtype];
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, pReq);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
-  return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
+  GetQueueSizeFp fp = pMsgCb->qsizeFp;
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, vgId, qtype);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
 int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
-  return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
+  SendReqFp fp = pMsgCb->sendReqFp;
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, epSet, pReq);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
-void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); }
+void tmsgSendRsp(const SRpcMsg* pRsp) {
+  SendRspFp fp = tsDefaultMsgCb.sendRspFp;
+  if (fp != NULL) {
+    return (*fp)(tsDefaultMsgCb.pWrapper, pRsp);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
+}
 
 void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) {
-  return (*tsDefaultMsgCb.sendRedirectRspFp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet);
+  SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp;
+  if (fp != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
-  (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg);
+  RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp;
+  if (fp != NULL) {
+    (*fp)(pMsgCb->pWrapper, pMsg);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgReleaseHandle(void* handle, int8_t type) {
-  (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type);
+  ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp;
+  if (fp != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, handle, type);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgReportStartup(const char* name, const char* desc) {
-  (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc);
+  ReportStartup fp = tsDefaultMsgCb.reportStartupFp;
+  if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, name, desc);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index e80a16b16e..6110d0be44 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -1126,7 +1126,7 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) {
   } else {
     pTrans->code = terrno;
     pTrans->failedTimes++;
-    mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr());
+    mError("trans:%d, stage keep on commitLog since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes);
     continueExec = false;
   }
 
@@ -1162,7 +1162,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) {
     continueExec = false;
   } else {
     pTrans->failedTimes++;
-    mError("trans:%d, stage keep on undoAction since %s", pTrans->id, terrstr());
+    mError("trans:%d, stage keep on undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes);
     continueExec = false;
   }
 
@@ -1179,7 +1179,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
     continueExec = true;
   } else {
     pTrans->failedTimes++;
-    mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr());
+    mError("trans:%d, stage keep on rollback since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes);
     continueExec = false;
   }
 
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index b38fd6178c..e2814e95f0 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -43,9 +43,6 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 
-#define MQ_TIMER_MS    2000
-#define TRNAS_TIMER_MS 6000
-
 static void *mndBuildTimerMsg(int32_t *pContLen) {
   SMTimerReq timerReq = {0};
 
@@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) {
     tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
   }
 
-  taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer);
+  taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer);
 }
 
 static void mndCalMqRebalance(void *param, void *tmrId) {
@@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 
-  taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer);
+  taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer);
 }
 
 static void mndPullupTelem(void *param, void *tmrId) {
@@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) {
     return -1;
   }
 
-  if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) {
+  if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) {
+  if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index aaf2c649e4..1fabf677c4 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -58,9 +58,12 @@ class MndTestTrans2 : public ::testing::Test {
     strcpy(opt.replicas[0].fqdn, "localhost");
     opt.msgCb = msgCb;
 
+    tsTransPullupMs = 1000;
+
     const char *mnodepath = "/tmp/mnode_test_trans";
     taosRemoveDir(mnodepath);
     pMnode = mndOpen(mnodepath, &opt);
+    mndStart(pMnode);
   }
 
   static void SetUpTestSuite() {
@@ -70,6 +73,7 @@ class MndTestTrans2 : public ::testing::Test {
   }
 
   static void TearDownTestSuite() {
+    mndStop(pMnode);
     mndClose(pMnode);
     walCleanUp();
     taosCloseLog();
@@ -128,4 +132,6 @@ TEST_F(MndTestTrans2, 01_CbFunc) {
   EXPECT_EQ(CreateUser(acct_invalid, user2), 0);
   pUser2 = mndAcquireUser(pMnode, user2);
   ASSERT_EQ(pUser2, nullptr);
+
+  mndTransPullup(pMnode);
 }

From 0218def49a41c7391d762ce6ee544a02292b4747 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 4 May 2022 23:58:42 +0800
Subject: [PATCH 18/24] test: add action test for transaction

---
 source/dnode/mnode/impl/inc/mndTrans.h        |   2 +
 source/dnode/mnode/impl/src/mndTrans.c        |  45 ++++--
 source/dnode/mnode/impl/test/trans/trans2.cpp | 128 ++++++++++++++++--
 3 files changed, 156 insertions(+), 19 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h
index 7644ec3c4c..2a011f414d 100644
--- a/source/dnode/mnode/impl/inc/mndTrans.h
+++ b/source/dnode/mnode/impl/inc/mndTrans.h
@@ -44,6 +44,8 @@ typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
 
 int32_t mndInitTrans(SMnode *pMnode);
 void    mndCleanupTrans(SMnode *pMnode);
+STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId);
+void    mndReleaseTrans(SMnode *pMnode, STrans *pTrans);
 
 STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq);
 void    mndTransDrop(STrans *pTrans);
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 6110d0be44..c5a31a6d85 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -537,7 +537,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
   return 0;
 }
 
-static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
+STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
   STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId);
   if (pTrans == NULL) {
     terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
@@ -545,7 +545,7 @@ static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
   return pTrans;
 }
 
-static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) {
+void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) {
   SSdb *pSdb = pMnode->pSdb;
   sdbRelease(pSdb, pTrans);
 }
@@ -919,27 +919,41 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
 
   if (arraySize == 0) return 0;
 
+  int32_t code = 0;
   for (int32_t i = 0; i < arraySize; ++i) {
     SSdbRaw *pRaw = taosArrayGetP(pArray, i);
-    int32_t  code = sdbWriteWithoutFree(pSdb, pRaw);
-    if (code != 0) {
-      return code;
+    if (sdbWriteWithoutFree(pSdb, pRaw) != 0) {
+      code = ((terrno != 0) ? terrno : -1);
     }
   }
 
-  return 0;
+  terrno = code;
+  return code;
 }
 
 static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
-  return mndTransExecuteLogs(pMnode, pTrans->redoLogs);
+  int32_t code = mndTransExecuteLogs(pMnode, pTrans->redoLogs);
+  if (code != 0) {
+    mError("failed to execute redoLogs since %s", terrstr());
+  }
+  return code;
 }
 
 static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
-  return mndTransExecuteLogs(pMnode, pTrans->undoLogs);
+  int32_t code = mndTransExecuteLogs(pMnode, pTrans->undoLogs);
+  if (code != 0) {
+    mError("failed to execute undoLogs since %s, return success", terrstr());
+  }
+
+  return 0;  // return success in any case
 }
 
 static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
-  return mndTransExecuteLogs(pMnode, pTrans->commitLogs);
+  int32_t code = mndTransExecuteLogs(pMnode, pTrans->commitLogs);
+  if (code != 0) {
+    mError("failed to execute commitLogs since %s", terrstr());
+  }
+  return code;
 }
 
 static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
@@ -983,6 +997,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr
       pAction->msgReceived = 0;
       pAction->errCode = 0;
     } else {
+      if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont);
       mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr());
       return -1;
     }
@@ -1029,11 +1044,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
 }
 
 static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
-  return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions);
+  int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions);
+  if (code != 0) {
+    mError("failed to execute redoActions since %s", terrstr());
+  }
+  return code;
 }
 
 static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
-  return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions);
+  int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions);
+  if (code != 0) {
+    mError("failed to execute undoActions since %s", terrstr());
+  }
+  return code;
 }
 
 static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index 1fabf677c4..e9d3b54c1d 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -86,7 +86,7 @@ class MndTestTrans2 : public ::testing::Test {
   void SetUp() override {}
   void TearDown() override {}
 
-  int32_t CreateUser(const char *acct, const char *user) {
+  int32_t CreateUserLog(const char *acct, const char *user) {
     SUserObj userObj = {0};
     taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
     tstrncpy(userObj.user, user, TSDB_USER_LEN);
@@ -101,7 +101,11 @@ class MndTestTrans2 : public ::testing::Test {
     mndTransAppendRedolog(pTrans, pRedoRaw);
     sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
 
-    char *param = strdup("====> test param <=====");
+    SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendUndolog(pTrans, pUndoRaw);
+    sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+    char *param = strdup("====> test log <=====");
     mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
 
     int32_t code = mndTransPrepare(pMnode, pTrans);
@@ -109,29 +113,137 @@ class MndTestTrans2 : public ::testing::Test {
 
     return code;
   }
+
+  int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) {
+    SUserObj userObj = {0};
+    taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
+    tstrncpy(userObj.user, user, TSDB_USER_LEN);
+    tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
+    userObj.createdTime = taosGetTimestampMs();
+    userObj.updateTime = userObj.createdTime;
+    userObj.superUser = 1;
+
+    SRpcMsg  rpcMsg = {0};
+    STrans  *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg);
+    SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendRedolog(pTrans, pRedoRaw);
+    sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+    SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendUndolog(pTrans, pUndoRaw);
+    sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+    char *param = strdup("====> test action <=====");
+    mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
+    {
+      STransAction action = {0};
+      action.epSet.inUse = 0;
+      action.epSet.numOfEps = 1;
+      action.epSet.eps[0].port = 9040;
+      strcpy(action.epSet.eps[0].fqdn, "localhost");
+
+      int32_t contLen = 1024;
+      void   *pReq = taosMemoryCalloc(1, contLen);
+      strcpy((char *)pReq, "hello world redo");
+      action.pCont = pReq;
+      action.contLen = contLen;
+      action.msgType = TDMT_DND_CREATE_MNODE;
+      action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+      mndTransAppendRedoAction(pTrans, &action);
+    }
+
+    if (hasUndoAction) {
+      STransAction action = {0};
+      action.epSet.inUse = 0;
+      action.epSet.numOfEps = 1;
+      action.epSet.eps[0].port = 9040;
+      strcpy(action.epSet.eps[0].fqdn, "localhost");
+
+      int32_t contLen = 1024;
+      void   *pReq = taosMemoryCalloc(1, contLen);
+      strcpy((char *)pReq, "hello world undo");
+      action.pCont = pReq;
+      action.contLen = contLen;
+      action.msgType = TDMT_DND_CREATE_MNODE;
+      action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+      mndTransAppendUndoAction(pTrans, &action);
+    }
+
+    int32_t code = mndTransPrepare(pMnode, pTrans);
+    mndTransDrop(pTrans);
+
+    return code;
+  }
 };
 
 SMnode *MndTestTrans2::pMnode;
 
-TEST_F(MndTestTrans2, 01_CbFunc) {
+TEST_F(MndTestTrans2, 01_Log) {
   const char *acct = "root";
   const char *acct_invalid = "root1";
-  const char *user1 = "test1";
-  const char *user2 = "test2";
+  const char *user1 = "log1";
+  const char *user2 = "log2";
   SUserObj   *pUser1 = NULL;
   SUserObj   *pUser2 = NULL;
 
   ASSERT_NE(pMnode, nullptr);
 
-  // create user success
-  EXPECT_EQ(CreateUser(acct, user1), 0);
+  EXPECT_EQ(CreateUserLog(acct, user1), 0);
   pUser1 = mndAcquireUser(pMnode, user1);
   ASSERT_NE(pUser1, nullptr);
 
   // failed to create user and rollback
-  EXPECT_EQ(CreateUser(acct_invalid, user2), 0);
+  EXPECT_EQ(CreateUserLog(acct_invalid, user2), 0);
   pUser2 = mndAcquireUser(pMnode, user2);
   ASSERT_EQ(pUser2, nullptr);
 
   mndTransPullup(pMnode);
 }
+
+TEST_F(MndTestTrans2, 02_Action) {
+  const char *acct = "root";
+  const char *acct_invalid = "root1";
+  const char *user1 = "action1";
+  const char *user2 = "action2";
+  SUserObj   *pUser1 = NULL;
+  SUserObj   *pUser2 = NULL;
+  STrans     *pTrans = NULL;
+  int32_t     transId = 4;
+  int32_t     action = 0;
+
+  ASSERT_NE(pMnode, nullptr);
+
+  // failed to create user and rollback
+  EXPECT_EQ(CreateUserAction(acct, user1, false), 0);
+  pUser1 = mndAcquireUser(pMnode, user1);
+  ASSERT_EQ(pUser1, nullptr);
+  mndReleaseUser(pMnode, pUser1);
+
+  // create user, and fake a response
+  EXPECT_EQ(CreateUserAction(acct, user1, true), 0);
+  pUser1 = mndAcquireUser(pMnode, user1);
+  ASSERT_NE(pUser1, nullptr);
+  mndReleaseUser(pMnode, pUser1);
+
+  pTrans = mndAcquireTrans(pMnode, transId);
+  EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+  EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+  EXPECT_EQ(pTrans->failedTimes, 1);
+
+  STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action);
+  pAction->msgSent = 1;
+
+  SNodeMsg rspMsg = {0};
+  rspMsg.pNode = pMnode;
+  int64_t signature = transId;
+  signature = (signature << 32);
+  signature += action;
+  rspMsg.rpcMsg.ahandle = (void *)signature;
+  mndTransProcessRsp(&rspMsg);
+  mndReleaseTrans(pMnode, pTrans);
+
+  pUser1 = mndAcquireUser(pMnode, user1);
+  ASSERT_EQ(pUser1, nullptr);
+  mndReleaseUser(pMnode, pUser1);
+}

From 43822106bc35ee92e930739244fdcd2d6c9bca2b Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 5 May 2022 00:12:11 +0800
Subject: [PATCH 19/24] test: add unitest for transaction

---
 source/dnode/mnode/impl/test/trans/trans2.cpp | 109 ++++++++++++++----
 1 file changed, 84 insertions(+), 25 deletions(-)

diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index e9d3b54c1d..599f1882d4 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -114,7 +114,7 @@ class MndTestTrans2 : public ::testing::Test {
     return code;
   }
 
-  int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) {
+  int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy) {
     SUserObj userObj = {0};
     taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
     tstrncpy(userObj.user, user, TSDB_USER_LEN);
@@ -124,7 +124,7 @@ class MndTestTrans2 : public ::testing::Test {
     userObj.superUser = 1;
 
     SRpcMsg  rpcMsg = {0};
-    STrans  *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg);
+    STrans  *pTrans = mndTransCreate(pMnode, policy, TRN_TYPE_CREATE_USER, &rpcMsg);
     SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
     mndTransAppendRedolog(pTrans, pRedoRaw);
     sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
@@ -209,41 +209,100 @@ TEST_F(MndTestTrans2, 02_Action) {
   SUserObj   *pUser1 = NULL;
   SUserObj   *pUser2 = NULL;
   STrans     *pTrans = NULL;
-  int32_t     transId = 4;
+  int32_t     transId = 0;
   int32_t     action = 0;
 
   ASSERT_NE(pMnode, nullptr);
 
   // failed to create user and rollback
-  EXPECT_EQ(CreateUserAction(acct, user1, false), 0);
+  EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0);
   pUser1 = mndAcquireUser(pMnode, user1);
   ASSERT_EQ(pUser1, nullptr);
   mndReleaseUser(pMnode, pUser1);
 
   // create user, and fake a response
-  EXPECT_EQ(CreateUserAction(acct, user1, true), 0);
-  pUser1 = mndAcquireUser(pMnode, user1);
-  ASSERT_NE(pUser1, nullptr);
-  mndReleaseUser(pMnode, pUser1);
+  {
+    EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_NE(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
 
-  pTrans = mndAcquireTrans(pMnode, transId);
-  EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
-  EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
-  EXPECT_EQ(pTrans->failedTimes, 1);
+    transId = 4;
+    pTrans = mndAcquireTrans(pMnode, transId);
+    EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+    EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+    EXPECT_EQ(pTrans->failedTimes, 1);
 
-  STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action);
-  pAction->msgSent = 1;
+    STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action);
+    pAction->msgSent = 1;
 
-  SNodeMsg rspMsg = {0};
-  rspMsg.pNode = pMnode;
-  int64_t signature = transId;
-  signature = (signature << 32);
-  signature += action;
-  rspMsg.rpcMsg.ahandle = (void *)signature;
-  mndTransProcessRsp(&rspMsg);
-  mndReleaseTrans(pMnode, pTrans);
+    SNodeMsg rspMsg = {0};
+    rspMsg.pNode = pMnode;
+    int64_t signature = transId;
+    signature = (signature << 32);
+    signature += action;
+    rspMsg.rpcMsg.ahandle = (void *)signature;
+    mndTransProcessRsp(&rspMsg);
+    mndReleaseTrans(pMnode, pTrans);
 
-  pUser1 = mndAcquireUser(pMnode, user1);
-  ASSERT_EQ(pUser1, nullptr);
-  mndReleaseUser(pMnode, pUser1);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_EQ(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+  }
+
+  {
+    EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY), 0);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_NE(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+
+    {
+      transId = 5;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 1);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_NE(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+    }
+
+    {
+      transId = 5;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 2);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_NE(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+    }
+  }
 }

From 7c5c1c32ee48b3a49728f654522e06f2936d7687 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 5 May 2022 03:17:34 +0000
Subject: [PATCH 20/24] implement TDB upsert

---
 source/libs/tdb/inc/tdb.h         |   2 +-
 source/libs/tdb/src/db/tdbBtree.c | 148 ++++++++++++++++++++----------
 source/libs/tdb/src/db/tdbDb.c    |   5 +-
 source/libs/tdb/src/inc/tdbInt.h  |   3 +-
 source/libs/tdb/test/tdbTest.cpp  |  69 +++++++++++++-
 5 files changed, 170 insertions(+), 57 deletions(-)

diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h
index cab242d5b3..9e79ec1d89 100644
--- a/source/libs/tdb/inc/tdb.h
+++ b/source/libs/tdb/inc/tdb.h
@@ -42,7 +42,7 @@ int tdbDbClose(TDB *pDb);
 int tdbDbDrop(TDB *pDb);
 int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
 int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn);
-int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
+int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index ed7aed552a..0d13ec4d2b 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -138,67 +138,25 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in
   }
 
   if (btc.idx == -1) {
-    idx = 0;
+    btc.idx = 0;
   } else {
     if (c > 0) {
-      idx = btc.idx + 1;
-    } else if (c < 0) {
-      idx = btc.idx;
-    } else {
-      // TDB does NOT allow same key
-      tdbBtcClose(&btc);
+      btc.idx++;
+    } else if (c == 0) {
+      // dup key not allowed
       ASSERT(0);
       return -1;
     }
   }
 
-  // make sure enough space to hold the cell
-  szBuf = kLen + vLen + 14;
-  pBuf = tdbRealloc(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize);
-  if (pBuf == NULL) {
-    tdbBtcClose(&btc);
-    ASSERT(0);
-    return -1;
-  }
-  pBt->pBuf = pBuf;
-  pCell = (SCell *)pBt->pBuf;
-
-  // encode cell
-  ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
+  ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1);
   if (ret < 0) {
-    tdbBtcClose(&btc);
     ASSERT(0);
-    return -1;
-  }
-
-  // mark the page dirty
-  ret = tdbPagerWrite(pBt->pPager, btc.pPage);
-  if (ret < 0) {
     tdbBtcClose(&btc);
-    ASSERT(0);
     return -1;
   }
 
-  // insert the cell
-  ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0);
-  if (ret < 0) {
-    tdbBtcClose(&btc);
-    ASSERT(0);
-    return -1;
-  }
-
-  // check if need balance
-  if (btc.pPage->nOverflow > 0) {
-    ret = tdbBtreeBalance(&btc);
-    if (ret < 0) {
-      tdbBtcClose(&btc);
-      ASSERT(0);
-      return -1;
-    }
-  }
-
   tdbBtcClose(&btc);
-
   return 0;
 }
 
@@ -232,6 +190,41 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
   return 0;
 }
 
+int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn) {
+  SBTC btc;
+  int  c;
+  int  ret;
+
+  tdbBtcOpen(&btc, pBt, pTxn);
+
+  // move the cursor
+  ret = tdbBtcMoveTo(&btc, pKey, nKey, &c);
+  if (ret < 0) {
+    ASSERT(0);
+    tdbBtcClose(&btc);
+    return -1;
+  }
+
+  if (btc.idx == -1) {
+    btc.idx = 0;
+    c = 1;
+  } else {
+    if (c > 0) {
+      btc.idx = btc.idx + 1;
+    }
+  }
+
+  ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c);
+  if (ret < 0) {
+    ASSERT(0);
+    tdbBtcClose(&btc);
+    return -1;
+  }
+
+  tdbBtcClose(&btc);
+  return 0;
+}
+
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) {
   return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen);
 }
@@ -1475,9 +1468,64 @@ int tdbBtcDelete(SBTC *pBtc) {
   return 0;
 }
 
-int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
-  ASSERT(0);
-  // TODO
+int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert) {
+  SCell *pCell;
+  int    szCell;
+  int    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+  int    szBuf;
+  void  *pBuf;
+  int    ret;
+
+  ASSERT(pBtc->idx >= 0);
+
+  // alloc space
+  szBuf = kLen + nData + 14;
+  pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize);
+  if (pBuf == NULL) {
+    ASSERT(0);
+    return -1;
+  }
+  pBtc->pBt->pBuf = pBuf;
+  pCell = (SCell *)pBtc->pBt->pBuf;
+
+  // encode cell
+  ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // mark dirty
+  ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // insert or update
+  if (insert) {
+    ASSERT(pBtc->idx <= nCells);
+
+    ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0);
+  } else {
+    ASSERT(pBtc->idx < nCells);
+
+    ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell);
+  }
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // check balance
+  if (pBtc->pPage->nOverflow > 0) {
+    ret = tdbBtreeBalance(pBtc);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
   return 0;
 }
 
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index 3358aefec0..c31441d849 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -81,9 +81,8 @@ int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int va
 
 int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); }
 
-int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
-  // TODO
-  return 0;
+int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
+  return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn);
 }
 
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index e18952505c..76dacf7b84 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -129,6 +129,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBT
 int tdbBtreeClose(SBTree *pBt);
 int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn);
+int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
@@ -143,7 +144,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
 int tdbBtcDelete(SBTC *pBtc);
-int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen);
+int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert);
 
 // tdbPager.c ====================================
 
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 994e53cd9d..e575ac156f 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -115,7 +115,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
   return cret;
 }
 
-TEST(tdb_test, simple_test) {
+TEST(tdb_test, simple_insert1) {
   int           ret;
   TENV         *pEnv;
   TDB          *pDb;
@@ -235,7 +235,7 @@ TEST(tdb_test, simple_test) {
   GTEST_ASSERT_EQ(ret, 0);
 }
 
-TEST(tdb_test, simple_test2) {
+TEST(tdb_test, simple_insert2) {
   int           ret;
   TENV         *pEnv;
   TDB          *pDb;
@@ -413,6 +413,71 @@ TEST(tdb_test, simple_delete1) {
 
   closePool(pPool);
 
+  tdbDbClose(pDb);
+  tdbEnvClose(pEnv);
+}
+
+TEST(tdb_test, simple_upsert1) {
+  int       ret;
+  TENV     *pEnv;
+  TDB      *pDb;
+  int       nData = 100000;
+  char      key[64];
+  char      data[64];
+  void     *pData = NULL;
+  SPoolMem *pPool;
+  TXN       txn;
+
+  taosRemoveDir("tdb");
+
+  // open env
+  ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open database
+  ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  pPool = openPool();
+  // insert some data
+  tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  tdbBegin(pEnv, &txn);
+
+  for (int iData = 0; iData < nData; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d", iData);
+    ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  // query the data
+  for (int iData = 0; iData < nData; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d", iData);
+    ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData);
+    GTEST_ASSERT_EQ(ret, 0);
+    GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
+  }
+
+  // upsert some data
+  for (int iData = 0; iData < nData; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d-u", iData);
+    ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  tdbCommit(pEnv, &txn);
+
+  // query the data
+  for (int iData = 0; iData < nData; iData++) {
+    sprintf(key, "key%d", iData);
+    sprintf(data, "data%d-u", iData);
+    ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData);
+    GTEST_ASSERT_EQ(ret, 0);
+    GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
+  }
+
   tdbDbClose(pDb);
   tdbEnvClose(pEnv);
 }
\ No newline at end of file

From c87eb6cc479353f85a60558d418fa0e736176f76 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 5 May 2022 03:24:00 +0000
Subject: [PATCH 21/24] refact

---
 source/libs/tdb/inc/tdb.h      |  4 +---
 source/libs/tdb/src/db/tdbDb.c | 16 ++++------------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h
index 9e79ec1d89..18f8ddec5c 100644
--- a/source/libs/tdb/inc/tdb.h
+++ b/source/libs/tdb/inc/tdb.h
@@ -55,11 +55,9 @@ int tdbDbcMoveToLast(TDBC *pDbc);
 int tdbDbcMoveToNext(TDBC *pDbc);
 int tdbDbcMoveToPrev(TDBC *pDbc);
 int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
-
-int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen);
-int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen);
 int tdbDbcDelete(TDBC *pDbc);
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
+int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert);
 
 // TXN
 #define TDB_TXN_WRITE            0x1
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index c31441d849..553bb2c646 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -123,24 +123,16 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in
   return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen);
 }
 
-int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
-
-int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
-
 int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); }
 
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
 }
 
+int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
+  return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert);
+}
+
 int tdbDbcClose(TDBC *pDbc) {
   if (pDbc) {
     tdbBtcClose(&pDbc->btc);

From af7000e3dc2ae6cae7583f0f4ee07f47b8053323 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Thu, 5 May 2022 11:48:52 +0800
Subject: [PATCH 22/24] feat: update taos-tools for 3.0 (#12112)

[TD-13052]
---
 tools/taos-tools | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/taos-tools b/tools/taos-tools
index bf6c766986..2f3dfddd4d 160000
--- a/tools/taos-tools
+++ b/tools/taos-tools
@@ -1 +1 @@
-Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32
+Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c

From 96372a7322f9f69765109a1d1af4ac11782f7291 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 5 May 2022 14:57:48 +0800
Subject: [PATCH 23/24] fix(query): set the correct join on column slot id.

---
 source/libs/executor/src/executorimpl.c | 35 +++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index b5d972321b..a91d204efa 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4830,6 +4830,7 @@ static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget);
 static SArray* createIndexMap(SNodeList* pNodeList);
 static SArray* extractPartitionColInfo(SNodeList* pNodeList);
 static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
+static void    setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode);
 
 static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) {
   SInterval interval = {
@@ -5624,25 +5625,29 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf
     goto _error;
   }
 
-  pOperator->resultInfo.capacity = 4096;
-  pOperator->resultInfo.threshold = 4096 * 0.75;
+  initResultSizeInfo(pOperator, 4096);
 
-  //  initResultRowInf
-  //  o(&pInfo->binfo.resultRowInfo, 8);
-  pInfo->pRes = pResBlock;
-
-  pOperator->name = "JoinOperator";
+  pInfo->pRes          = pResBlock;
+  pOperator->name      = "MergeJoinOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN;
-  pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->pExpr = pExprInfo;
+  pOperator->blocking  = false;
+  pOperator->status    = OP_NOT_OPENED;
+  pOperator->pExpr     = pExprInfo;
   pOperator->numOfOutput = numOfCols;
-  pOperator->info = pInfo;
+  pOperator->info      = pInfo;
   pOperator->pTaskInfo = pTaskInfo;
 
+  SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
+  setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
+  setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
+
   pOperator->fpSet =
       createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL);
   int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   return pOperator;
 
 _error:
@@ -5651,3 +5656,11 @@ _error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   return NULL;
 }
+
+void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
+  pColumn->slotId    = pColumnNode->slotId;
+  pColumn->type      = pColumnNode->node.resType.type;
+  pColumn->bytes     = pColumnNode->node.resType.bytes;
+  pColumn->precision = pColumnNode->node.resType.precision;
+  pColumn->scale     = pColumnNode->node.resType.scale;
+}

From d457f80afc829cd0ec06a8bdb84588ae660b478a Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Thu, 5 May 2022 15:55:11 +0800
Subject: [PATCH 24/24] fix: turn fma off by default (#12114)

[TD-15262]
---
 cmake/cmake.define | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmake/cmake.define b/cmake/cmake.define
index 851d8f9a73..1655154506 100644
--- a/cmake/cmake.define
+++ b/cmake/cmake.define
@@ -67,7 +67,13 @@ ELSE ()
     IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
         ADD_DEFINITIONS("-D_TD_ARM_")
     ELSE ()
-        ADD_DEFINITIONS("-msse4.2 -mfma")
+        ADD_DEFINITIONS("-msse4.2")
+        IF("${FMA_SUPPORT}" MATCHES "true")
+            MESSAGE(STATUS "turn fma function support on")
+            ADD_DEFINITIONS("-mfma")
+        ELSE ()
+            MESSAGE(STATUS "turn fma function support off")
+        ENDIF()
     ENDIF ()
 
 ENDIF ()