Merge branch '3.0' into feature/udf

include/common/taosdef.h

@@ -86,6 +86,11 @@ typedef enum {
   TSDB_RETENTION_MAX = 3
 } ERetentionLevel;
 
+typedef enum {
+  TSDB_BITMODE_DEFAULT = 0,  // 2 bits
+  TSDB_BITMODE_ONE_BIT = 1,  // 1 bit
+} EBitmapMode;
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP 11

include/common/tdataformat.h

@@ -313,8 +313,9 @@ typedef struct {
   SDataCol *cols;
 } SDataCols;
 
-static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; }
+static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; }
-static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; }
+static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; }
+static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; }
 
 #define keyCol(pCols)              (&((pCols)->cols[0]))                    // Key column
 #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]  // the idx row of column-wised data

include/os/osDir.h

@@ -31,6 +31,12 @@
 extern "C" {
 #endif
 
+#ifdef WINDOWS
+#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\"
+#else
+#define TD_TMP_DIR_PATH "/tmp/"
+#endif
+
 typedef struct TdDir *TdDirPtr;
 typedef struct TdDirEntry *TdDirEntryPtr;
 

include/util/taoserror.h

@@ -419,6 +419,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN        TAOS_DEF_ERROR_CODE(0, 0x0A09)
 #define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN        TAOS_DEF_ERROR_CODE(0, 0x0A0A)
 #define TSDB_CODE_TQ_GROUP_NOT_SET              TAOS_DEF_ERROR_CODE(0, 0x0A0B)
+#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND     TAOS_DEF_ERROR_CODE(0, 0x0A0B)
 
 // wal
 #define TSDB_CODE_WAL_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x1000)

source/client/src/clientHb.c

@@ -310,6 +310,8 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
         taosArrayDestroy(desc.subDesc);
         desc.subDesc = NULL;
       }
+    } else {
+      desc.subDesc = NULL;
     }
 
     releaseRequest(*rid);

source/common/src/tdataformat.c

@@ -855,7 +855,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
   pCols->maxCols = maxCols;
   pCols->numOfRows = 0;
   pCols->numOfCols = 0;
-  // pCols->bitmapMode = 0; // calloc already set 0
+  pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
 
   if (maxCols > 0) {
     pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol));
@@ -899,7 +899,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
 #endif
 
   pCols->numOfRows = 0;
-  pCols->bitmapMode = 0;
+  pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
   pCols->numOfCols = schemaNCols(pSchema);
 
   for (i = 0; i < schemaNCols(pSchema); ++i) {

source/common/src/trow.c

@@ -341,18 +341,19 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8
 bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
   int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
   uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
+  uint8_t *qBitmap = (uint8_t*)pBitmap;
   for (int i = 0; i < nBytes; ++i) {
-    if (*((uint8_t *)pBitmap) != vTypeByte) {
+    if (*qBitmap != vTypeByte) {
       return false;
     }
-    pBitmap = POINTER_SHIFT(pBitmap, i);
+    qBitmap = (uint8_t *)POINTER_SHIFT(pBitmap, i);
   }
 
   int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
 
   for (int j = 0; j < nLeft; ++j) {
     uint8_t vType;
-    tdGetBitmapValType(pBitmap, j, &vType, bitmapMode);
+    tdGetBitmapValType(qBitmap, j, &vType, bitmapMode);
     if (vType != TD_VTYPE_NORM) {
       return false;
     }

source/dnode/mgmt/test/bnode/dbnode.cpp

@@ -14,7 +14,7 @@
 class DndTestBnode : public ::testing::Test {
  protected:
   static void SetUpTestSuite() {
-    test.Init("/tmp/dbnodeTest", 9112);
+    test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112);
     taosMsleep(1100);
   }
   static void TearDownTestSuite() { test.Cleanup(); }

source/dnode/mgmt/test/mnode/dmnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestMnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dmnodeTest", 9114); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dmnodeTest", 9114); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mgmt/test/qnode/dqnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestQnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dqnodeTest", 9111); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dqnodeTest", 9111); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mgmt/test/snode/dsnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestSnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dsnodeTest", 9113); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dsnodeTest", 9113); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -48,7 +48,7 @@ void Testbase::Init(const char* path, int16_t port) {
   strcpy(tsDataDir, path);
   taosRemoveDir(path);
   taosMkDir(path);
-  InitLog("/tmp/td");
+  InitLog(TD_TMP_DIR_PATH "td");
 
   server.Start();
   client.Init("root", "taosdata");

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestVnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dvnodeTest", 9115); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dvnodeTest", 9115); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/acct/acct.cpp

@@ -13,7 +13,7 @@
 
 class MndTestAcct : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/acctTest", 9012); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "acctTest", 9012); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/bnode/mbnode.cpp

@@ -18,11 +18,11 @@ class MndTestBnode : public ::testing::Test {
 
  public:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_bnode1", 9018);
+    test.Init(TD_TMP_DIR_PATH "mnode_test_bnode1", 9018);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9018";
 
-    server2.Start("/tmp/mnode_test_bnode2", 9019);
+    server2.Start(TD_TMP_DIR_PATH "mnode_test_bnode2", 9019);
     taosMsleep(300);
   }
 

source/dnode/mnode/impl/test/db/db.cpp

@@ -13,7 +13,7 @@
 
 class MndTestDb : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_db", 9030); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/dnode/mdnode.cpp

@@ -18,14 +18,14 @@ class MndTestDnode : public ::testing::Test {
 
  public:
   static void SetUpTestSuite() {
-    test.Init("/tmp/dnode_test_dnode1", 9023);
+    test.Init(TD_TMP_DIR_PATH "dnode_test_dnode1", 9023);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9023";
 
-    // server2.Start("/tmp/dnode_test_dnode2", fqdn, 9024, firstEp);
+    // server2.Start(TD_TMP_DIR_PATH "dnode_test_dnode2", fqdn, 9024, firstEp);
-    // server3.Start("/tmp/dnode_test_dnode3", fqdn, 9025, firstEp);
+    // server3.Start(TD_TMP_DIR_PATH "dnode_test_dnode3", fqdn, 9025, firstEp);
-    // server4.Start("/tmp/dnode_test_dnode4", fqdn, 9026, firstEp);
+    // server4.Start(TD_TMP_DIR_PATH "dnode_test_dnode4", fqdn, 9026, firstEp);
-    // server5.Start("/tmp/dnode_test_dnode5", fqdn, 9027, firstEp);
+    // server5.Start(TD_TMP_DIR_PATH "dnode_test_dnode5", fqdn, 9027, firstEp);
     taosMsleep(300);
   }
 

source/dnode/mnode/impl/test/func/func.cpp

@@ -13,7 +13,7 @@
 
 class MndTestFunc : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_func", 9038); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/mnode/mnode.cpp

@@ -18,11 +18,11 @@ class MndTestMnode : public ::testing::Test {
 
  public:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_mnode1", 9028);
+    test.Init(TD_TMP_DIR_PATH "mnode_test_mnode1", 9028);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9028";
 
-    // server2.Start("/tmp/mnode_test_mnode2", fqdn, 9029, firstEp);
+    // server2.Start(TD_TMP_DIR_PATH "mnode_test_mnode2", fqdn, 9029, firstEp);
     taosMsleep(300);
   }
 

source/dnode/mnode/impl/test/profile/profile.cpp

@@ -13,7 +13,7 @@
 
 class MndTestProfile : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9031); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_profile", 9031); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/qnode/qnode.cpp

@@ -18,11 +18,11 @@ class MndTestQnode : public ::testing::Test {
 
  public:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_qnode1", 9014);
+    test.Init(TD_TMP_DIR_PATH "mnode_test_qnode1", 9014);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9014";
 
-    // server2.Start("/tmp/mnode_test_qnode2", fqdn, 9015, firstEp);
+    // server2.Start(TD_TMP_DIR_PATH "mnode_test_qnode2", fqdn, 9015, firstEp);
     taosMsleep(300);
   }
 

source/dnode/mnode/impl/test/sdb/sdbTest.cpp

@@ -31,7 +31,7 @@ class MndTestSdb : public ::testing::Test {
     tsLogEmbedded = 1;
     tsAsyncLog = 0;
 
-    const char *path = "/tmp/td";
+    const char *path = TD_TMP_DIR_PATH "td";
     taosRemoveDir(path);
     taosMkDir(path);
     tstrncpy(tsLogDir, path, PATH_MAX);
@@ -385,7 +385,7 @@ TEST_F(MndTestSdb, 01_Write_Str) {
   mnode.v100 = 100;
   mnode.v200 = 200;
   opt.pMnode = &mnode;
-  opt.path = "/tmp/mnode_test_sdb";
+  opt.path = TD_TMP_DIR_PATH "mnode_test_sdb";
   taosRemoveDir(opt.path);
 
   SSdbTable strTable1;
@@ -730,7 +730,7 @@ TEST_F(MndTestSdb, 01_Read_Str) {
   mnode.v100 = 100;
   mnode.v200 = 200;
   opt.pMnode = &mnode;
-  opt.path = "/tmp/mnode_test_sdb";
+  opt.path = TD_TMP_DIR_PATH "mnode_test_sdb";
 
   SSdbTable strTable1;
   memset(&strTable1, 0, sizeof(SSdbTable));

source/dnode/mnode/impl/test/show/show.cpp

@@ -13,7 +13,7 @@
 
 class MndTestShow : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_show", 9021); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -13,7 +13,7 @@
 
 class MndTestSma : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_sma", 9035); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/snode/snode.cpp

@@ -18,11 +18,11 @@ class MndTestSnode : public ::testing::Test {
 
  public:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_snode1", 9016);
+    test.Init(TD_TMP_DIR_PATH "mnode_test_snode1", 9016);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9016";
 
-    // server2.Start("/tmp/mnode_test_snode2", fqdn, 9017, firstEp);
+    // server2.Start(TD_TMP_DIR_PATH "mnode_test_snode2", fqdn, 9017, firstEp);
     taosMsleep(300);
   }
 

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -13,7 +13,7 @@
 
 class MndTestStb : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_stb", 9034); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_stb", 9034); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -13,7 +13,7 @@
 
 class MndTestTopic : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_topic", 9039); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_topic", 9039); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mnode/impl/test/trans/trans1.cpp

@@ -14,10 +14,10 @@
 class MndTestTrans1 : public ::testing::Test {
  protected:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_trans1", 9013);
+    test.Init(TD_TMP_DIR_PATH "mnode_test_trans1", 9013);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9013";
-    // server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp);
+    // server2.Start(TD_TMP_DIR_PATH "mnode_test_trans2", fqdn, 9020, firstEp);
   }
 
   static void TearDownTestSuite() {
@@ -26,7 +26,7 @@ class MndTestTrans1 : public ::testing::Test {
   }
 
   static void KillThenRestartServer() {
-    char      file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data";
+    char      file[PATH_MAX] = TD_TMP_DIR_PATH "mnode_test_trans1/mnode/data/sdb.data";
     TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
     int32_t   size = 3 * 1024 * 1024;
     void*     buffer = taosMemoryMalloc(size);

source/dnode/mnode/impl/test/trans/trans2.cpp

@@ -41,7 +41,7 @@ class MndTestTrans2 : public ::testing::Test {
     tsLogEmbedded = 1;
     tsAsyncLog = 0;
 
-    const char *logpath = "/tmp/td";
+    const char *logpath = TD_TMP_DIR_PATH "td";
     taosRemoveDir(logpath);
     taosMkDir(logpath);
     tstrncpy(tsLogDir, logpath, PATH_MAX);
@@ -68,7 +68,7 @@ class MndTestTrans2 : public ::testing::Test {
 
     tsTransPullupInterval = 1;
 
-    const char *mnodepath = "/tmp/mnode_test_trans";
+    const char *mnodepath = TD_TMP_DIR_PATH "mnode_test_trans";
     taosRemoveDir(mnodepath);
     pMnode = mndOpen(mnodepath, &opt);
     mndStart(pMnode);

source/dnode/mnode/impl/test/user/user.cpp

@@ -13,7 +13,7 @@
 
 class MndTestUser : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_user", 9011); }
+  static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_user", 9011); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/vnode/src/tq/tq.c

@@ -600,6 +600,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           SSDataBlock block = {0};
           if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
                                   &block.info.numOfCols) < 0) {
+            if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
             ASSERT(0);
           }
           int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);

source/dnode/vnode/src/tq/tqRead.c

@@ -91,12 +91,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) {
     pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
     if (pHandle->pSchema == NULL) {
-      tqError("cannot found schema for table: %ld, version %d", pHandle->msgIter.suid, pHandle->sver);
+      tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
+             pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->sver);
+      /*ASSERT(0);*/
+      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
       return -1;
     }
 
     // this interface use suid instead of uid
     pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true);
+    if (pHandle->pSchemaWrapper == NULL) {
+      tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
+             pHandle->msgIter.suid, pHandle->sver);
+      /*ASSERT(0);*/
+      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
+      return -1;
+    }
     pHandle->sver = sversion;
     pHandle->cachedSchemaUid = pHandle->msgIter.suid;
   }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1138,6 +1138,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
       memcpy(tptr, pDataCol->pData, flen);
       if (tBitmaps > 0) {
         bptr = POINTER_SHIFT(pBlockData, lsize + flen);
+        if (isSuper && !tdDataColsIsBitmapI(pDataCols)) {
+          tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap);
+        }
         memcpy(bptr, pDataCol->pBitmap, tBitmaps);
         tBitmapsLen = tBitmaps;
         flen += tBitmapsLen;
@@ -1503,13 +1506,16 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
       tSkipListIterNext(pCommitIter->pIter);
     } else {
       if (lastKey != key1) {
+        if (lastKey != TSKEY_INITIAL_VAL) {
+          ++pTarget->numOfRows;
+        }
         lastKey = key1;
-        ++pTarget->numOfRows;
       }
 
       // copy disk data
       for (int i = 0; i < pDataCols->numOfCols; ++i) {
         SCellVal sVal = {0};
+        // no duplicated TS keys in pDataCols from file
         if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
           TASSERT(0);
         }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -1965,7 +1965,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
   SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
   assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
          cur->pos >= 0 && cur->pos < pBlock->numOfRows);
-
+  // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface.
   TSKEY* tsArray = pCols->cols[0].pData;
   assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst &&
          tsArray[pBlock->numOfRows - 1] == pBlock->keyLast);
@@ -1995,6 +1995,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
   int32_t pos = cur->pos;
   cur->win = TSWINDOW_INITIALIZER;
+  bool adjustPos = false;
 
   // no data in buffer, load data from file directly
   if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
@@ -2016,6 +2017,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         break;
       }
 
+      if (adjustPos) {
+        if (key == lastKeyAppend) {
+          pos -= step;
+        }
+        adjustPos = false;
+      }
+
       if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) ||
           ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) {
         break;
@@ -2107,7 +2115,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           moveToNextRowInMem(pCheckInfo);
 
           pos += step;
+          adjustPos = true;
         } else {
+          // discard the memory record
           moveToNextRowInMem(pCheckInfo);
         }
       } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) {

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -20,11 +20,11 @@
 static void tsdbResetReadTable(SReadH *pReadh);
 static void tsdbResetReadFile(SReadH *pReadh);
 static int  tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
-static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
+static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode);
 static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
                                          int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize);
 static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
-                                      int numOfColIds);
+                                      int numOfColIds, int8_t bitmapMode);
 static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
 
 int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
@@ -266,10 +266,11 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
     }
   }
 
-  if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1;
+  
+  if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1;
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
-    if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1;
     // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
                         TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
@@ -309,10 +310,10 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
     }
   }
 
-  if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1;
+  if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1;
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
-    if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1;
     // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
                         TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
@@ -543,14 +544,14 @@ static void tsdbResetReadFile(SReadH *pReadh) {
   tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
 }
 
-static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) {
+static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode) {
   ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
 
   SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
 
   tdResetDataCols(pDataCols);
 
-  if (tsdbIsSupBlock(pBlock)) {
+  if (tdIsBitmapModeI(bitmapMode)) {
     tdDataColsSetBitmapI(pDataCols);
   }
 
@@ -730,7 +731,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
 }
 
 static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
-                                     int numOfColIds) {
+                                     int numOfColIds, int8_t bitmapMode) {
   ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
   ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);
 
@@ -739,7 +740,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 
   tdResetDataCols(pDataCols);
 
-  if (tsdbIsSupBlock(pBlock)) {
+  if (tdIsBitmapModeI(bitmapMode)) {
     tdDataColsSetBitmapI(pDataCols);
   }
 

source/dnode/vnode/test/tqMetaTest.cpp

@@ -41,7 +41,7 @@ class TqMetaUpdateAppendTest : public ::testing::Test {
   void TearDown() override { tqStoreClose(pMeta); }
 
   STqMetaStore* pMeta;
-  const char*   pathName = "/tmp/tq_test";
+  const char*   pathName = TD_TMP_DIR_PATH "tq_test";
 };
 
 TEST_F(TqMetaUpdateAppendTest, copyPutTest) {

source/libs/executor/src/executorimpl.c

@@ -4059,7 +4059,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
     defaultBufsz = defaultPgsz * 4;
   }
 
-  int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, "/tmp/");
+  int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, TD_TMP_DIR_PATH);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }

source/libs/executor/src/groupoperator.c

@@ -614,7 +614,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo*
     goto _error;
   }
 
-  int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/");
+  int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, TD_TMP_DIR_PATH);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/scanoperator.c

@@ -990,7 +990,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
 
   size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
   initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize,
-      "StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan
+      "StreamFinalInterval", TD_TMP_DIR_PATH); // TODO(liuyao) get row size from phy plan
 
   pOperator->name       = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;

source/libs/executor/src/tlinearhash.c

@@ -247,7 +247,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
     return NULL;
   }
 
-  int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, "/tmp");
+  int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, TD_TMP_DIR_PATH);
   if (code != 0) {
     terrno = code;
     return NULL;

source/libs/executor/src/tsort.c

@@ -155,7 +155,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
   int32_t start = 0;
 
   if (pHandle->pBuf == NULL) {
-    int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", "/tmp");
+    int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", TD_TMP_DIR_PATH);
     dBufSetPrintInfo(pHandle->pBuf);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -217,7 +217,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
   } else {
     // multi-pass internal merge sort is required
     if (pHandle->pBuf == NULL) {
-      code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", "/tmp");
+      code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", TD_TMP_DIR_PATH);
       dBufSetPrintInfo(pHandle->pBuf);
       if (code != TSDB_CODE_SUCCESS) {
         return code;

source/libs/executor/test/index_executor_tests.cpp

@@ -57,7 +57,7 @@ void sifInitLogFile() {
 
   tsAsyncLog = 0;
   qDebugFlag = 159;
-  strcpy(tsLogDir, "/tmp/sif");
+  strcpy(tsLogDir, TD_TMP_DIR_PATH "sif");
   taosRemoveDir(tsLogDir);
   taosMkDir(tsLogDir);
 

source/libs/function/src/tpercentile.c

@@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
 
   resetSlotInfo(pBucket);
 
-  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp");
+  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", TD_TMP_DIR_PATH);
   if (ret != 0) {
     tMemBucketDestroy(pBucket);
     return NULL;

source/libs/function/src/udfd.c

@@ -400,7 +400,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
     udf->bufSize = pFuncInfo->bufSize;
 
     char path[PATH_MAX] = {0};
-    snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name);
+    snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name);
     TdFilePtr file =
         taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
     int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);

source/libs/index/src/indexTfile.c

@@ -141,7 +141,6 @@ void tfileCacheDestroy(TFileCache* tcache) {
     TFileReader* p = *reader;
     indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
               p->header.colType);
-
     tfileReaderUnRef(p);
     reader = taosHashIterate(tcache->tableCache, reader);
   }

source/libs/index/test/fstTest.cc

@@ -15,7 +15,7 @@
 #include "tutil.h"
 void* callback(void* s) { return s; }
 
-static std::string fileName = "/tmp/tindex.tindex";
+static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex";
 class FstWriter {
  public:
   FstWriter() {
@@ -48,7 +48,7 @@ class FstWriter {
 
 class FstReadMemory {
  public:
-  FstReadMemory(int32_t size, const std::string& fileName = "/tmp/tindex.tindex") {
+  FstReadMemory(int32_t size, const std::string& fileName = TD_TMP_DIR_PATH "tindex.tindex") {
     _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024);
     _w = fstCountingWriterCreate(_wc);
     _size = size;

source/libs/index/test/fstUT.cc

@@ -17,7 +17,7 @@
 #include "tskiplist.h"
 #include "tutil.h"
 
-static std::string dir = "/tmp/index";
+static std::string dir = TD_TMP_DIR_PATH "index";
 
 static char indexlog[PATH_MAX] = {0};
 static char tindex[PATH_MAX] = {0};

source/libs/index/test/indexTests.cc

@@ -51,7 +51,7 @@ class DebugInfo {
 class FstWriter {
  public:
   FstWriter() {
-    _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024);
+    _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024);
     _b = fstBuilderCreate(NULL, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
@@ -75,7 +75,7 @@ class FstWriter {
 class FstReadMemory {
  public:
   FstReadMemory(size_t size) {
-    _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024);
+    _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024);
     _w = fstCountingWriterCreate(_wc);
     _size = size;
     memset((void*)&_s, 0, sizeof(_s));
@@ -285,7 +285,7 @@ class IndexEnv : public ::testing::Test {
     indexOptsDestroy(opts);
   }
 
-  const char* path = "/tmp/tindex";
+  const char* path = TD_TMP_DIR_PATH "tindex";
   SIndexOpts* opts;
   SIndex*     index;
 };
@@ -342,7 +342,7 @@ class IndexEnv : public ::testing::Test {
 
 class TFileObj {
  public:
-  TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage")
+  TFileObj(const std::string& path = TD_TMP_DIR_PATH "tindex", const std::string& colName = "voltage")
       : path_(path), colName_(colName) {
     colId_ = 10;
     reader_ = NULL;
@@ -437,7 +437,7 @@ class IndexTFileEnv : public ::testing::Test {
     // tfileWriterDestroy(twrite);
   }
   TFileObj*   fObj;
-  std::string dir = "/tmp/tindex";
+  std::string dir = TD_TMP_DIR_PATH "tindex";
   std::string colName = "voltage";
 
   int coldId = 2;
@@ -822,7 +822,7 @@ class IndexEnv2 : public ::testing::Test {
   IndexObj*    index;
 };
 TEST_F(IndexEnv2, testIndexOpen) {
-  std::string path = "/tmp/test";
+  std::string path = TD_TMP_DIR_PATH "test";
   if (index->Init(path) != 0) {
     std::cout << "failed to init index" << std::endl;
     exit(1);
@@ -891,7 +891,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
 }
 
 TEST_F(IndexEnv2, testIndex_TrigeFlush) {
-  std::string path = "/tmp/testxxx";
+  std::string path = TD_TMP_DIR_PATH "testxxx";
   if (index->Init(path) != 0) {
     // r
     std::cout << "failed to init" << std::endl;
@@ -914,7 +914,7 @@ static void multi_write_and_search(IndexObj* idx) {
   idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10);
 }
 TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
     // opt
   }
@@ -934,7 +934,7 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
   }
 }
 TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
 
@@ -950,14 +950,14 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
 }
 
 // TEST_F(IndexEnv2, testIndex_restart) {
-//  std::string path = "/tmp/cache_and_tfile";
+//  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
 //  if (index->Init(path) != 0) {
 //  }
 //  index->SearchOneTarget("tag1", "Hello", 10);
 //  index->SearchOneTarget("tag2", "Test", 10);
 //}
 // TEST_F(IndexEnv2, testIndex_restart1) {
-//  std::string path = "/tmp/cache_and_tfile";
+//  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
 //  if (index->Init(path) != 0) {
 //  }
 //  index->ReadMultiMillonData("tag1", "coding");
@@ -966,7 +966,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
 //}
 
 // TEST_F(IndexEnv2, testIndex_read_performance) {
-//  std::string path = "/tmp/cache_and_tfile";
+//  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
 //  if (index->Init(path) != 0) {
 //  }
 //  index->PutOneTarge("tag1", "Hello", 12);
@@ -976,7 +976,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
 //  assert(3 == index->SearchOne("tag1", "Hello"));
 //}
 // TEST_F(IndexEnv2, testIndexMultiTag) {
-//  std::string path = "/tmp/multi_tag";
+//  std::string path = TD_TMP_DIR_PATH "multi_tag";
 //  if (index->Init(path) != 0) {
 //  }
 //  int64_t st = taosGetTimestampUs();
@@ -986,7 +986,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) {
 //  // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000);
 //}
 TEST_F(IndexEnv2, testLongComVal1) {
-  std::string path = "/tmp/long_colVal";
+  std::string path = TD_TMP_DIR_PATH "long_colVal";
   if (index->Init(path) != 0) {
   }
   // gen colVal by randstr
@@ -995,7 +995,7 @@ TEST_F(IndexEnv2, testLongComVal1) {
 }
 
 TEST_F(IndexEnv2, testLongComVal2) {
-  std::string path = "/tmp/long_colVal";
+  std::string path = TD_TMP_DIR_PATH "long_colVal";
   if (index->Init(path) != 0) {
   }
   // gen colVal by randstr
@@ -1003,7 +1003,7 @@ TEST_F(IndexEnv2, testLongComVal2) {
   index->WriteMultiMillonData("tag1", randstr, 100 * 1000);
 }
 TEST_F(IndexEnv2, testLongComVal3) {
-  std::string path = "/tmp/long_colVal";
+  std::string path = TD_TMP_DIR_PATH "long_colVal";
   if (index->Init(path) != 0) {
   }
   // gen colVal by randstr
@@ -1011,7 +1011,7 @@ TEST_F(IndexEnv2, testLongComVal3) {
   index->WriteMultiMillonData("tag1", randstr, 100 * 1000);
 }
 TEST_F(IndexEnv2, testLongComVal4) {
-  std::string path = "/tmp/long_colVal";
+  std::string path = TD_TMP_DIR_PATH "long_colVal";
   if (index->Init(path) != 0) {
   }
   // gen colVal by randstr
@@ -1019,7 +1019,7 @@ TEST_F(IndexEnv2, testLongComVal4) {
   index->WriteMultiMillonData("tag1", randstr, 100 * 100);
 }
 TEST_F(IndexEnv2, testIndex_read_performance1) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   index->PutOneTarge("tag1", "Hello", 12);
@@ -1029,7 +1029,7 @@ TEST_F(IndexEnv2, testIndex_read_performance1) {
   EXPECT_EQ(2, index->SearchOne("tag1", "Hello"));
 }
 TEST_F(IndexEnv2, testIndex_read_performance2) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   index->PutOneTarge("tag1", "Hello", 12);
@@ -1039,7 +1039,7 @@ TEST_F(IndexEnv2, testIndex_read_performance2) {
   EXPECT_EQ(2, index->SearchOne("tag1", "Hello"));
 }
 TEST_F(IndexEnv2, testIndex_read_performance3) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   index->PutOneTarge("tag1", "Hello", 12);
@@ -1049,7 +1049,7 @@ TEST_F(IndexEnv2, testIndex_read_performance3) {
   EXPECT_EQ(2, index->SearchOne("tag1", "Hello"));
 }
 TEST_F(IndexEnv2, testIndex_read_performance4) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   index->PutOneTarge("tag10", "Hello", 12);
@@ -1059,7 +1059,7 @@ TEST_F(IndexEnv2, testIndex_read_performance4) {
   EXPECT_EQ(1, index->SearchOne("tag10", "Hello"));
 }
 TEST_F(IndexEnv2, testIndex_cache_del) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   for (int i = 0; i < 100; i++) {
@@ -1098,7 +1098,7 @@ TEST_F(IndexEnv2, testIndex_cache_del) {
 }
 
 TEST_F(IndexEnv2, testIndex_del) {
-  std::string path = "/tmp/cache_and_tfile";
+  std::string path = TD_TMP_DIR_PATH "cache_and_tfile";
   if (index->Init(path) != 0) {
   }
   for (int i = 0; i < 100; i++) {

source/libs/index/test/jsonUT.cc

@@ -16,8 +16,8 @@
 #include "tskiplist.h"
 #include "tutil.h"
 
-static std::string dir = "/tmp/json";
+static std::string dir = TD_TMP_DIR_PATH "json";
-static std::string logDir = "/tmp/log";
+static std::string logDir = TD_TMP_DIR_PATH "log";
 
 static void initLog() {
   const char*   defaultLogFileNamePrefix = "taoslog";

source/libs/planner/test/planTestMain.cpp

@@ -25,7 +25,7 @@ class PlannerEnv : public testing::Environment {
   virtual void SetUp() {
     initMetaDataEnv();
     generateMetaData();
-    initLog("/tmp/td");
+    initLog(TD_TMP_DIR_PATH "td");
   }
 
   virtual void TearDown() { destroyMetaDataEnv(); }

source/libs/tfs/test/tfsTest.cpp

@@ -16,7 +16,7 @@
 
 class TfsTest : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { root = "/tmp/tfsTest"; }
+  static void SetUpTestSuite() { root = TD_TMP_DIR_PATH "tfsTest"; }
   static void TearDownTestSuite() {}
 
  public:
@@ -299,15 +299,15 @@ TEST_F(TfsTest, 04_File) {
 TEST_F(TfsTest, 05_MultiDisk) {
   int32_t code = 0;
 
-  const char *root00 = "/tmp/tfsTest00";
+  const char *root00 = TD_TMP_DIR_PATH "tfsTest00";
-  const char *root01 = "/tmp/tfsTest01";
+  const char *root01 = TD_TMP_DIR_PATH "tfsTest01";
-  const char *root10 = "/tmp/tfsTest10";
+  const char *root10 = TD_TMP_DIR_PATH "tfsTest10";
-  const char *root11 = "/tmp/tfsTest11";
+  const char *root11 = TD_TMP_DIR_PATH "tfsTest11";
-  const char *root12 = "/tmp/tfsTest12";
+  const char *root12 = TD_TMP_DIR_PATH "tfsTest12";
-  const char *root20 = "/tmp/tfsTest20";
+  const char *root20 = TD_TMP_DIR_PATH "tfsTest20";
-  const char *root21 = "/tmp/tfsTest21";
+  const char *root21 = TD_TMP_DIR_PATH "tfsTest21";
-  const char *root22 = "/tmp/tfsTest22";
+  const char *root22 = TD_TMP_DIR_PATH "tfsTest22";
-  const char *root23 = "/tmp/tfsTest23";
+  const char *root23 = TD_TMP_DIR_PATH "tfsTest23";
 
   SDiskCfg dCfg[9] = {0};
   tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN);

source/libs/transport/test/rclient.c

@@ -161,7 +161,7 @@ int main(int argc, char *argv[]) {
     }
   }
 
-  const char *path = "/tmp/transport/client";
+  const char *path = TD_TMP_DIR_PATH "transport/client";
   taosRemoveDir(path);
   taosMkDir(path);
   tstrncpy(tsLogDir, path, PATH_MAX);

source/libs/transport/test/rserver.c

@@ -160,7 +160,7 @@ int main(int argc, char *argv[]) {
   tsAsyncLog = 0;
   rpcInit.connType = TAOS_CONN_SERVER;
 
-  const char *path = "/tmp/transport/server";
+  const char *path = TD_TMP_DIR_PATH "transport/server";
   taosRemoveDir(path);
   taosMkDir(path);
   tstrncpy(tsLogDir, path, PATH_MAX);

source/libs/transport/test/transUT.cpp

@@ -43,7 +43,7 @@ static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
 class Client {
  public:
   void Init(int nThread) {
-    memcpy(tsTempDir, "/tmp", strlen("/tmp"));
+    memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH));
     memset(&rpcInit_, 0, sizeof(rpcInit_));
     rpcInit_.localPort = 0;
     rpcInit_.label = (char *)label;
@@ -105,7 +105,7 @@ class Client {
 class Server {
  public:
   Server() {
-    memcpy(tsTempDir, "/tmp", strlen("/tmp"));
+    memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH));
     memset(&rpcInit_, 0, sizeof(rpcInit_));
 
     memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost"));
@@ -219,7 +219,7 @@ static void initEnv() {
   tsLogEmbedded = 1;
   tsAsyncLog = 0;
 
-  std::string path = "/tmp/transport";
+  std::string path = TD_TMP_DIR_PATH "transport";
   // taosRemoveDir(path.c_str());
   taosMkDir(path.c_str());
 

source/libs/wal/src/walWrite.c

@@ -121,6 +121,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   pWal->vers.lastVer = ver - 1;
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
+  taosCloseFile(&pIdxTFile);
+  taosCloseFile(&pLogTFile);
 
   // unlock
   taosThreadMutexUnlock(&pWal->mutex);

source/libs/wal/test/walMetaTest.cpp

@@ -37,7 +37,7 @@ class WalCleanEnv : public ::testing::Test {
   }
 
   SWal*       pWal = NULL;
-  const char* pathName = "/tmp/wal_test";
+  const char* pathName = TD_TMP_DIR_PATH "wal_test";
 };
 
 class WalCleanDeleteEnv : public ::testing::Test {
@@ -67,7 +67,7 @@ class WalCleanDeleteEnv : public ::testing::Test {
   }
 
   SWal*       pWal = NULL;
-  const char* pathName = "/tmp/wal_test";
+  const char* pathName = TD_TMP_DIR_PATH "wal_test";
 };
 
 class WalKeepEnv : public ::testing::Test {
@@ -104,7 +104,7 @@ class WalKeepEnv : public ::testing::Test {
   }
 
   SWal*       pWal = NULL;
-  const char* pathName = "/tmp/wal_test";
+  const char* pathName = TD_TMP_DIR_PATH "wal_test";
 };
 
 class WalRetentionEnv : public ::testing::Test {
@@ -141,7 +141,7 @@ class WalRetentionEnv : public ::testing::Test {
   }
 
   SWal*       pWal = NULL;
-  const char* pathName = "/tmp/wal_test";
+  const char* pathName = TD_TMP_DIR_PATH "wal_test";
 };
 
 TEST_F(WalCleanEnv, createNew) {
@@ -325,6 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
       EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
     }
   }
+  walCloseReadHandle(pRead);
 }
 
 TEST_F(WalRetentionEnv, repairMeta1) {

source/util/test/cfgTest.cpp

@@ -59,7 +59,7 @@ TEST_F(CfgTest, 02_Basic) {
   EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0);
   EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0);
   EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0);
-  EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0);
+  EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0), 0);
 
   EXPECT_EQ(cfgGetSize(pConfig), 6);
 
@@ -126,7 +126,7 @@ TEST_F(CfgTest, 02_Basic) {
   EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
   EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR);
   EXPECT_STREQ(pItem->name, "test_dir");
-  EXPECT_STREQ(pItem->str, "/tmp");
+  EXPECT_STREQ(pItem->str, TD_TMP_DIR_PATH);
 
   cfgCleanup(pConfig);
 }

source/util/test/pageBufferTest.cpp

@@ -13,7 +13,7 @@ namespace {
 // simple test
 void simpleTest() {
   SDiskbasedBuf* pBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/");
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", TD_TMP_DIR_PATH);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -57,7 +57,7 @@ void simpleTest() {
 
 void writeDownTest() {
   SDiskbasedBuf* pBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH);
 
   int32_t pageId = 0;
   int32_t writePageId = 0;
@@ -106,7 +106,7 @@ void writeDownTest() {
 
 void recyclePageTest() {
   SDiskbasedBuf* pBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH);
 
   int32_t pageId = 0;
   int32_t writePageId = 0;

source/util/test/procTest.cpp

@@ -38,9 +38,9 @@ class UtilTesProc : public ::testing::Test {
     head.noResp = 3;
     head.persistHandle = 4;
 
-    taosRemoveDir("/tmp/td");
+    taosRemoveDir(TD_TMP_DIR_PATH "td");
-    taosMkDir("/tmp/td");
+    taosMkDir(TD_TMP_DIR_PATH "td");
-    tstrncpy(tsLogDir, "/tmp/td", PATH_MAX);
+    tstrncpy(tsLogDir, TD_TMP_DIR_PATH "td", PATH_MAX);
     if (taosInitLog("taosdlog", 1) != 0) {
       printf("failed to init log file\n");
     }

tests/script/tsim/insert/commit-merge0.sim

@@ -0,0 +1,262 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database
+sql create database db  days 300 keep 365000d,365000d,365000d
+sql show databases
+if $rows != 3 then
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use db
+sql create table stb1(ts timestamp, c6 double) tags (t1 int);
+sql create table ct1 using stb1 tags ( 1 );
+sql create table ct2 using stb1 tags ( 2 );
+sql create table ct3 using stb1 tags ( 3 );
+sql create table ct4 using stb1 tags ( 4 );
+sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0);
+sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0);
+sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11);
+sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11);
+sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22);
+sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22);
+sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33);
+sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33);
+sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44);
+sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44);
+sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55);
+sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55);
+sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66);
+sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66);
+sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77);
+sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77);
+sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88);
+sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88);
+sql insert into ct1 values ('2022-05-01 18:30:20.019', 0);
+sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99);
+sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL);
+sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99);
+sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ;
+sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ;
+sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ;
+
+print =============== select * from ct1 - memory
+sql select * from stb1;
+if $rows != 25 then
+  print rows = $rows != 25
+  return -1
+endi
+
+
+print =============== stop and restart taosd
+
+$reboot_max = 10;
+
+$reboot_cnt = 0
+
+reboot_and_check:
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+print =============== insert duplicated records to memory - loop $reboot_max - $reboot_cnt 
+sql use db
+sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0);
+sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0);
+sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11);
+sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11);
+sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22);
+sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22);
+sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33);
+sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33);
+sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44);
+sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44);
+sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55);
+sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55);
+sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66);
+sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66);
+sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77);
+sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77);
+sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88);
+sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88);
+sql insert into ct1 values ('2022-05-01 18:30:20.019', 0);
+sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99);
+sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL);
+sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99);
+sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ;
+sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ;
+sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ;
+
+print =============== select * from ct1 - merge memory and file - loop $reboot_max - $reboot_cnt 
+sql select * from ct1;
+if $rows != 13 then
+  print rows = $rows != 13
+  return -1
+endi
+print $data[0][0] $data[0][1]
+print $data[1][0] $data[1][1]
+print $data[2][0] $data[2][1]
+print $data[3][0] $data[3][1]
+print $data[4][0] $data[4][1]
+print $data[5][0] $data[5][1]
+print $data[6][0] $data[6][1]
+print $data[7][0] $data[7][1]
+print $data[8][0] $data[8][1]
+print $data[9][0] $data[9][1]
+print $data[10][0] $data[10][1]
+print $data[11][0] $data[11][1]
+print $data[12][0] $data[12][1]
+
+if $data[0][1] != 44.440000000 then
+    print $data[0][1] != 44.440000000
+    return -1
+endi
+if $data[1][1] != 55.550000000 then
+    print $data[1][1] != 55.550000000
+    return -1
+endi
+if $data[2][1] != 66.660000000 then
+    print $data[2][1] != 66.660000000
+    return -1
+endi
+if $data[3][1] != 77.770000000 then
+    print $data[3][1] != 77.770000000
+    return -1
+endi
+if $data[4][1] != 88.880000000 then
+    print $data[4][1] != 88.880000000
+    return -1
+endi
+if $data[5][1] != 33.330000000 then
+    print $data[5][1] != 33.330000000
+    return -1
+endi
+if $data[6][1] != 22.220000000 then
+    print $data[6][1] != 22.220000000
+    return -1
+endi
+if $data[7][1] != 11.110000000 then
+    print $data[7][1] != 11.110000000
+    return -1
+endi
+if $data[8][1] != 0.000000000 then
+    print $data[8][1] != 0.000000000
+    return -1
+endi
+if $data[9][1] != 0.000000000 then
+    print $data[9][1] != 0.000000000
+    return -1
+endi
+if $data[10][1] != -99.990000000 then
+    print $data[10][1] != -99.990000000
+    return -1
+endi
+if $data[11][1] != NULL then
+    print $data[11][1] != NULL
+    return -1
+endi
+if $data[12][1] != -99.990000000 then
+    print $data[12][1] != -99.990000000
+    return -1
+endi
+ 
+print =============== select * from ct4 - merge memory and file - loop $reboot_max - $reboot_cnt 
+sql select * from ct4;
+if $rows != 12 then
+  print rows = $rows != 12
+  return -1
+endi
+
+print $data[0][0] $data[0][1]
+print $data[1][0] $data[1][1]
+print $data[2][0] $data[2][1]
+print $data[3][0] $data[3][1]
+print $data[4][0] $data[4][1]
+print $data[5][0] $data[5][1]
+print $data[6][0] $data[6][1]
+print $data[7][0] $data[7][1]
+print $data[8][0] $data[8][1]
+print $data[9][0] $data[9][1]
+print $data[10][0] $data[10][1]
+print $data[11][0] $data[11][1]
+
+if $data[0][1] != NULL then
+    print $data[0][1] != NULL
+    return -1
+endi
+if $data[1][1] != 88.880000000 then
+    print $data[1][1] != 88.880000000
+    return -1
+endi
+if $data[2][1] != 77.770000000 then
+    print $data[2][1] != 77.770000000
+    return -1
+endi
+if $data[3][1] != 66.660000000 then
+    print $data[3][1] != 66.660000000
+    return -1
+endi
+if $data[4][1] != 55.550000000 then
+    print $data[4][1] != 55.550000000
+    return -1
+endi
+if $data[5][1] != NULL then
+    print $data[5][1] != NULL
+    return -1
+endi
+if $data[6][1] != 44.440000000 then
+    print $data[6][1] != 44.440000000
+    return -1
+endi
+if $data[7][1] != 22.220000000 then
+    print $data[7][1] != 22.220000000
+    return -1
+endi
+if $data[8][1] != 11.110000000 then
+    print $data[8][1] != 11.110000000
+    return -1
+endi
+if $data[9][1] != 0.000000000 then
+    print $data[9][1] != 0.000000000
+    return -1
+endi
+if $data[10][1] != 33.330000000 then
+    print $data[10][1] != 33.330000000
+    return -1
+endi
+if $data[11][1] != NULL then
+    print $data[11][1] != NULL
+    return -1
+endi
+
+
+if $reboot_cnt > $reboot_max then
+    print reboot_cnt $reboot_cnt > reboot_max $reboot_max
+    return 0
+else
+    print reboot_cnt $reboot_cnt <= reboot_max $reboot_max
+    $reboot_cnt = $reboot_cnt + 1
+    goto reboot_and_check
+endi

tests/script/tsim/insert/update0.sim

@@ -99,6 +99,23 @@ endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
 
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
 print =============== step3-2 query records of ct1 from file
 sql select * from ct1;
 print $data00 $data01