From 66e62bb010281357f6c58dc2431697fd53fa6943 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 6 Jan 2023 19:19:31 +0800
Subject: [PATCH 01/16] fix: tsdb read invalid memory read issue

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 55703002b8..f30308845b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -775,7 +775,16 @@ _exit:
   return code;
 }
 
-void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); }
+void tRowMergerClear(SRowMerger *pMerger) { 
+  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
+    }
+  }
+
+  taosArrayDestroy(pMerger->pArray); 
+}
 
 int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
   int32_t  code = 0;
@@ -789,7 +798,17 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
 
     if (key.version > pMerger->version) {
       if (!COL_VAL_IS_NONE(pColVal)) {
-        taosArraySet(pMerger->pArray, iCol, pColVal);
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+
+          code = tRealloc(pTColVal->value.pData, pColVal->value.nData);
+          if (code) goto _exit;
+
+          pTColVal->value.nData = pColVal->value.nData;
+          memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal.value.nData);
+        } else {
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
       }
     } else if (key.version < pMerger->version) {
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);

From d70e32e7d36b14a5f7497d33ef310a68580bdef9 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Sat, 7 Jan 2023 14:56:43 +0800
Subject: [PATCH 02/16] fix: compile issue

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index f30308845b..112fbb61c6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -801,11 +801,11 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
         if (IS_VAR_DATA_TYPE(pColVal->type)) {
           SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
 
-          code = tRealloc(pTColVal->value.pData, pColVal->value.nData);
+          code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
           if (code) goto _exit;
 
           pTColVal->value.nData = pColVal->value.nData;
-          memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal.value.nData);
+          memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }

From 80586ad997994924f77810c79850d39d14171d41 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 9 Jan 2023 11:33:26 +0800
Subject: [PATCH 03/16] fix: invalid free issue

---
 source/dnode/vnode/src/inc/tsdb.h      |  1 +
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 ++++++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 5a2e462c8c..77a3bb7a2f 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -573,6 +573,7 @@ struct STSDBRowIter {
 struct SRowMerger {
   STSchema *pTSchema;
   int64_t   version;
+  bool      merged;
   SArray   *pArray;  // SArray<SColVal>
 };
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 112fbb61c6..a9c31c19cb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -776,10 +776,12 @@ _exit:
 }
 
 void tRowMergerClear(SRowMerger *pMerger) { 
-  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
-    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
-    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
-      tFree(pTColVal->value.pData);
+  if (pMerger->merged) {
+    for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
+      SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+      if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+        tFree(pTColVal->value.pData);
+      }
     }
   }
 
@@ -801,6 +803,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
         if (IS_VAR_DATA_TYPE(pColVal->type)) {
           SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
 
+          pTColVal->value.pData = NULL;
           code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
           if (code) goto _exit;
 
@@ -821,6 +824,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
   }
 
   pMerger->version = key.version;
+  pMerger->merged = true;
 
 _exit:
   return code;

From 907cb73243807fd6846658bf43bde0daea3b29e4 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 9 Jan 2023 16:31:16 +0800
Subject: [PATCH 04/16] fix: return dropping dnode in status resp

---
 source/dnode/mnode/impl/inc/mndDnode.h | 1 -
 source/dnode/mnode/impl/src/mndDnode.c | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h
index ebbabdfa33..cf1e7422be 100644
--- a/source/dnode/mnode/impl/inc/mndDnode.h
+++ b/source/dnode/mnode/impl/inc/mndDnode.h
@@ -29,7 +29,6 @@ void       mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode);
 SEpSet     mndGetDnodeEpset(SDnodeObj *pDnode);
 int32_t    mndGetDnodeSize(SMnode *pMnode);
 bool       mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs);
-void       mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index c7a416d444..ddb54a95ea 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -308,7 +308,8 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) {
   void   *pIter = NULL;
   while (1) {
     SDnodeObj *pDnode = NULL;
-    pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
+    ESdbStatus objStatus = 0;
+    pIter = sdbFetchAll(pSdb, SDB_DNODE, pIter, (void **)&pDnode, &objStatus, true);
     if (pIter == NULL) break;
 
     SDnodeEp dnodeEp = {0};

From 6ba5b6a287879076599eac8acaa2bfb9bf159d2c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 9 Jan 2023 17:28:27 +0800
Subject: [PATCH 05/16] fix: compile error

---
 source/dnode/mnode/impl/inc/mndDnode.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h
index cf1e7422be..ebbabdfa33 100644
--- a/source/dnode/mnode/impl/inc/mndDnode.h
+++ b/source/dnode/mnode/impl/inc/mndDnode.h
@@ -29,6 +29,7 @@ void       mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode);
 SEpSet     mndGetDnodeEpset(SDnodeObj *pDnode);
 int32_t    mndGetDnodeSize(SMnode *pMnode);
 bool       mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs);
+void       mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps);
 
 #ifdef __cplusplus
 }

From 4bfc2fab45dd20b19019483ca11259da5c887b2d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 9 Jan 2023 19:19:12 +0800
Subject: [PATCH 06/16] fix: add test cases;

---
 tests/script/tsim/parser/regressiontest.sim | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/script/tsim/parser/regressiontest.sim b/tests/script/tsim/parser/regressiontest.sim
index 98cb0248a1..1b127155cb 100644
--- a/tests/script/tsim/parser/regressiontest.sim
+++ b/tests/script/tsim/parser/regressiontest.sim
@@ -58,4 +58,9 @@ if $data40 != @18-09-17 09:06:49.600@ then
  return -1
 endi
 
+sql select * from $tb order by ts desc;
+if $rows != 8198 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 16f0900fe994a417ec106d594b1b7e5d77474115 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 9 Jan 2023 19:26:39 +0800
Subject: [PATCH 07/16] fix(query): fix error for retrieve data only in last
 files.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 64 ++++++++++++++++++++++++--
 1 file changed, 61 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index dcfc78fd1a..91690af4c8 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2834,7 +2834,37 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
 
   if (pBlockInfo == NULL) {  // build data block from last data file
-    code = buildComposedDataBlock(pReader);
+    SBlockData* pBData = &pReader->status.fileBlockData;
+    tBlockDataReset(pBData);
+
+    SSDataBlock* pResBlock = pReader->pResBlock;
+    tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr);
+
+    int64_t st = taosGetTimestampUs();
+
+    while (1) {
+      bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+
+      // no data in last block and block, no need to proceed.
+      if (hasBlockLData == false) {
+        break;
+      }
+
+      buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+      if (pResBlock->info.rows >= pReader->capacity) {
+        break;
+      }
+    }
+
+    double el = (taosGetTimestampUs() - st) / 1000.0;
+    updateComposedBlockInfo(pReader, el, pScanInfo);
+
+    if (pResBlock->info.rows > 0) {
+      tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
+                    " rows:%d, elapsed time:%.2f ms %s",
+                pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+                pResBlock->info.rows, el, pReader->idStr);
+    }
   } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) {
     code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);
     if (code != TSDB_CODE_SUCCESS) {
@@ -2853,10 +2883,38 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       // only return the rows in last block
       int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
       ASSERT(tsLast >= pBlock->maxKey.ts);
-      tBlockDataReset(&pReader->status.fileBlockData);
 
+      SBlockData* pBData = &pReader->status.fileBlockData;
+      tBlockDataReset(pBData);
+
+      SSDataBlock* pResBlock = pReader->pResBlock;
       tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr);
-      code = buildComposedDataBlock(pReader);
+
+      int64_t st = taosGetTimestampUs();
+
+      while (1) {
+        bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+
+        // no data in last block and block, no need to proceed.
+        if (hasBlockLData == false) {
+          break;
+        }
+
+        buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+        if (pResBlock->info.rows >= pReader->capacity) {
+          break;
+        }
+      }
+
+      double el = (taosGetTimestampUs() - st) / 1000.0;
+      updateComposedBlockInfo(pReader, el, pScanInfo);
+
+      if (pResBlock->info.rows > 0) {
+        tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
+                      " rows:%d, elapsed time:%.2f ms %s",
+                  pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+                  pResBlock->info.rows, el, pReader->idStr);
+      }
     } else {  // whole block is required, return it directly
       SDataBlockInfo* pInfo = &pReader->pResBlock->info;
       pInfo->rows = pBlock->nRow;

From df2175087b9c04c073d7b258b6fec037002a1676 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 9 Jan 2023 19:47:17 +0800
Subject: [PATCH 08/16] fix: memory leak and invalid read issue

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 83 +++++++++++++++++++++-----
 1 file changed, 67 insertions(+), 16 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index a9c31c19cb..9bc903a0ba 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -682,6 +682,16 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
     }
 
     tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
+    if ((!COL_VAL_IS_NONE(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+      uint8_t *pVal = pColVal->value.pData;
+    
+      pColVal->value.pData = NULL;
+      code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
+      if (code) goto _exit;
+    
+      memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+    }
+
     if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
@@ -720,12 +730,28 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
 
     if (key.version > pMerger->version) {
       if (!COL_VAL_IS_NONE(pColVal)) {
-        taosArraySet(pMerger->pArray, iCol, pColVal);
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol);
+          code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
+          if (code) return code;
+        
+          memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+        } else {
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
       }
     } else if (key.version < pMerger->version) {
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
       if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        taosArraySet(pMerger->pArray, iCol, pColVal);
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol);
+          code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
+          if (code) return code;
+        
+          memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+        } else {
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
       }
     } else {
       ASSERT(0 && "dup versions not allowed");
@@ -765,6 +791,16 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
   // other
   for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
     tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+    if ((!COL_VAL_IS_NONE(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+      uint8_t *pVal = pColVal->value.pData;
+    
+      pColVal->value.pData = NULL;
+      code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
+      if (code) goto _exit;
+    
+      memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+    }
+    
     if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
@@ -776,12 +812,10 @@ _exit:
 }
 
 void tRowMergerClear(SRowMerger *pMerger) { 
-  if (pMerger->merged) {
-    for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
-      SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
-      if (IS_VAR_DATA_TYPE(pTColVal->type)) {
-        tFree(pTColVal->value.pData);
-      }
+  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
     }
   }
 
@@ -802,13 +836,17 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
       if (!COL_VAL_IS_NONE(pColVal)) {
         if (IS_VAR_DATA_TYPE(pColVal->type)) {
           SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+          if (!COL_VAL_IS_NULL(pColVal)) {
+            code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
+            if (code) goto _exit;
 
-          pTColVal->value.pData = NULL;
-          code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
-          if (code) goto _exit;
-
-          pTColVal->value.nData = pColVal->value.nData;
-          memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+            pTColVal->value.nData = pColVal->value.nData;
+            memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+          } else {
+            tFree(pTColVal->value.pData);
+            pTColVal->value.pData = NULL;
+            taosArraySet(pMerger->pArray, iCol, pColVal);
+          }
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
@@ -816,7 +854,21 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
     } else if (key.version < pMerger->version) {
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
       if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        taosArraySet(pMerger->pArray, iCol, pColVal);
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          if (!COL_VAL_IS_NULL(pColVal)) {
+            code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
+            if (code) goto _exit;
+
+            tColVal->value.nData = pColVal->value.nData;
+            memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
+          } else {
+            tFree(tColVal->value.pData);
+            tColVal->value.pData = NULL;
+            taosArraySet(pMerger->pArray, iCol, pColVal);
+          }
+        } else {
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
       }
     } else {
       ASSERT(0);
@@ -824,7 +876,6 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
   }
 
   pMerger->version = key.version;
-  pMerger->merged = true;
 
 _exit:
   return code;

From 91821c9bb45e21ce80d1db1d6190cf0feb7dfca2 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Tue, 10 Jan 2023 09:41:20 +0800
Subject: [PATCH 09/16] test: add cases for ts-2440

---
 tests/system-test/1-insert/time_range_wise.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py
index 3d5c9197d1..df1cc516c5 100644
--- a/tests/system-test/1-insert/time_range_wise.py
+++ b/tests/system-test/1-insert/time_range_wise.py
@@ -600,6 +600,11 @@ class TDTestCase:
         tdLog.printNoPrefix("==========step4:after wal, all check again ")
         self.all_test()
 
+        # add for TS-2440
+        for i in range(self.rows):
+            tdSql.execute("drop database if exists db3 ")
+            tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m")
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

From e244a484f8edb5201eb42bd628032041eea15c96 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 10 Jan 2023 10:49:35 +0800
Subject: [PATCH 10/16] fix(query): add additional check for imem/mem

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 49 +++++++++++++-------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 91690af4c8..b21050b2ae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2333,32 +2333,33 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
                                           SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
+  TSDBROW *pRow = NULL, *piRow = NULL;
   int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN;
-  if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) {
-    return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
-  } else {
-    TSDBROW *pRow = NULL, *piRow = NULL;
-    if (pBlockScanInfo->iter.hasVal) {
-      pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
-    }
-
-    if (pBlockScanInfo->iiter.hasVal) {
-      piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
-    }
-
-    // imem + file + last block
-    if (pBlockScanInfo->iiter.hasVal) {
-      return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader);
-    }
-
-    // mem + file + last block
-    if (pBlockScanInfo->iter.hasVal) {
-      return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader);
-    }
-
-    // files data blocks + last block
-    return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData);
+  if (pBlockScanInfo->iter.hasVal) {
+    pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
   }
+
+  if (pBlockScanInfo->iiter.hasVal) {
+    piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
+  }
+
+  // two levels of mem-table does contain the valid rows
+  if (pRow != NULL && piRow != NULL) {
+    return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader);
+  }
+
+  // imem + file + last block
+  if (pBlockScanInfo->iiter.hasVal) {
+    return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader);
+  }
+
+  // mem + file + last block
+  if (pBlockScanInfo->iter.hasVal) {
+    return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader);
+  }
+
+  // files data blocks + last block
+  return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData);
 }
 
 static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo,

From df6e9631e162ec82be3374e298c456a64e301499 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Tue, 10 Jan 2023 11:23:52 +0800
Subject: [PATCH 11/16] fix: synchronize access within walFsync

---
 source/libs/wal/src/walWrite.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index 51307dc17d..db31692da9 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -635,6 +635,7 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
 }
 
 void walFsync(SWal *pWal, bool forceFsync) {
+  taosThreadMutexLock(&pWal->mutex);
   if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
     wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
     if (taosFsyncFile(pWal->pIdxFile) < 0) {
@@ -647,4 +648,5 @@ void walFsync(SWal *pWal, bool forceFsync) {
              strerror(errno));
     }
   }
+  taosThreadMutexUnlock(&pWal->mutex);
 }

From bfc483aa30ef156fba14eb4c8d8593371249d534 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 10 Jan 2023 11:32:13 +0800
Subject: [PATCH 12/16] fix: row merge flag issue

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 9bc903a0ba..ede1e9a424 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -682,13 +682,13 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
     }
 
     tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
-    if ((!COL_VAL_IS_NONE(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+    if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
       uint8_t *pVal = pColVal->value.pData;
     
       pColVal->value.pData = NULL;
       code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
       if (code) goto _exit;
-    
+
       memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
     }
 
@@ -730,12 +730,14 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
 
     if (key.version > pMerger->version) {
       if (!COL_VAL_IS_NONE(pColVal)) {
-        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+        if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
           SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol);
           code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
           if (code) return code;
-        
+
+          tColVal->value.nData = pColVal->value.nData;
           memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          tColVal->flag = 0;
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
@@ -743,12 +745,13 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
     } else if (key.version < pMerger->version) {
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
       if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        if (IS_VAR_DATA_TYPE(pColVal->type)) {
-          SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol);
+        if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
           code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
           if (code) return code;
-        
+
+          tColVal->value.nData = pColVal->value.nData;        
           memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          tColVal->flag = 0;
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
@@ -791,7 +794,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
   // other
   for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
     tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
-    if ((!COL_VAL_IS_NONE(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+    if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
       uint8_t *pVal = pColVal->value.pData;
     
       pColVal->value.pData = NULL;
@@ -842,6 +845,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
 
             pTColVal->value.nData = pColVal->value.nData;
             memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+            pTColVal->flag = 0;
           } else {
             tFree(pTColVal->value.pData);
             pTColVal->value.pData = NULL;
@@ -861,6 +865,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
 
             tColVal->value.nData = pColVal->value.nData;
             memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
+            tColVal->flag = 0;
           } else {
             tFree(tColVal->value.pData);
             tColVal->value.pData = NULL;

From 459d2932b1727fc3eee0cb529ae25ab2a01914f5 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 10 Jan 2023 13:44:24 +0800
Subject: [PATCH 13/16] fix: memcpy empty value issue

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index ede1e9a424..86adc1dc80 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -689,7 +689,9 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
       code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
       if (code) goto _exit;
 
-      memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+      if (pColVal->value.nData) {
+        memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+      }
     }
 
     if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
@@ -736,7 +738,9 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           if (code) return code;
 
           tColVal->value.nData = pColVal->value.nData;
-          memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          if (pColVal->value.nData) {
+            memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          }
           tColVal->flag = 0;
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
@@ -750,7 +754,9 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           if (code) return code;
 
           tColVal->value.nData = pColVal->value.nData;        
-          memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          if (pColVal->value.nData) {
+            memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+          }
           tColVal->flag = 0;
         } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
@@ -800,8 +806,10 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
       pColVal->value.pData = NULL;
       code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
       if (code) goto _exit;
-    
-      memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+
+      if (pColVal->value.nData) {
+        memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+      }
     }
     
     if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
@@ -844,7 +852,9 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
             if (code) goto _exit;
 
             pTColVal->value.nData = pColVal->value.nData;
-            memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+            if (pTColVal->value.nData) {
+              memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+            }
             pTColVal->flag = 0;
           } else {
             tFree(pTColVal->value.pData);
@@ -864,7 +874,9 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
             if (code) goto _exit;
 
             tColVal->value.nData = pColVal->value.nData;
-            memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
+            if (tColVal->value.nData) {
+              memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
+            }
             tColVal->flag = 0;
           } else {
             tFree(tColVal->value.pData);

From fcbac7236e3b9effb813d2183259b1eb76929a50 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 10 Jan 2023 13:47:18 +0800
Subject: [PATCH 14/16] fix: reset table scan status

---
 source/libs/executor/src/scanoperator.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1d7f27d0cf..eb38299938 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -173,7 +173,7 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro
   if (NULL == *pPage) {
     return NULL;
   }
-  
+
   return (SResultRow*)((char*)(*pPage) + p1->offset);
 }
 
@@ -1729,6 +1729,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     /*resetTableScanInfo(pTSInfo, pWin);*/
     tsdbReaderClose(pTSInfo->base.dataReader);
     pTSInfo->base.dataReader = NULL;
+    pInfo->pTableScanOp->status = OP_OPENED;
 
     pTSInfo->scanTimes = 0;
     pTSInfo->currentGroupId = -1;

From fd0d4bb83031d99807974616d413c367b0317625 Mon Sep 17 00:00:00 2001
From: dapan1121 <72057773+dapan1121@users.noreply.github.com>
Date: Tue, 10 Jan 2023 15:39:16 +0800
Subject: [PATCH 15/16] Update tsdb.h

fix: remove tmp field
---
 source/dnode/vnode/src/inc/tsdb.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 77a3bb7a2f..5a2e462c8c 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -573,7 +573,6 @@ struct STSDBRowIter {
 struct SRowMerger {
   STSchema *pTSchema;
   int64_t   version;
-  bool      merged;
   SArray   *pArray;  // SArray<SColVal>
 };
 

From dfed4e7650dd079b1a18922e10847b9e41f8bd8b Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Wed, 11 Jan 2023 11:32:46 +0800
Subject: [PATCH 16/16] fix: taos-tools deb rpm compn for main (#19489)

* fix: taos-tools deb/rpm compn for main

* fix: update taos-tools 5aa25e9
---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 packaging/release.sh              | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 599b508c93..d01928cfe8 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 94d6895
+        GIT_TAG 5aa25e9
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
diff --git a/packaging/release.sh b/packaging/release.sh
index 7a8a08352f..1dfbf2b112 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -273,7 +273,7 @@ if [ "$osType" != "Darwin" ]; then
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
           ${csudo}./make-taos-tools-deb.sh ${top_dir} \
-            ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
+            ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp}
         fi
       fi
     else
@@ -298,7 +298,7 @@ if [ "$osType" != "Darwin" ]; then
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
           ${csudo}./make-taos-tools-rpm.sh ${top_dir} \
-            ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
+            ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp}
         fi
       fi
     else